diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysis.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysis.scala index b02f4d6a..1bdf2c7b 100755 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysis.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysis.scala @@ -10,8 +10,7 @@ import org.locationtech.jts.geom.Geometry import org.apache.spark.rdd.RDD import org.apache.spark.sql.SparkSession import org.apache.sedona.core.spatialRDD.SpatialRDD -import org.globalforestwatch.features.{CombinedFeatureId, FeatureId, GfwProFeatureId, GridId} -import org.globalforestwatch.grids.GridId.pointGridId +import org.globalforestwatch.features.FeatureId import org.globalforestwatch.summarystats.{Location, NoIntersectionError, SummaryAnalysis, ValidatedLocation} import org.globalforestwatch.util.SpatialJoinRDD import org.apache.spark.storage.StorageLevel @@ -33,40 +32,20 @@ object ForestChangeDiagnosticAnalysis extends SummaryAnalysis { */ def apply( features: RDD[ValidatedLocation[Geometry]], - intermediateResultsRDD: Option[RDD[ValidatedLocation[ForestChangeDiagnosticData]]], fireAlerts: SpatialRDD[Geometry], - saveIntermediateResults: RDD[ValidatedLocation[ForestChangeDiagnosticData]] => Unit, kwargs: Map[String, Any] )(implicit spark: SparkSession): RDD[ValidatedLocation[ForestChangeDiagnosticData]] = { features.persist(StorageLevel.MEMORY_AND_DISK) try { - val diffGridIds: List[GridId] = - if (intermediateResultsRDD.nonEmpty) collectDiffGridIds(features) - else List.empty - - // These records are not covered by diff geometry, they're still valid and can be re-used - val cachedIntermediateResultsRDD = intermediateResultsRDD.map { rdd => - rdd.filter { - case Valid(Location(CombinedFeatureId(fid1, fid2), _)) => - !diffGridIds.contains(fid2) - case Invalid(Location(CombinedFeatureId(fid1, fid2), _)) => - !diffGridIds.contains(fid2) - case _ => - false - } - } - val partialResult: RDD[ValidatedLocation[ForestChangeDiagnosticData]] = { ValidatedWorkflow(features) .flatMap { locationGeometries => - val diffLocations = filterDiffGridCells(locationGeometries, diffGridIds) - val fireCount: RDD[Location[ForestChangeDiagnosticDataLossYearly]] = - fireStats(diffLocations, fireAlerts, spark) + fireStats(locationGeometries, fireAlerts, spark) val locationSummaries: RDD[ValidatedLocation[ForestChangeDiagnosticSummary]] = { - val tmp = diffLocations.map { case Location(id, geom) => Feature(geom, id) } + val tmp = locationGeometries.map { case Location(id, geom) => Feature(geom, id) } // This is where the main analysis happens, in ErrorSummaryRDD.apply(), // which eventually calls into ForestChangeDiagnosticSummary via @@ -111,13 +90,10 @@ object ForestChangeDiagnosticAnalysis extends SummaryAnalysis { .persist(StorageLevel.MEMORY_AND_DISK) } - cachedIntermediateResultsRDD match { - case Some(cachedResults) => - val mergedResults = partialResult.union(cachedResults) - saveIntermediateResults(mergedResults) - combineGridResults(mergedResults) - case None => - combineGridResults(partialResult) + partialResult.map { + case Valid(Location(fid, data)) if data.equals(ForestChangeDiagnosticData.empty) => + Invalid(Location(fid, NoIntersectionError)) + case data => data } } catch { case e: StackOverflowError => @@ -126,65 +102,6 @@ object ForestChangeDiagnosticAnalysis extends SummaryAnalysis { } } - /** Filter only to those rows covered by gridFilter, these are areas where location - * geometries have changed. If gridFilter is empty, all locations will be preserved. - */ - def filterDiffGridCells( - rdd: RDD[Location[Geometry]], - gridFilter: List[GridId] - ): RDD[Location[Geometry]] = { - def keepLocationCell(locationId: Int, geom: Geometry): Boolean = - (locationId >= -1) && (gridFilter.isEmpty || gridFilter.contains(GridId(pointGridId(geom.getCentroid, 1)))) - - rdd.collect { - case Location(gfwFid @ GfwProFeatureId(_, lid), geom) if keepLocationCell(lid, geom) => - val grid = pointGridId(geom.getCentroid, 1) - val fid = CombinedFeatureId(gfwFid, GridId(grid)) - Location(fid, geom) - } - } - - /** Collect lists of GridIds for which diff geometry is present (id=-2) */ - def collectDiffGridIds(rdd: RDD[ValidatedLocation[Geometry]]): List[GridId] = { - // new logic: get ID with new old geom, get grid IDs, join on same grid ID, collect - // IDs where grid geometry is not the same - rdd - .collect { - case Valid(Location(GfwProFeatureId(_, locationId), geom)) if locationId == -1 => - GridId(pointGridId(geom.getCentroid, 1)) - } - .collect - .toList - } - - /** Combine per grid results named by CombinedFeatureId to per location results named by FeatureId Some of the per-grid results fo may - * be Invalid errors. Combining per-grid results will aggregate errors up to Location level. - */ - def combineGridResults( - rdd: RDD[ValidatedLocation[ForestChangeDiagnosticData]] - )(implicit spark: SparkSession): RDD[ValidatedLocation[ForestChangeDiagnosticData]] = { - rdd - .map { - case Valid(Location(CombinedFeatureId(fid, _), data)) => - (fid, Valid(data)) - case Invalid(Location(CombinedFeatureId(fid, _), err)) => - (fid, Invalid(err)) - case Valid(Location(fid, data)) => - (fid, Valid(data)) - case Invalid(Location(fid, err)) => - (fid, Invalid(err)) - } - .reduceByKey(_ combine _) - .map { - case (fid, Valid(data)) if data.equals(ForestChangeDiagnosticData.empty) => - Invalid(Location(fid, NoIntersectionError)) - case (fid, Valid(data)) => - Valid(Location(fid, data.withUpdatedCommodityRisk())) - case (fid, Invalid(err)) => - Invalid(Location(fid, err)) - } - } - def fireStats( featureRDD: RDD[Location[Geometry]], fireAlertRDD: SpatialRDD[Geometry], diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticCommand.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticCommand.scala index 88042e4e..5a6f830b 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticCommand.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticCommand.scala @@ -6,8 +6,6 @@ import cats.implicits._ import com.monovore.decline.Opts import org.globalforestwatch.features._ import com.typesafe.scalalogging.LazyLogging -import org.globalforestwatch.summarystats.ValidatedLocation -import org.apache.spark.rdd.RDD import org.globalforestwatch.config.GfwConfig import org.globalforestwatch.util.Config @@ -17,23 +15,15 @@ object ForestChangeDiagnosticCommand extends SummaryCommand with LazyLogging { val TreeCoverLossYearStart: Int = 2001 val TreeCoverLossYearEnd: Int = 2023 - val intermediateListSourceOpt: Opts[Option[NonEmptyList[String]]] = Opts - .options[String]( - "intermediate_list_source", - help = "URI of intermediate list results in TSV format" - ) - .orNone - val forestChangeDiagnosticCommand: Opts[Unit] = Opts.subcommand( name = ForestChangeDiagnosticAnalysis.name, help = "Compute summary statistics for GFW Pro Forest Change Diagnostic." ) { ( defaultOptions, - intermediateListSourceOpt, requiredFireAlertOptions, featureFilterOptions - ).mapN { (default, intermediateListSource, fireAlert, filterOptions) => + ).mapN { (default, fireAlert, filterOptions) => val kwargs = Map( "outputUrl" -> default.outputUrl, "noOutputPathSuffix" -> default.noOutputPathSuffix, @@ -49,19 +39,9 @@ object ForestChangeDiagnosticCommand extends SummaryCommand with LazyLogging { val featureRDD = ValidatedFeatureRDD(default.featureUris, default.featureType, featureFilter, splitFeatures = true) val fireAlertRDD = FireAlertRDD(spark, fireAlert.alertType, fireAlert.alertSource, FeatureFilter.empty) - val intermediateResultsRDD = intermediateListSource.map { sources => - ForestChangeDiagnosticDF.readIntermidateRDD(sources, spark) - } - val saveIntermidateResults: RDD[ValidatedLocation[ForestChangeDiagnosticData]] => Unit = { rdd => - val df = ForestChangeDiagnosticDF.getGridFeatureDataFrame(rdd, spark) - ForestChangeDiagnosticExport.export("intermediate", df, default.outputUrl, kwargs) - } - val fcdRDD = ForestChangeDiagnosticAnalysis( featureRDD, - intermediateResultsRDD, fireAlertRDD, - saveIntermidateResults, kwargs ) diff --git a/src/test/resources/argbra-fcd-output/part-00000-3d339776-5c3b-4db8-8cac-3d284de2cca7-c000.csv b/src/test/resources/argbra-fcd-output/part-00000-efae4f02-aebc-474b-9532-27a99e03ecaa-c000.csv similarity index 71% rename from src/test/resources/argbra-fcd-output/part-00000-3d339776-5c3b-4db8-8cac-3d284de2cca7-c000.csv rename to src/test/resources/argbra-fcd-output/part-00000-efae4f02-aebc-474b-9532-27a99e03ecaa-c000.csv index b9994490..a688c831 100644 --- a/src/test/resources/argbra-fcd-output/part-00000-3d339776-5c3b-4db8-8cac-3d284de2cca7-c000.csv +++ b/src/test/resources/argbra-fcd-output/part-00000-efae4f02-aebc-474b-9532-27a99e03ecaa-c000.csv @@ -1,2 +1,2 @@ list_id location_id status_code location_error tree_cover_loss_total_yearly tree_cover_loss_primary_forest_yearly tree_cover_loss_peat_yearly tree_cover_loss_intact_forest_yearly tree_cover_loss_protected_areas_yearly tree_cover_loss_by_country_yearly tree_cover_loss_by_country_wdpa_yearly tree_cover_loss_by_country_landmark_yearly tree_cover_loss_by_country_classified_region_yearly tree_cover_loss_arg_otbn_yearly tree_cover_loss_sea_landcover_yearly tree_cover_loss_idn_landcover_yearly tree_cover_loss_soy_yearly tree_cover_loss_idn_legal_yearly tree_cover_loss_idn_forest_moratorium_yearly tree_cover_loss_prodes_yearly tree_cover_loss_prodes_wdpa_yearly tree_cover_loss_prodes_primary_forest_yearly country_area country_specific_deforestation_yearly country_specific_deforestation_wdpa_yearly country_specific_deforestation_landmark_yearly country_specific_deforestation_classified_region_yearly classified_region_area tree_cover_loss_brazil_biomes_yearly tree_cover_extent_total tree_cover_extent_primary_forest tree_cover_extent_protected_areas tree_cover_extent_peat tree_cover_extent_intact_forest natural_habitat_primary natural_habitat_intact_forest total_area protected_areas_area peat_area arg_otbn_area protected_areas_by_category_area landmark_by_category_area brazil_biomes idn_legal_area sea_landcover_area idn_landcover_area idn_forest_moratorium_area south_america_presence legal_amazon_presence brazil_biomes_presence cerrado_biome_presence southeast_asia_presence indonesia_presence argentina_presence commodity_value_forest_extent commodity_value_peat commodity_value_protected_areas commodity_threat_deforestation commodity_threat_peat commodity_threat_protected_areas commodity_threat_fires -166 1 2 {"2001":390.3801,"2002":337.1303,"2003":591.747,"2004":522.3103,"2005":905.0571,"2006":861.6216,"2007":688.9958,"2008":719.6228,"2009":682.3401,"2010":342.0608,"2011":832.8575,"2012":822.2161,"2013":698.7244,"2014":1061.5944,"2015":623.7254,"2016":1506.2833,"2017":2137.3785,"2018":999.0414,"2019":1187.6768,"2020":1380.8694,"2021":1630.7097,"2022":1173.8577,"2023":935.707} {"2001":4.4721,"2002":4.6137,"2003":6.8156,"2004":2.1372,"2005":6.1991,"2006":69.8527,"2007":54.001,"2008":103.172,"2009":68.8725,"2010":25.6512,"2011":40.7394,"2012":61.5105,"2013":77.835,"2014":58.1765,"2015":40.3091,"2016":68.4074,"2017":163.7431,"2018":35.5811,"2019":48.25,"2020":30.8854,"2021":33.2186,"2022":47.9914,"2023":34.3442} {"2001":0.4816,"2002":0.2062,"2003":0.2062,"2004":0.6189,"2005":0.6193,"2006":0.413,"2007":1.3096,"2008":0.3435,"2009":0.6872,"2010":0.619,"2011":2.2689,"2012":1.4448,"2013":1.8558,"2014":5.7042,"2015":1.6496,"2016":2.7495,"2017":11.0043,"2018":2.4757,"2019":3.92,"2020":3.3036,"2021":5.0209,"2022":2.4071,"2023":1.7176} {} {"2001":8.538,"2002":8.7495,"2003":7.5728,"2004":14.4597,"2005":90.2472,"2006":211.8131,"2007":177.1202,"2008":66.799,"2009":69.0354,"2010":10.0578,"2011":39.6777,"2012":35.612,"2013":11.0741,"2014":20.1668,"2015":18.4411,"2016":11.7733,"2017":36.2816,"2018":7.9929,"2019":22.3112,"2020":8.1953,"2021":25.8224,"2022":11.5036,"2023":0.4131} {"ARG":{"2001":169.7492,"2002":110.0137,"2003":279.3034,"2004":243.8448,"2005":503.571,"2006":590.8377,"2007":450.6176,"2008":539.3522,"2009":450.1686,"2010":266.6501,"2011":375.4212,"2012":535.6973,"2013":450.3713,"2014":590.4731,"2015":350.2188,"2016":606.0677,"2017":909.5434,"2018":472.945,"2019":546.4409,"2020":656.0203,"2021":878.4245,"2022":433.7453,"2023":356.2457},"BRA":{"2001":220.6309,"2002":227.1166,"2003":312.4436,"2004":278.4655,"2005":401.4861,"2006":270.7838,"2007":237.4137,"2008":180.2706,"2009":232.1715,"2010":75.4107,"2011":457.3676,"2012":286.5188,"2013":248.2153,"2014":470.7769,"2015":273.5065,"2016":899.9401,"2017":1222.5303,"2018":526.0963,"2019":639.9957,"2020":724.7802,"2021":751.8719,"2022":739.5609,"2023":579.4613}} {"ARG":{"UNESCO-MAB Biosphere Reserve":{"2001":8.538,"2002":8.7495,"2003":7.5728,"2004":14.4597,"2005":90.2472,"2006":211.8131,"2007":177.1202,"2008":66.799,"2009":69.0354,"2010":10.0578,"2011":39.6089,"2012":35.5432,"2013":10.9364,"2014":20.0291,"2015":18.4411,"2016":11.5667,"2017":36.1438,"2018":7.9929,"2019":22.3112,"2020":8.1264,"2021":25.409,"2022":11.5036,"2023":0.4131}}} {"ARG":{"Not Reported":{"2001":39.904,"2002":36.1895,"2003":118.8314,"2004":99.1471,"2005":211.2132,"2006":230.4318,"2007":162.2055,"2008":142.2109,"2009":130.0801,"2010":68.1056,"2011":108.2244,"2012":241.5664,"2013":127.9175,"2014":210.2876,"2015":139.705,"2016":247.6686,"2017":361.3402,"2018":201.2981,"2019":248.205,"2020":277.617,"2021":433.2285,"2022":149.1649,"2023":117.5902}}} {"ARG":{"Category I":{"2001":1.0996,"2002":0.8246,"2003":2.1303,"2004":0.0,"2005":0.9622,"2006":0.55,"2007":0.0,"2008":0.3437,"2009":3.7798,"2010":0.0,"2011":0.5498,"2012":0.4125,"2013":1.6495,"2014":1.5121,"2015":6.3928,"2016":1.6495,"2017":2.0618,"2018":0.1375,"2019":0.6873,"2020":1.5121,"2021":2.8869,"2022":0.2064,"2023":0.0},"Category II":{"2001":15.5017,"2002":13.3015,"2003":31.5663,"2004":34.1258,"2005":67.4062,"2006":55.745,"2007":46.5984,"2008":73.8253,"2009":131.6487,"2010":73.2082,"2011":100.0926,"2012":133.2506,"2013":130.0616,"2014":156.9412,"2015":106.1384,"2016":151.6383,"2017":269.1299,"2018":137.5692,"2019":158.3015,"2020":163.4794,"2021":243.9769,"2022":130.3351,"2023":128.5559},"Category III":{"2001":26.2709,"2002":13.8555,"2003":42.6008,"2004":23.2359,"2005":54.4591,"2006":44.0493,"2007":27.9229,"2008":52.6735,"2009":78.9423,"2010":73.1421,"2011":68.5956,"2012":112.4446,"2013":151.5104,"2014":217.1291,"2015":113.4622,"2016":201.8612,"2017":294.2968,"2018":150.2793,"2019":194.1712,"2020":223.0738,"2021":300.5095,"2022":110.6482,"2023":98.4466}}} {"Category I":{"2001":1.0996,"2002":0.8246,"2003":2.1303,"2004":0.0,"2005":0.9622,"2006":0.55,"2007":0.0,"2008":0.3437,"2009":3.7798,"2010":0.0,"2011":0.5498,"2012":0.4125,"2013":1.6495,"2014":1.5121,"2015":6.3928,"2016":1.6495,"2017":2.0618,"2018":0.1375,"2019":0.6873,"2020":1.5121,"2021":2.8869,"2022":0.2064,"2023":0.0},"Category II":{"2001":15.5017,"2002":13.3015,"2003":31.5663,"2004":34.1258,"2005":67.4062,"2006":55.745,"2007":46.5984,"2008":73.8253,"2009":131.6487,"2010":73.2082,"2011":100.0926,"2012":133.2506,"2013":130.0616,"2014":157.0101,"2015":106.1384,"2016":151.6383,"2017":269.1299,"2018":137.5692,"2019":158.3015,"2020":163.4794,"2021":244.0458,"2022":130.3351,"2023":128.5559},"Category III":{"2001":26.2709,"2002":13.8555,"2003":42.6008,"2004":23.2359,"2005":54.4591,"2006":44.0493,"2007":27.9229,"2008":52.6735,"2009":78.9423,"2010":73.1421,"2011":68.5956,"2012":112.4446,"2013":151.5104,"2014":217.1291,"2015":113.4622,"2016":201.8612,"2017":294.2968,"2018":150.2793,"2019":194.1712,"2020":223.0738,"2021":300.5095,"2022":110.6482,"2023":98.4466}} {} {} {"2001":21.624,"2002":20.8704,"2003":35.0478,"2004":35.6016,"2005":35.1084,"2006":29.4707,"2007":23.7689,"2008":26.0851,"2009":23.4664,"2010":9.7672,"2011":76.1005,"2012":28.4983,"2013":18.1678,"2014":49.4618,"2015":21.3388,"2016":63.0798,"2017":82.2563,"2018":30.9672,"2019":32.1454,"2020":38.9536,"2021":3.9932,"2022":1.102} {} {} {} {} {} {"BRA":221184.6524,"ARG":194985.6634} {"ARG":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":186.9026,"2008":0.0,"2009":0.0,"2010approx":417.5112,"2011":15.1053,"2012":0.0,"2013approx":117.2137,"2014":79.0631,"2015":3.5847,"2016":8.0709,"2017":163.5411,"2018":31.4482,"2019":47.2747,"2020":181.2011,"2021":110.5264,"2022":18.9678,"2023":0.0}} {"ARG":{"UNESCO-MAB Biosphere Reserve":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.0,"2008":0.0,"2009":0.0,"2010approx":1.0342,"2011":0.0,"2012":0.0,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":17.5539,"2020":23.7832,"2021":0.0,"2022":0.0,"2023":0.0}}} {"ARG":{"Not Reported":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":92.4229,"2008":0.0,"2009":0.0,"2010approx":46.7539,"2011":0.0,"2012":0.0,"2013approx":20.4653,"2014":46.5229,"2015":0.0,"2016":0.0,"2017":44.9934,"2018":5.5842,"2019":31.0673,"2020":76.3528,"2021":21.096,"2022":0.0,"2023":0.0}}} {"ARG":{"Category I":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.0,"2008":0.0,"2009":0.0,"2010":0.0,"2011":0.0,"2012":0.0,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":0.0,"2020":21.3774,"2021":0.0,"2022":0.0,"2023":0.0},"Category III":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":14.683,"2008":0.0,"2009":0.0,"2010approx":75.2363,"2011":12.4841,"2012":0.0,"2013approx":30.8262,"2014":39.6991,"2015":0.0,"2016":6.8293,"2017":94.9476,"2018":22.8267,"2019":13.5189,"2020":40.3374,"2021":34.8144,"2022":10.8284,"2023":0.0},"Category II":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":25.0997,"2008":0.0,"2009":0.0,"2010approx":174.5175,"2011":1.7935,"2012":0.0,"2013approx":71.4984,"2014":27.7832,"2015":3.5847,"2016":1.2417,"2017":54.5984,"2018":8.3456,"2019":32.4472,"2020":73.3842,"2021":66.2684,"2022":7.1053,"2023":0.0}}} {"ARG":{"Category II":122991.3614,"Category I":30089.6869,"Category III":11275.7574}} {"Mata Atlântica":{"2001":222.281,"2002":229.664,"2003":313.889,"2004":281.566,"2005":404.1011,"2006":274.0218,"2007":244.2288,"2008":182.4026,"2009":237.3952,"2010":77.4035,"2011":460.1882,"2012":303.0526,"2013":257.9101,"2014":491.4761,"2015":279.4206,"2016":909.6394,"2017":1238.9614,"2018":529.7395,"2019":649.9751,"2020":733.7938,"2021":765.01,"2022":743.0739,"2023":584.0024}} 310497.0062 151998.5232 145836.3381 3133.6034 0.0 152001.8248 0.0 416412.5376 145907.1635 3633.9932 {"Category II":123789.927,"Category I":30089.6869,"Category III":11275.7574} {"UNESCO-MAB Biosphere Reserve":144721.3787} {"Not Reported":91313.3874} {"Mata Atlântica":235455.6349} {} {} {} 0.0 true false true false false false true {"2001":228399.0302,"2002":228399.0302,"2003":228218.695,"2004":228019.807,"2005":227669.5127,"2006":227360.6322,"2007":226793.1945,"2008":226294.0263,"2009":225908.1898,"2010":225416.7831,"2011":224990.0743,"2012":224720.0969,"2013":224202.2054,"2014":223703.987,"2015":223262.4402,"2016":222610.9217,"2017":222244.0711,"2018":221464.0632,"2019":220326.1549,"2020":219785.4113,"2021":219159.8818,"2022":218487.8012} {"2001":3633.9932,"2002":3633.9932,"2003":3633.9932,"2004":3633.9932,"2005":3633.9932,"2006":3633.9932,"2007":3633.9932,"2008":3633.9932,"2009":3633.9932,"2010":3633.9932,"2011":3633.9932,"2012":3633.9932,"2013":3633.9932,"2014":3633.9932,"2015":3633.9932,"2016":3633.9932,"2017":3633.9932,"2018":3633.9932,"2019":3633.9932,"2020":3633.9932,"2021":3633.9932,"2022":3633.9932} {"2001":145907.1635,"2002":145907.1635,"2003":145907.1635,"2004":145907.1635,"2005":145907.1635,"2006":145907.1635,"2007":145907.1635,"2008":145907.1635,"2009":145907.1635,"2010":145907.1635,"2011":145907.1635,"2012":145907.1635,"2013":145907.1635,"2014":145907.1635,"2015":145907.1635,"2016":145907.1635,"2017":145907.1635,"2018":145907.1635,"2019":145907.1635,"2020":145907.1635,"2021":145907.1635,"2022":145907.1635} {"2002":379.2232,"2003":549.1823,"2004":659.1748,"2005":876.3182,"2006":1066.606,"2007":885.0047,"2008":877.2432,"2009":918.1155,"2010":696.6862,"2011":787.8689,"2012":1016.1099,"2013":939.7652,"2014":1093.0653,"2015":1018.3691,"2016":1146.8585,"2017":1917.9162,"2018":1678.6519,"2019":1166.2731,"2020":1297.6101,"2021":1606.9852,"2022":1513.9363} {"2002":20.9771,"2003":20.5645,"2004":20.8397,"2005":21.1152,"2006":21.1155,"2007":21.9433,"2008":21.8738,"2009":21.0455,"2010":21.321,"2011":21.8711,"2012":22.0087,"2013":21.9396,"2014":22.4891,"2015":22.2141,"2016":22.4208,"2017":26.4784,"2018":25.7222,"2019":22.0779,"2020":22.2157,"2021":23.3861,"2022":23.3168} {"2002":1412.3095,"2003":1411.9654,"2004":1409.5569,"2005":1412.3853,"2006":1422.5074,"2007":1433.1795,"2008":1430.6331,"2009":1426.769,"2010":1421.6073,"2011":1416.8048,"2012":1421.3469,"2013":1419.7406,"2014":1420.2105,"2015":1426.2686,"2016":1424.1418,"2017":1427.8639,"2018":1422.7713,"2019":1422.0236,"2020":1426.6357,"2021":1427.8733,"2022":1431.9397} {} +166 1 2 {"2001":390.3801,"2002":337.1303,"2003":591.747,"2004":522.3103,"2005":905.0571,"2006":861.6216,"2007":688.9958,"2008":719.6228,"2009":682.3401,"2010":342.0608,"2011":832.8575,"2012":822.2161,"2013":698.7244,"2014":1061.5944,"2015":623.7254,"2016":1506.2833,"2017":2137.3785,"2018":999.0414,"2019":1187.6768,"2020":1380.8694,"2021":1630.7097,"2022":1173.8577,"2023":935.707} {"2001":4.4721,"2002":4.6137,"2003":6.8156,"2004":2.1372,"2005":6.1991,"2006":69.8527,"2007":54.001,"2008":103.172,"2009":68.8725,"2010":25.6512,"2011":40.7394,"2012":61.5105,"2013":77.835,"2014":58.1765,"2015":40.3091,"2016":68.4074,"2017":163.7431,"2018":35.5811,"2019":48.25,"2020":30.8854,"2021":33.2186,"2022":47.9914,"2023":34.3442} {"2001":0.4816,"2002":0.2062,"2003":0.2062,"2004":0.6189,"2005":0.6193,"2006":0.413,"2007":1.3096,"2008":0.3435,"2009":0.6872,"2010":0.619,"2011":2.2689,"2012":1.4448,"2013":1.8558,"2014":5.7042,"2015":1.6496,"2016":2.7495,"2017":11.0043,"2018":2.4757,"2019":3.92,"2020":3.3036,"2021":5.0209,"2022":2.4071,"2023":1.7176} {} {"2001":8.538,"2002":8.7495,"2003":7.5728,"2004":14.4597,"2005":90.2472,"2006":211.8131,"2007":177.1202,"2008":66.799,"2009":69.0354,"2010":10.0578,"2011":39.6777,"2012":35.612,"2013":11.0741,"2014":20.1668,"2015":18.4411,"2016":11.7733,"2017":36.2816,"2018":7.9929,"2019":22.3112,"2020":8.1953,"2021":25.8224,"2022":11.5036,"2023":0.4131} {"ARG":{"2001":169.7492,"2002":110.0137,"2003":279.3034,"2004":243.8448,"2005":503.571,"2006":590.8377,"2007":450.6176,"2008":539.3522,"2009":450.1686,"2010":266.6501,"2011":375.4212,"2012":535.6973,"2013":450.3713,"2014":590.4731,"2015":350.2188,"2016":606.0677,"2017":909.5434,"2018":472.945,"2019":546.4409,"2020":656.0203,"2021":878.4245,"2022":433.7453,"2023":356.2457},"BRA":{"2001":220.6309,"2002":227.1166,"2003":312.4436,"2004":278.4655,"2005":401.4861,"2006":270.7838,"2007":237.4137,"2008":180.2706,"2009":232.1715,"2010":75.4107,"2011":457.3676,"2012":286.5188,"2013":248.2153,"2014":470.7769,"2015":273.5065,"2016":899.9401,"2017":1222.5303,"2018":526.0963,"2019":639.9957,"2020":724.7802,"2021":751.8719,"2022":739.5609,"2023":579.4613}} {"ARG":{"UNESCO-MAB Biosphere Reserve":{"2001":8.538,"2002":8.7495,"2003":7.5728,"2004":14.4597,"2005":90.2472,"2006":211.8131,"2007":177.1202,"2008":66.799,"2009":69.0354,"2010":10.0578,"2011":39.6089,"2012":35.5432,"2013":10.9364,"2014":20.0291,"2015":18.4411,"2016":11.5667,"2017":36.1438,"2018":7.9929,"2019":22.3112,"2020":8.1264,"2021":25.409,"2022":11.5036,"2023":0.4131}}} {"ARG":{"Not Reported":{"2001":39.904,"2002":36.1895,"2003":118.8314,"2004":99.1471,"2005":211.2132,"2006":230.4318,"2007":162.2055,"2008":142.2109,"2009":130.0801,"2010":68.1056,"2011":108.2244,"2012":241.5664,"2013":127.9175,"2014":210.2876,"2015":139.705,"2016":247.6686,"2017":361.3402,"2018":201.2981,"2019":248.205,"2020":277.617,"2021":433.2285,"2022":149.1649,"2023":117.5902}}} {"ARG":{"Category II":{"2001":15.5017,"2002":13.3015,"2003":31.5663,"2004":34.1258,"2005":67.4062,"2006":55.745,"2007":46.5984,"2008":73.8253,"2009":131.6487,"2010":73.2082,"2011":100.0926,"2012":133.2506,"2013":130.0616,"2014":156.9412,"2015":106.1384,"2016":151.6383,"2017":269.1299,"2018":137.5692,"2019":158.3015,"2020":163.4794,"2021":243.9769,"2022":130.3351,"2023":128.5559},"Category III":{"2001":26.2709,"2002":13.8555,"2003":42.6008,"2004":23.2359,"2005":54.4591,"2006":44.0493,"2007":27.9229,"2008":52.6735,"2009":78.9423,"2010":73.1421,"2011":68.5956,"2012":112.4446,"2013":151.5104,"2014":217.1291,"2015":113.4622,"2016":201.8612,"2017":294.2968,"2018":150.2793,"2019":194.1712,"2020":223.0738,"2021":300.5095,"2022":110.6482,"2023":98.4466},"Category I":{"2001":1.0996,"2002":0.8246,"2003":2.1303,"2004":0.0,"2005":0.9622,"2006":0.55,"2007":0.0,"2008":0.3437,"2009":3.7798,"2010":0.0,"2011":0.5498,"2012":0.4125,"2013":1.6495,"2014":1.5121,"2015":6.3928,"2016":1.6495,"2017":2.0618,"2018":0.1375,"2019":0.6873,"2020":1.5121,"2021":2.8869,"2022":0.2064,"2023":0.0}}} {"Category II":{"2001":15.5017,"2002":13.3015,"2003":31.5663,"2004":34.1258,"2005":67.4062,"2006":55.745,"2007":46.5984,"2008":73.8253,"2009":131.6487,"2010":73.2082,"2011":100.0926,"2012":133.2506,"2013":130.0616,"2014":157.0101,"2015":106.1384,"2016":151.6383,"2017":269.1299,"2018":137.5692,"2019":158.3015,"2020":163.4794,"2021":244.0458,"2022":130.3351,"2023":128.5559},"Category III":{"2001":26.2709,"2002":13.8555,"2003":42.6008,"2004":23.2359,"2005":54.4591,"2006":44.0493,"2007":27.9229,"2008":52.6735,"2009":78.9423,"2010":73.1421,"2011":68.5956,"2012":112.4446,"2013":151.5104,"2014":217.1291,"2015":113.4622,"2016":201.8612,"2017":294.2968,"2018":150.2793,"2019":194.1712,"2020":223.0738,"2021":300.5095,"2022":110.6482,"2023":98.4466},"Category I":{"2001":1.0996,"2002":0.8246,"2003":2.1303,"2004":0.0,"2005":0.9622,"2006":0.55,"2007":0.0,"2008":0.3437,"2009":3.7798,"2010":0.0,"2011":0.5498,"2012":0.4125,"2013":1.6495,"2014":1.5121,"2015":6.3928,"2016":1.6495,"2017":2.0618,"2018":0.1375,"2019":0.6873,"2020":1.5121,"2021":2.8869,"2022":0.2064,"2023":0.0}} {} {} {"2001":21.624,"2002":20.8704,"2003":35.0478,"2004":35.6016,"2005":35.1084,"2006":29.4707,"2007":23.7689,"2008":26.0851,"2009":23.4664,"2010":9.7672,"2011":76.1005,"2012":28.4983,"2013":18.1678,"2014":49.4618,"2015":21.3388,"2016":63.0798,"2017":82.2563,"2018":30.9672,"2019":32.1454,"2020":38.9536,"2021":3.9932,"2022":1.102} {} {} {} {} {} {"ARG":194985.6634,"BRA":221184.6524} {"ARG":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":186.9026,"2008":0.0,"2009":0.0,"2010approx":417.5112,"2011":15.1053,"2012":0.0,"2013approx":117.2137,"2014":79.0631,"2015":3.5847,"2016":8.0709,"2017":163.5411,"2018":31.4482,"2019":47.2747,"2020":181.2011,"2021":110.5264,"2022":18.9678,"2023":0.0}} {"ARG":{"UNESCO-MAB Biosphere Reserve":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.0,"2008":0.0,"2009":0.0,"2010approx":1.0342,"2011":0.0,"2012":0.0,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":17.5539,"2020":23.7832,"2021":0.0,"2022":0.0,"2023":0.0}}} {"ARG":{"Not Reported":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":92.4229,"2008":0.0,"2009":0.0,"2010approx":46.7539,"2011":0.0,"2012":0.0,"2013approx":20.4653,"2014":46.5229,"2015":0.0,"2016":0.0,"2017":44.9934,"2018":5.5842,"2019":31.0673,"2020":76.3528,"2021":21.096,"2022":0.0,"2023":0.0}}} {"ARG":{"Category III":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":14.683,"2008":0.0,"2009":0.0,"2010approx":75.2363,"2011":12.4841,"2012":0.0,"2013approx":30.8262,"2014":39.6991,"2015":0.0,"2016":6.8293,"2017":94.9476,"2018":22.8267,"2019":13.5189,"2020":40.3374,"2021":34.8144,"2022":10.8284,"2023":0.0},"Category II":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":25.0997,"2008":0.0,"2009":0.0,"2010approx":174.5175,"2011":1.7935,"2012":0.0,"2013approx":71.4984,"2014":27.7832,"2015":3.5847,"2016":1.2417,"2017":54.5984,"2018":8.3456,"2019":32.4472,"2020":73.3842,"2021":66.2684,"2022":7.1053,"2023":0.0},"Category I":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.0,"2008":0.0,"2009":0.0,"2010":0.0,"2011":0.0,"2012":0.0,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":0.0,"2020":21.3774,"2021":0.0,"2022":0.0,"2023":0.0}}} {"ARG":{"Category II":122991.3614,"Category III":11275.7574,"Category I":30089.6869}} {"Mata Atlântica":{"2001":222.281,"2002":229.664,"2003":313.889,"2004":281.566,"2005":404.1011,"2006":274.0218,"2007":244.2288,"2008":182.4026,"2009":237.3952,"2010":77.4035,"2011":460.1882,"2012":303.0526,"2013":257.9101,"2014":491.4761,"2015":279.4206,"2016":909.6394,"2017":1238.9614,"2018":529.7395,"2019":649.9751,"2020":733.7938,"2021":765.01,"2022":743.0739,"2023":584.0024}} 310497.0062 151998.5232 145836.3381 3133.6034 0.0 152001.8248 0.0 416412.5376 145907.1635 3633.9932 {"Category II":123789.927,"Category III":11275.7574,"Category I":30089.6869} {"UNESCO-MAB Biosphere Reserve":144721.3787} {"Not Reported":91313.3874} {"Mata Atlântica":235455.6349} {} {} {} 0.0 true false true false false false true {"2001":228399.0302,"2002":228399.0302,"2003":228218.695,"2004":228019.807,"2005":227669.5127,"2006":227360.6322,"2007":226793.1945,"2008":226294.0263,"2009":225908.1898,"2010":225416.7831,"2011":224990.0743,"2012":224720.0969,"2013":224202.2054,"2014":223703.987,"2015":223262.4402,"2016":222610.9217,"2017":222244.0711,"2018":221464.0632,"2019":220326.1549,"2020":219785.4113,"2021":219159.8818,"2022":218487.8012} {"2001":3633.9932,"2002":3633.9932,"2003":3633.9932,"2004":3633.9932,"2005":3633.9932,"2006":3633.9932,"2007":3633.9932,"2008":3633.9932,"2009":3633.9932,"2010":3633.9932,"2011":3633.9932,"2012":3633.9932,"2013":3633.9932,"2014":3633.9932,"2015":3633.9932,"2016":3633.9932,"2017":3633.9932,"2018":3633.9932,"2019":3633.9932,"2020":3633.9932,"2021":3633.9932,"2022":3633.9932} {"2001":145907.1635,"2002":145907.1635,"2003":145907.1635,"2004":145907.1635,"2005":145907.1635,"2006":145907.1635,"2007":145907.1635,"2008":145907.1635,"2009":145907.1635,"2010":145907.1635,"2011":145907.1635,"2012":145907.1635,"2013":145907.1635,"2014":145907.1635,"2015":145907.1635,"2016":145907.1635,"2017":145907.1635,"2018":145907.1635,"2019":145907.1635,"2020":145907.1635,"2021":145907.1635,"2022":145907.1635} {"2002":379.2232,"2003":549.1823,"2004":659.1748,"2005":876.3182,"2006":1066.606,"2007":885.0047,"2008":877.2432,"2009":918.1155,"2010":696.6862,"2011":787.8689,"2012":1016.1099,"2013":939.7652,"2014":1093.0653,"2015":1018.3691,"2016":1146.8585,"2017":1917.9162,"2018":1678.6519,"2019":1166.2731,"2020":1297.6101,"2021":1606.9852,"2022":1513.9363} {"2002":20.9771,"2003":20.5645,"2004":20.8397,"2005":21.1152,"2006":21.1155,"2007":21.9433,"2008":21.8738,"2009":21.0455,"2010":21.321,"2011":21.8711,"2012":22.0087,"2013":21.9396,"2014":22.4891,"2015":22.2141,"2016":22.4208,"2017":26.4784,"2018":25.7222,"2019":22.0779,"2020":22.2157,"2021":23.3861,"2022":23.3168} {"2002":1412.3095,"2003":1411.9654,"2004":1409.5569,"2005":1412.3853,"2006":1422.5074,"2007":1433.1795,"2008":1430.6331,"2009":1426.769,"2010":1421.6073,"2011":1416.8048,"2012":1421.3469,"2013":1419.7406,"2014":1420.2105,"2015":1426.2686,"2016":1424.1418,"2017":1427.8639,"2018":1422.7713,"2019":1422.0236,"2020":1426.6357,"2021":1427.8733,"2022":1431.9397} {} diff --git a/src/test/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysisSpec.scala b/src/test/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysisSpec.scala index 294cff40..86f5fffb 100644 --- a/src/test/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysisSpec.scala +++ b/src/test/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysisSpec.scala @@ -35,9 +35,7 @@ class ForestChangeDiagnosticAnalysisSpec extends TestEnvironment with DataFrameC ForestChangeDiagnosticAnalysis( features, - intermediateResultsRDD = None, fireAlerts = fireAlertsRdd, - saveIntermediateResults = identity, kwargs = Map("config" -> GfwConfig.get())) }