From f9c27da6f1dc2b7dafff75892feaca4381276624 Mon Sep 17 00:00:00 2001 From: Dan Scales Date: Thu, 11 Jul 2024 11:09:42 -0700 Subject: [PATCH] GTC-2889 Remove code in FCD related to geometry diffs and intermediate results We've been ignoring geometry diffs (location id == -2) for a long time, and we never pass in any intermediate results. By removing this code, we avoid computing the centroid of each location geometry, inserting it into the feature id, then removing it from the feature id without having used it at all. Move the only remaining code from combineGridResults() into the main function. Also, we had a duplicate call to data.withUpdatedCommodityRisk() in combineGridResults(), so got rid of that (already called data.withUpdatedCommodityRisk() in main analysis code above). The test change was just a change in order of categories within a single result, which is no actual change of the results. --- .../ForestChangeDiagnosticAnalysis.scala | 97 ++----------------- .../ForestChangeDiagnosticCommand.scala | 22 +---- ...4f02-aebc-474b-9532-27a99e03ecaa-c000.csv} | 2 +- .../ForestChangeDiagnosticAnalysisSpec.scala | 2 - 4 files changed, 9 insertions(+), 114 deletions(-) rename src/test/resources/argbra-fcd-output/{part-00000-3d339776-5c3b-4db8-8cac-3d284de2cca7-c000.csv => part-00000-efae4f02-aebc-474b-9532-27a99e03ecaa-c000.csv} (71%) diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysis.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysis.scala index b02f4d6a..1bdf2c7b 100755 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysis.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysis.scala @@ -10,8 +10,7 @@ import org.locationtech.jts.geom.Geometry import org.apache.spark.rdd.RDD import org.apache.spark.sql.SparkSession import org.apache.sedona.core.spatialRDD.SpatialRDD -import org.globalforestwatch.features.{CombinedFeatureId, FeatureId, GfwProFeatureId, GridId} -import org.globalforestwatch.grids.GridId.pointGridId +import org.globalforestwatch.features.FeatureId import org.globalforestwatch.summarystats.{Location, NoIntersectionError, SummaryAnalysis, ValidatedLocation} import org.globalforestwatch.util.SpatialJoinRDD import org.apache.spark.storage.StorageLevel @@ -33,40 +32,20 @@ object ForestChangeDiagnosticAnalysis extends SummaryAnalysis { */ def apply( features: RDD[ValidatedLocation[Geometry]], - intermediateResultsRDD: Option[RDD[ValidatedLocation[ForestChangeDiagnosticData]]], fireAlerts: SpatialRDD[Geometry], - saveIntermediateResults: RDD[ValidatedLocation[ForestChangeDiagnosticData]] => Unit, kwargs: Map[String, Any] )(implicit spark: SparkSession): RDD[ValidatedLocation[ForestChangeDiagnosticData]] = { features.persist(StorageLevel.MEMORY_AND_DISK) try { - val diffGridIds: List[GridId] = - if (intermediateResultsRDD.nonEmpty) collectDiffGridIds(features) - else List.empty - - // These records are not covered by diff geometry, they're still valid and can be re-used - val cachedIntermediateResultsRDD = intermediateResultsRDD.map { rdd => - rdd.filter { - case Valid(Location(CombinedFeatureId(fid1, fid2), _)) => - !diffGridIds.contains(fid2) - case Invalid(Location(CombinedFeatureId(fid1, fid2), _)) => - !diffGridIds.contains(fid2) - case _ => - false - } - } - val partialResult: RDD[ValidatedLocation[ForestChangeDiagnosticData]] = { ValidatedWorkflow(features) .flatMap { locationGeometries => - val diffLocations = filterDiffGridCells(locationGeometries, diffGridIds) - val fireCount: RDD[Location[ForestChangeDiagnosticDataLossYearly]] = - fireStats(diffLocations, fireAlerts, spark) + fireStats(locationGeometries, fireAlerts, spark) val locationSummaries: RDD[ValidatedLocation[ForestChangeDiagnosticSummary]] = { - val tmp = diffLocations.map { case Location(id, geom) => Feature(geom, id) } + val tmp = locationGeometries.map { case Location(id, geom) => Feature(geom, id) } // This is where the main analysis happens, in ErrorSummaryRDD.apply(), // which eventually calls into ForestChangeDiagnosticSummary via @@ -111,13 +90,10 @@ object ForestChangeDiagnosticAnalysis extends SummaryAnalysis { .persist(StorageLevel.MEMORY_AND_DISK) } - cachedIntermediateResultsRDD match { - case Some(cachedResults) => - val mergedResults = partialResult.union(cachedResults) - saveIntermediateResults(mergedResults) - combineGridResults(mergedResults) - case None => - combineGridResults(partialResult) + partialResult.map { + case Valid(Location(fid, data)) if data.equals(ForestChangeDiagnosticData.empty) => + Invalid(Location(fid, NoIntersectionError)) + case data => data } } catch { case e: StackOverflowError => @@ -126,65 +102,6 @@ object ForestChangeDiagnosticAnalysis extends SummaryAnalysis { } } - /** Filter only to those rows covered by gridFilter, these are areas where location - * geometries have changed. If gridFilter is empty, all locations will be preserved. - */ - def filterDiffGridCells( - rdd: RDD[Location[Geometry]], - gridFilter: List[GridId] - ): RDD[Location[Geometry]] = { - def keepLocationCell(locationId: Int, geom: Geometry): Boolean = - (locationId >= -1) && (gridFilter.isEmpty || gridFilter.contains(GridId(pointGridId(geom.getCentroid, 1)))) - - rdd.collect { - case Location(gfwFid @ GfwProFeatureId(_, lid), geom) if keepLocationCell(lid, geom) => - val grid = pointGridId(geom.getCentroid, 1) - val fid = CombinedFeatureId(gfwFid, GridId(grid)) - Location(fid, geom) - } - } - - /** Collect lists of GridIds for which diff geometry is present (id=-2) */ - def collectDiffGridIds(rdd: RDD[ValidatedLocation[Geometry]]): List[GridId] = { - // new logic: get ID with new old geom, get grid IDs, join on same grid ID, collect - // IDs where grid geometry is not the same - rdd - .collect { - case Valid(Location(GfwProFeatureId(_, locationId), geom)) if locationId == -1 => - GridId(pointGridId(geom.getCentroid, 1)) - } - .collect - .toList - } - - /** Combine per grid results named by CombinedFeatureId to per location results named by FeatureId Some of the per-grid results fo may - * be Invalid errors. Combining per-grid results will aggregate errors up to Location level. - */ - def combineGridResults( - rdd: RDD[ValidatedLocation[ForestChangeDiagnosticData]] - )(implicit spark: SparkSession): RDD[ValidatedLocation[ForestChangeDiagnosticData]] = { - rdd - .map { - case Valid(Location(CombinedFeatureId(fid, _), data)) => - (fid, Valid(data)) - case Invalid(Location(CombinedFeatureId(fid, _), err)) => - (fid, Invalid(err)) - case Valid(Location(fid, data)) => - (fid, Valid(data)) - case Invalid(Location(fid, err)) => - (fid, Invalid(err)) - } - .reduceByKey(_ combine _) - .map { - case (fid, Valid(data)) if data.equals(ForestChangeDiagnosticData.empty) => - Invalid(Location(fid, NoIntersectionError)) - case (fid, Valid(data)) => - Valid(Location(fid, data.withUpdatedCommodityRisk())) - case (fid, Invalid(err)) => - Invalid(Location(fid, err)) - } - } - def fireStats( featureRDD: RDD[Location[Geometry]], fireAlertRDD: SpatialRDD[Geometry], diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticCommand.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticCommand.scala index 88042e4e..5a6f830b 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticCommand.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticCommand.scala @@ -6,8 +6,6 @@ import cats.implicits._ import com.monovore.decline.Opts import org.globalforestwatch.features._ import com.typesafe.scalalogging.LazyLogging -import org.globalforestwatch.summarystats.ValidatedLocation -import org.apache.spark.rdd.RDD import org.globalforestwatch.config.GfwConfig import org.globalforestwatch.util.Config @@ -17,23 +15,15 @@ object ForestChangeDiagnosticCommand extends SummaryCommand with LazyLogging { val TreeCoverLossYearStart: Int = 2001 val TreeCoverLossYearEnd: Int = 2023 - val intermediateListSourceOpt: Opts[Option[NonEmptyList[String]]] = Opts - .options[String]( - "intermediate_list_source", - help = "URI of intermediate list results in TSV format" - ) - .orNone - val forestChangeDiagnosticCommand: Opts[Unit] = Opts.subcommand( name = ForestChangeDiagnosticAnalysis.name, help = "Compute summary statistics for GFW Pro Forest Change Diagnostic." ) { ( defaultOptions, - intermediateListSourceOpt, requiredFireAlertOptions, featureFilterOptions - ).mapN { (default, intermediateListSource, fireAlert, filterOptions) => + ).mapN { (default, fireAlert, filterOptions) => val kwargs = Map( "outputUrl" -> default.outputUrl, "noOutputPathSuffix" -> default.noOutputPathSuffix, @@ -49,19 +39,9 @@ object ForestChangeDiagnosticCommand extends SummaryCommand with LazyLogging { val featureRDD = ValidatedFeatureRDD(default.featureUris, default.featureType, featureFilter, splitFeatures = true) val fireAlertRDD = FireAlertRDD(spark, fireAlert.alertType, fireAlert.alertSource, FeatureFilter.empty) - val intermediateResultsRDD = intermediateListSource.map { sources => - ForestChangeDiagnosticDF.readIntermidateRDD(sources, spark) - } - val saveIntermidateResults: RDD[ValidatedLocation[ForestChangeDiagnosticData]] => Unit = { rdd => - val df = ForestChangeDiagnosticDF.getGridFeatureDataFrame(rdd, spark) - ForestChangeDiagnosticExport.export("intermediate", df, default.outputUrl, kwargs) - } - val fcdRDD = ForestChangeDiagnosticAnalysis( featureRDD, - intermediateResultsRDD, fireAlertRDD, - saveIntermidateResults, kwargs ) diff --git a/src/test/resources/argbra-fcd-output/part-00000-3d339776-5c3b-4db8-8cac-3d284de2cca7-c000.csv b/src/test/resources/argbra-fcd-output/part-00000-efae4f02-aebc-474b-9532-27a99e03ecaa-c000.csv similarity index 71% rename from src/test/resources/argbra-fcd-output/part-00000-3d339776-5c3b-4db8-8cac-3d284de2cca7-c000.csv rename to src/test/resources/argbra-fcd-output/part-00000-efae4f02-aebc-474b-9532-27a99e03ecaa-c000.csv index b9994490..a688c831 100644 --- a/src/test/resources/argbra-fcd-output/part-00000-3d339776-5c3b-4db8-8cac-3d284de2cca7-c000.csv +++ b/src/test/resources/argbra-fcd-output/part-00000-efae4f02-aebc-474b-9532-27a99e03ecaa-c000.csv @@ -1,2 +1,2 @@ list_id location_id status_code location_error tree_cover_loss_total_yearly tree_cover_loss_primary_forest_yearly tree_cover_loss_peat_yearly tree_cover_loss_intact_forest_yearly tree_cover_loss_protected_areas_yearly tree_cover_loss_by_country_yearly tree_cover_loss_by_country_wdpa_yearly tree_cover_loss_by_country_landmark_yearly tree_cover_loss_by_country_classified_region_yearly tree_cover_loss_arg_otbn_yearly tree_cover_loss_sea_landcover_yearly tree_cover_loss_idn_landcover_yearly tree_cover_loss_soy_yearly tree_cover_loss_idn_legal_yearly tree_cover_loss_idn_forest_moratorium_yearly tree_cover_loss_prodes_yearly tree_cover_loss_prodes_wdpa_yearly tree_cover_loss_prodes_primary_forest_yearly country_area country_specific_deforestation_yearly country_specific_deforestation_wdpa_yearly country_specific_deforestation_landmark_yearly country_specific_deforestation_classified_region_yearly classified_region_area tree_cover_loss_brazil_biomes_yearly tree_cover_extent_total tree_cover_extent_primary_forest tree_cover_extent_protected_areas tree_cover_extent_peat tree_cover_extent_intact_forest natural_habitat_primary natural_habitat_intact_forest total_area protected_areas_area peat_area arg_otbn_area protected_areas_by_category_area landmark_by_category_area brazil_biomes idn_legal_area sea_landcover_area idn_landcover_area idn_forest_moratorium_area south_america_presence legal_amazon_presence brazil_biomes_presence cerrado_biome_presence southeast_asia_presence indonesia_presence argentina_presence commodity_value_forest_extent commodity_value_peat commodity_value_protected_areas commodity_threat_deforestation commodity_threat_peat commodity_threat_protected_areas commodity_threat_fires -166 1 2 {"2001":390.3801,"2002":337.1303,"2003":591.747,"2004":522.3103,"2005":905.0571,"2006":861.6216,"2007":688.9958,"2008":719.6228,"2009":682.3401,"2010":342.0608,"2011":832.8575,"2012":822.2161,"2013":698.7244,"2014":1061.5944,"2015":623.7254,"2016":1506.2833,"2017":2137.3785,"2018":999.0414,"2019":1187.6768,"2020":1380.8694,"2021":1630.7097,"2022":1173.8577,"2023":935.707} {"2001":4.4721,"2002":4.6137,"2003":6.8156,"2004":2.1372,"2005":6.1991,"2006":69.8527,"2007":54.001,"2008":103.172,"2009":68.8725,"2010":25.6512,"2011":40.7394,"2012":61.5105,"2013":77.835,"2014":58.1765,"2015":40.3091,"2016":68.4074,"2017":163.7431,"2018":35.5811,"2019":48.25,"2020":30.8854,"2021":33.2186,"2022":47.9914,"2023":34.3442} {"2001":0.4816,"2002":0.2062,"2003":0.2062,"2004":0.6189,"2005":0.6193,"2006":0.413,"2007":1.3096,"2008":0.3435,"2009":0.6872,"2010":0.619,"2011":2.2689,"2012":1.4448,"2013":1.8558,"2014":5.7042,"2015":1.6496,"2016":2.7495,"2017":11.0043,"2018":2.4757,"2019":3.92,"2020":3.3036,"2021":5.0209,"2022":2.4071,"2023":1.7176} {} {"2001":8.538,"2002":8.7495,"2003":7.5728,"2004":14.4597,"2005":90.2472,"2006":211.8131,"2007":177.1202,"2008":66.799,"2009":69.0354,"2010":10.0578,"2011":39.6777,"2012":35.612,"2013":11.0741,"2014":20.1668,"2015":18.4411,"2016":11.7733,"2017":36.2816,"2018":7.9929,"2019":22.3112,"2020":8.1953,"2021":25.8224,"2022":11.5036,"2023":0.4131} {"ARG":{"2001":169.7492,"2002":110.0137,"2003":279.3034,"2004":243.8448,"2005":503.571,"2006":590.8377,"2007":450.6176,"2008":539.3522,"2009":450.1686,"2010":266.6501,"2011":375.4212,"2012":535.6973,"2013":450.3713,"2014":590.4731,"2015":350.2188,"2016":606.0677,"2017":909.5434,"2018":472.945,"2019":546.4409,"2020":656.0203,"2021":878.4245,"2022":433.7453,"2023":356.2457},"BRA":{"2001":220.6309,"2002":227.1166,"2003":312.4436,"2004":278.4655,"2005":401.4861,"2006":270.7838,"2007":237.4137,"2008":180.2706,"2009":232.1715,"2010":75.4107,"2011":457.3676,"2012":286.5188,"2013":248.2153,"2014":470.7769,"2015":273.5065,"2016":899.9401,"2017":1222.5303,"2018":526.0963,"2019":639.9957,"2020":724.7802,"2021":751.8719,"2022":739.5609,"2023":579.4613}} {"ARG":{"UNESCO-MAB Biosphere Reserve":{"2001":8.538,"2002":8.7495,"2003":7.5728,"2004":14.4597,"2005":90.2472,"2006":211.8131,"2007":177.1202,"2008":66.799,"2009":69.0354,"2010":10.0578,"2011":39.6089,"2012":35.5432,"2013":10.9364,"2014":20.0291,"2015":18.4411,"2016":11.5667,"2017":36.1438,"2018":7.9929,"2019":22.3112,"2020":8.1264,"2021":25.409,"2022":11.5036,"2023":0.4131}}} {"ARG":{"Not Reported":{"2001":39.904,"2002":36.1895,"2003":118.8314,"2004":99.1471,"2005":211.2132,"2006":230.4318,"2007":162.2055,"2008":142.2109,"2009":130.0801,"2010":68.1056,"2011":108.2244,"2012":241.5664,"2013":127.9175,"2014":210.2876,"2015":139.705,"2016":247.6686,"2017":361.3402,"2018":201.2981,"2019":248.205,"2020":277.617,"2021":433.2285,"2022":149.1649,"2023":117.5902}}} {"ARG":{"Category I":{"2001":1.0996,"2002":0.8246,"2003":2.1303,"2004":0.0,"2005":0.9622,"2006":0.55,"2007":0.0,"2008":0.3437,"2009":3.7798,"2010":0.0,"2011":0.5498,"2012":0.4125,"2013":1.6495,"2014":1.5121,"2015":6.3928,"2016":1.6495,"2017":2.0618,"2018":0.1375,"2019":0.6873,"2020":1.5121,"2021":2.8869,"2022":0.2064,"2023":0.0},"Category II":{"2001":15.5017,"2002":13.3015,"2003":31.5663,"2004":34.1258,"2005":67.4062,"2006":55.745,"2007":46.5984,"2008":73.8253,"2009":131.6487,"2010":73.2082,"2011":100.0926,"2012":133.2506,"2013":130.0616,"2014":156.9412,"2015":106.1384,"2016":151.6383,"2017":269.1299,"2018":137.5692,"2019":158.3015,"2020":163.4794,"2021":243.9769,"2022":130.3351,"2023":128.5559},"Category III":{"2001":26.2709,"2002":13.8555,"2003":42.6008,"2004":23.2359,"2005":54.4591,"2006":44.0493,"2007":27.9229,"2008":52.6735,"2009":78.9423,"2010":73.1421,"2011":68.5956,"2012":112.4446,"2013":151.5104,"2014":217.1291,"2015":113.4622,"2016":201.8612,"2017":294.2968,"2018":150.2793,"2019":194.1712,"2020":223.0738,"2021":300.5095,"2022":110.6482,"2023":98.4466}}} {"Category I":{"2001":1.0996,"2002":0.8246,"2003":2.1303,"2004":0.0,"2005":0.9622,"2006":0.55,"2007":0.0,"2008":0.3437,"2009":3.7798,"2010":0.0,"2011":0.5498,"2012":0.4125,"2013":1.6495,"2014":1.5121,"2015":6.3928,"2016":1.6495,"2017":2.0618,"2018":0.1375,"2019":0.6873,"2020":1.5121,"2021":2.8869,"2022":0.2064,"2023":0.0},"Category II":{"2001":15.5017,"2002":13.3015,"2003":31.5663,"2004":34.1258,"2005":67.4062,"2006":55.745,"2007":46.5984,"2008":73.8253,"2009":131.6487,"2010":73.2082,"2011":100.0926,"2012":133.2506,"2013":130.0616,"2014":157.0101,"2015":106.1384,"2016":151.6383,"2017":269.1299,"2018":137.5692,"2019":158.3015,"2020":163.4794,"2021":244.0458,"2022":130.3351,"2023":128.5559},"Category III":{"2001":26.2709,"2002":13.8555,"2003":42.6008,"2004":23.2359,"2005":54.4591,"2006":44.0493,"2007":27.9229,"2008":52.6735,"2009":78.9423,"2010":73.1421,"2011":68.5956,"2012":112.4446,"2013":151.5104,"2014":217.1291,"2015":113.4622,"2016":201.8612,"2017":294.2968,"2018":150.2793,"2019":194.1712,"2020":223.0738,"2021":300.5095,"2022":110.6482,"2023":98.4466}} {} {} {"2001":21.624,"2002":20.8704,"2003":35.0478,"2004":35.6016,"2005":35.1084,"2006":29.4707,"2007":23.7689,"2008":26.0851,"2009":23.4664,"2010":9.7672,"2011":76.1005,"2012":28.4983,"2013":18.1678,"2014":49.4618,"2015":21.3388,"2016":63.0798,"2017":82.2563,"2018":30.9672,"2019":32.1454,"2020":38.9536,"2021":3.9932,"2022":1.102} {} {} {} {} {} {"BRA":221184.6524,"ARG":194985.6634} {"ARG":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":186.9026,"2008":0.0,"2009":0.0,"2010approx":417.5112,"2011":15.1053,"2012":0.0,"2013approx":117.2137,"2014":79.0631,"2015":3.5847,"2016":8.0709,"2017":163.5411,"2018":31.4482,"2019":47.2747,"2020":181.2011,"2021":110.5264,"2022":18.9678,"2023":0.0}} {"ARG":{"UNESCO-MAB Biosphere Reserve":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.0,"2008":0.0,"2009":0.0,"2010approx":1.0342,"2011":0.0,"2012":0.0,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":17.5539,"2020":23.7832,"2021":0.0,"2022":0.0,"2023":0.0}}} {"ARG":{"Not Reported":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":92.4229,"2008":0.0,"2009":0.0,"2010approx":46.7539,"2011":0.0,"2012":0.0,"2013approx":20.4653,"2014":46.5229,"2015":0.0,"2016":0.0,"2017":44.9934,"2018":5.5842,"2019":31.0673,"2020":76.3528,"2021":21.096,"2022":0.0,"2023":0.0}}} {"ARG":{"Category I":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.0,"2008":0.0,"2009":0.0,"2010":0.0,"2011":0.0,"2012":0.0,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":0.0,"2020":21.3774,"2021":0.0,"2022":0.0,"2023":0.0},"Category III":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":14.683,"2008":0.0,"2009":0.0,"2010approx":75.2363,"2011":12.4841,"2012":0.0,"2013approx":30.8262,"2014":39.6991,"2015":0.0,"2016":6.8293,"2017":94.9476,"2018":22.8267,"2019":13.5189,"2020":40.3374,"2021":34.8144,"2022":10.8284,"2023":0.0},"Category II":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":25.0997,"2008":0.0,"2009":0.0,"2010approx":174.5175,"2011":1.7935,"2012":0.0,"2013approx":71.4984,"2014":27.7832,"2015":3.5847,"2016":1.2417,"2017":54.5984,"2018":8.3456,"2019":32.4472,"2020":73.3842,"2021":66.2684,"2022":7.1053,"2023":0.0}}} {"ARG":{"Category II":122991.3614,"Category I":30089.6869,"Category III":11275.7574}} {"Mata Atlântica":{"2001":222.281,"2002":229.664,"2003":313.889,"2004":281.566,"2005":404.1011,"2006":274.0218,"2007":244.2288,"2008":182.4026,"2009":237.3952,"2010":77.4035,"2011":460.1882,"2012":303.0526,"2013":257.9101,"2014":491.4761,"2015":279.4206,"2016":909.6394,"2017":1238.9614,"2018":529.7395,"2019":649.9751,"2020":733.7938,"2021":765.01,"2022":743.0739,"2023":584.0024}} 310497.0062 151998.5232 145836.3381 3133.6034 0.0 152001.8248 0.0 416412.5376 145907.1635 3633.9932 {"Category II":123789.927,"Category I":30089.6869,"Category III":11275.7574} {"UNESCO-MAB Biosphere Reserve":144721.3787} {"Not Reported":91313.3874} {"Mata Atlântica":235455.6349} {} {} {} 0.0 true false true false false false true {"2001":228399.0302,"2002":228399.0302,"2003":228218.695,"2004":228019.807,"2005":227669.5127,"2006":227360.6322,"2007":226793.1945,"2008":226294.0263,"2009":225908.1898,"2010":225416.7831,"2011":224990.0743,"2012":224720.0969,"2013":224202.2054,"2014":223703.987,"2015":223262.4402,"2016":222610.9217,"2017":222244.0711,"2018":221464.0632,"2019":220326.1549,"2020":219785.4113,"2021":219159.8818,"2022":218487.8012} {"2001":3633.9932,"2002":3633.9932,"2003":3633.9932,"2004":3633.9932,"2005":3633.9932,"2006":3633.9932,"2007":3633.9932,"2008":3633.9932,"2009":3633.9932,"2010":3633.9932,"2011":3633.9932,"2012":3633.9932,"2013":3633.9932,"2014":3633.9932,"2015":3633.9932,"2016":3633.9932,"2017":3633.9932,"2018":3633.9932,"2019":3633.9932,"2020":3633.9932,"2021":3633.9932,"2022":3633.9932} {"2001":145907.1635,"2002":145907.1635,"2003":145907.1635,"2004":145907.1635,"2005":145907.1635,"2006":145907.1635,"2007":145907.1635,"2008":145907.1635,"2009":145907.1635,"2010":145907.1635,"2011":145907.1635,"2012":145907.1635,"2013":145907.1635,"2014":145907.1635,"2015":145907.1635,"2016":145907.1635,"2017":145907.1635,"2018":145907.1635,"2019":145907.1635,"2020":145907.1635,"2021":145907.1635,"2022":145907.1635} {"2002":379.2232,"2003":549.1823,"2004":659.1748,"2005":876.3182,"2006":1066.606,"2007":885.0047,"2008":877.2432,"2009":918.1155,"2010":696.6862,"2011":787.8689,"2012":1016.1099,"2013":939.7652,"2014":1093.0653,"2015":1018.3691,"2016":1146.8585,"2017":1917.9162,"2018":1678.6519,"2019":1166.2731,"2020":1297.6101,"2021":1606.9852,"2022":1513.9363} {"2002":20.9771,"2003":20.5645,"2004":20.8397,"2005":21.1152,"2006":21.1155,"2007":21.9433,"2008":21.8738,"2009":21.0455,"2010":21.321,"2011":21.8711,"2012":22.0087,"2013":21.9396,"2014":22.4891,"2015":22.2141,"2016":22.4208,"2017":26.4784,"2018":25.7222,"2019":22.0779,"2020":22.2157,"2021":23.3861,"2022":23.3168} {"2002":1412.3095,"2003":1411.9654,"2004":1409.5569,"2005":1412.3853,"2006":1422.5074,"2007":1433.1795,"2008":1430.6331,"2009":1426.769,"2010":1421.6073,"2011":1416.8048,"2012":1421.3469,"2013":1419.7406,"2014":1420.2105,"2015":1426.2686,"2016":1424.1418,"2017":1427.8639,"2018":1422.7713,"2019":1422.0236,"2020":1426.6357,"2021":1427.8733,"2022":1431.9397} {} +166 1 2 {"2001":390.3801,"2002":337.1303,"2003":591.747,"2004":522.3103,"2005":905.0571,"2006":861.6216,"2007":688.9958,"2008":719.6228,"2009":682.3401,"2010":342.0608,"2011":832.8575,"2012":822.2161,"2013":698.7244,"2014":1061.5944,"2015":623.7254,"2016":1506.2833,"2017":2137.3785,"2018":999.0414,"2019":1187.6768,"2020":1380.8694,"2021":1630.7097,"2022":1173.8577,"2023":935.707} {"2001":4.4721,"2002":4.6137,"2003":6.8156,"2004":2.1372,"2005":6.1991,"2006":69.8527,"2007":54.001,"2008":103.172,"2009":68.8725,"2010":25.6512,"2011":40.7394,"2012":61.5105,"2013":77.835,"2014":58.1765,"2015":40.3091,"2016":68.4074,"2017":163.7431,"2018":35.5811,"2019":48.25,"2020":30.8854,"2021":33.2186,"2022":47.9914,"2023":34.3442} {"2001":0.4816,"2002":0.2062,"2003":0.2062,"2004":0.6189,"2005":0.6193,"2006":0.413,"2007":1.3096,"2008":0.3435,"2009":0.6872,"2010":0.619,"2011":2.2689,"2012":1.4448,"2013":1.8558,"2014":5.7042,"2015":1.6496,"2016":2.7495,"2017":11.0043,"2018":2.4757,"2019":3.92,"2020":3.3036,"2021":5.0209,"2022":2.4071,"2023":1.7176} {} {"2001":8.538,"2002":8.7495,"2003":7.5728,"2004":14.4597,"2005":90.2472,"2006":211.8131,"2007":177.1202,"2008":66.799,"2009":69.0354,"2010":10.0578,"2011":39.6777,"2012":35.612,"2013":11.0741,"2014":20.1668,"2015":18.4411,"2016":11.7733,"2017":36.2816,"2018":7.9929,"2019":22.3112,"2020":8.1953,"2021":25.8224,"2022":11.5036,"2023":0.4131} {"ARG":{"2001":169.7492,"2002":110.0137,"2003":279.3034,"2004":243.8448,"2005":503.571,"2006":590.8377,"2007":450.6176,"2008":539.3522,"2009":450.1686,"2010":266.6501,"2011":375.4212,"2012":535.6973,"2013":450.3713,"2014":590.4731,"2015":350.2188,"2016":606.0677,"2017":909.5434,"2018":472.945,"2019":546.4409,"2020":656.0203,"2021":878.4245,"2022":433.7453,"2023":356.2457},"BRA":{"2001":220.6309,"2002":227.1166,"2003":312.4436,"2004":278.4655,"2005":401.4861,"2006":270.7838,"2007":237.4137,"2008":180.2706,"2009":232.1715,"2010":75.4107,"2011":457.3676,"2012":286.5188,"2013":248.2153,"2014":470.7769,"2015":273.5065,"2016":899.9401,"2017":1222.5303,"2018":526.0963,"2019":639.9957,"2020":724.7802,"2021":751.8719,"2022":739.5609,"2023":579.4613}} {"ARG":{"UNESCO-MAB Biosphere Reserve":{"2001":8.538,"2002":8.7495,"2003":7.5728,"2004":14.4597,"2005":90.2472,"2006":211.8131,"2007":177.1202,"2008":66.799,"2009":69.0354,"2010":10.0578,"2011":39.6089,"2012":35.5432,"2013":10.9364,"2014":20.0291,"2015":18.4411,"2016":11.5667,"2017":36.1438,"2018":7.9929,"2019":22.3112,"2020":8.1264,"2021":25.409,"2022":11.5036,"2023":0.4131}}} {"ARG":{"Not Reported":{"2001":39.904,"2002":36.1895,"2003":118.8314,"2004":99.1471,"2005":211.2132,"2006":230.4318,"2007":162.2055,"2008":142.2109,"2009":130.0801,"2010":68.1056,"2011":108.2244,"2012":241.5664,"2013":127.9175,"2014":210.2876,"2015":139.705,"2016":247.6686,"2017":361.3402,"2018":201.2981,"2019":248.205,"2020":277.617,"2021":433.2285,"2022":149.1649,"2023":117.5902}}} {"ARG":{"Category II":{"2001":15.5017,"2002":13.3015,"2003":31.5663,"2004":34.1258,"2005":67.4062,"2006":55.745,"2007":46.5984,"2008":73.8253,"2009":131.6487,"2010":73.2082,"2011":100.0926,"2012":133.2506,"2013":130.0616,"2014":156.9412,"2015":106.1384,"2016":151.6383,"2017":269.1299,"2018":137.5692,"2019":158.3015,"2020":163.4794,"2021":243.9769,"2022":130.3351,"2023":128.5559},"Category III":{"2001":26.2709,"2002":13.8555,"2003":42.6008,"2004":23.2359,"2005":54.4591,"2006":44.0493,"2007":27.9229,"2008":52.6735,"2009":78.9423,"2010":73.1421,"2011":68.5956,"2012":112.4446,"2013":151.5104,"2014":217.1291,"2015":113.4622,"2016":201.8612,"2017":294.2968,"2018":150.2793,"2019":194.1712,"2020":223.0738,"2021":300.5095,"2022":110.6482,"2023":98.4466},"Category I":{"2001":1.0996,"2002":0.8246,"2003":2.1303,"2004":0.0,"2005":0.9622,"2006":0.55,"2007":0.0,"2008":0.3437,"2009":3.7798,"2010":0.0,"2011":0.5498,"2012":0.4125,"2013":1.6495,"2014":1.5121,"2015":6.3928,"2016":1.6495,"2017":2.0618,"2018":0.1375,"2019":0.6873,"2020":1.5121,"2021":2.8869,"2022":0.2064,"2023":0.0}}} {"Category II":{"2001":15.5017,"2002":13.3015,"2003":31.5663,"2004":34.1258,"2005":67.4062,"2006":55.745,"2007":46.5984,"2008":73.8253,"2009":131.6487,"2010":73.2082,"2011":100.0926,"2012":133.2506,"2013":130.0616,"2014":157.0101,"2015":106.1384,"2016":151.6383,"2017":269.1299,"2018":137.5692,"2019":158.3015,"2020":163.4794,"2021":244.0458,"2022":130.3351,"2023":128.5559},"Category III":{"2001":26.2709,"2002":13.8555,"2003":42.6008,"2004":23.2359,"2005":54.4591,"2006":44.0493,"2007":27.9229,"2008":52.6735,"2009":78.9423,"2010":73.1421,"2011":68.5956,"2012":112.4446,"2013":151.5104,"2014":217.1291,"2015":113.4622,"2016":201.8612,"2017":294.2968,"2018":150.2793,"2019":194.1712,"2020":223.0738,"2021":300.5095,"2022":110.6482,"2023":98.4466},"Category I":{"2001":1.0996,"2002":0.8246,"2003":2.1303,"2004":0.0,"2005":0.9622,"2006":0.55,"2007":0.0,"2008":0.3437,"2009":3.7798,"2010":0.0,"2011":0.5498,"2012":0.4125,"2013":1.6495,"2014":1.5121,"2015":6.3928,"2016":1.6495,"2017":2.0618,"2018":0.1375,"2019":0.6873,"2020":1.5121,"2021":2.8869,"2022":0.2064,"2023":0.0}} {} {} {"2001":21.624,"2002":20.8704,"2003":35.0478,"2004":35.6016,"2005":35.1084,"2006":29.4707,"2007":23.7689,"2008":26.0851,"2009":23.4664,"2010":9.7672,"2011":76.1005,"2012":28.4983,"2013":18.1678,"2014":49.4618,"2015":21.3388,"2016":63.0798,"2017":82.2563,"2018":30.9672,"2019":32.1454,"2020":38.9536,"2021":3.9932,"2022":1.102} {} {} {} {} {} {"ARG":194985.6634,"BRA":221184.6524} {"ARG":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":186.9026,"2008":0.0,"2009":0.0,"2010approx":417.5112,"2011":15.1053,"2012":0.0,"2013approx":117.2137,"2014":79.0631,"2015":3.5847,"2016":8.0709,"2017":163.5411,"2018":31.4482,"2019":47.2747,"2020":181.2011,"2021":110.5264,"2022":18.9678,"2023":0.0}} {"ARG":{"UNESCO-MAB Biosphere Reserve":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.0,"2008":0.0,"2009":0.0,"2010approx":1.0342,"2011":0.0,"2012":0.0,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":17.5539,"2020":23.7832,"2021":0.0,"2022":0.0,"2023":0.0}}} {"ARG":{"Not Reported":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":92.4229,"2008":0.0,"2009":0.0,"2010approx":46.7539,"2011":0.0,"2012":0.0,"2013approx":20.4653,"2014":46.5229,"2015":0.0,"2016":0.0,"2017":44.9934,"2018":5.5842,"2019":31.0673,"2020":76.3528,"2021":21.096,"2022":0.0,"2023":0.0}}} {"ARG":{"Category III":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":14.683,"2008":0.0,"2009":0.0,"2010approx":75.2363,"2011":12.4841,"2012":0.0,"2013approx":30.8262,"2014":39.6991,"2015":0.0,"2016":6.8293,"2017":94.9476,"2018":22.8267,"2019":13.5189,"2020":40.3374,"2021":34.8144,"2022":10.8284,"2023":0.0},"Category II":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":25.0997,"2008":0.0,"2009":0.0,"2010approx":174.5175,"2011":1.7935,"2012":0.0,"2013approx":71.4984,"2014":27.7832,"2015":3.5847,"2016":1.2417,"2017":54.5984,"2018":8.3456,"2019":32.4472,"2020":73.3842,"2021":66.2684,"2022":7.1053,"2023":0.0},"Category I":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.0,"2008":0.0,"2009":0.0,"2010":0.0,"2011":0.0,"2012":0.0,"2013":0.0,"2014":0.0,"2015":0.0,"2016":0.0,"2017":0.0,"2018":0.0,"2019":0.0,"2020":21.3774,"2021":0.0,"2022":0.0,"2023":0.0}}} {"ARG":{"Category II":122991.3614,"Category III":11275.7574,"Category I":30089.6869}} {"Mata Atlântica":{"2001":222.281,"2002":229.664,"2003":313.889,"2004":281.566,"2005":404.1011,"2006":274.0218,"2007":244.2288,"2008":182.4026,"2009":237.3952,"2010":77.4035,"2011":460.1882,"2012":303.0526,"2013":257.9101,"2014":491.4761,"2015":279.4206,"2016":909.6394,"2017":1238.9614,"2018":529.7395,"2019":649.9751,"2020":733.7938,"2021":765.01,"2022":743.0739,"2023":584.0024}} 310497.0062 151998.5232 145836.3381 3133.6034 0.0 152001.8248 0.0 416412.5376 145907.1635 3633.9932 {"Category II":123789.927,"Category III":11275.7574,"Category I":30089.6869} {"UNESCO-MAB Biosphere Reserve":144721.3787} {"Not Reported":91313.3874} {"Mata Atlântica":235455.6349} {} {} {} 0.0 true false true false false false true {"2001":228399.0302,"2002":228399.0302,"2003":228218.695,"2004":228019.807,"2005":227669.5127,"2006":227360.6322,"2007":226793.1945,"2008":226294.0263,"2009":225908.1898,"2010":225416.7831,"2011":224990.0743,"2012":224720.0969,"2013":224202.2054,"2014":223703.987,"2015":223262.4402,"2016":222610.9217,"2017":222244.0711,"2018":221464.0632,"2019":220326.1549,"2020":219785.4113,"2021":219159.8818,"2022":218487.8012} {"2001":3633.9932,"2002":3633.9932,"2003":3633.9932,"2004":3633.9932,"2005":3633.9932,"2006":3633.9932,"2007":3633.9932,"2008":3633.9932,"2009":3633.9932,"2010":3633.9932,"2011":3633.9932,"2012":3633.9932,"2013":3633.9932,"2014":3633.9932,"2015":3633.9932,"2016":3633.9932,"2017":3633.9932,"2018":3633.9932,"2019":3633.9932,"2020":3633.9932,"2021":3633.9932,"2022":3633.9932} {"2001":145907.1635,"2002":145907.1635,"2003":145907.1635,"2004":145907.1635,"2005":145907.1635,"2006":145907.1635,"2007":145907.1635,"2008":145907.1635,"2009":145907.1635,"2010":145907.1635,"2011":145907.1635,"2012":145907.1635,"2013":145907.1635,"2014":145907.1635,"2015":145907.1635,"2016":145907.1635,"2017":145907.1635,"2018":145907.1635,"2019":145907.1635,"2020":145907.1635,"2021":145907.1635,"2022":145907.1635} {"2002":379.2232,"2003":549.1823,"2004":659.1748,"2005":876.3182,"2006":1066.606,"2007":885.0047,"2008":877.2432,"2009":918.1155,"2010":696.6862,"2011":787.8689,"2012":1016.1099,"2013":939.7652,"2014":1093.0653,"2015":1018.3691,"2016":1146.8585,"2017":1917.9162,"2018":1678.6519,"2019":1166.2731,"2020":1297.6101,"2021":1606.9852,"2022":1513.9363} {"2002":20.9771,"2003":20.5645,"2004":20.8397,"2005":21.1152,"2006":21.1155,"2007":21.9433,"2008":21.8738,"2009":21.0455,"2010":21.321,"2011":21.8711,"2012":22.0087,"2013":21.9396,"2014":22.4891,"2015":22.2141,"2016":22.4208,"2017":26.4784,"2018":25.7222,"2019":22.0779,"2020":22.2157,"2021":23.3861,"2022":23.3168} {"2002":1412.3095,"2003":1411.9654,"2004":1409.5569,"2005":1412.3853,"2006":1422.5074,"2007":1433.1795,"2008":1430.6331,"2009":1426.769,"2010":1421.6073,"2011":1416.8048,"2012":1421.3469,"2013":1419.7406,"2014":1420.2105,"2015":1426.2686,"2016":1424.1418,"2017":1427.8639,"2018":1422.7713,"2019":1422.0236,"2020":1426.6357,"2021":1427.8733,"2022":1431.9397} {} diff --git a/src/test/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysisSpec.scala b/src/test/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysisSpec.scala index 294cff40..86f5fffb 100644 --- a/src/test/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysisSpec.scala +++ b/src/test/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysisSpec.scala @@ -35,9 +35,7 @@ class ForestChangeDiagnosticAnalysisSpec extends TestEnvironment with DataFrameC ForestChangeDiagnosticAnalysis( features, - intermediateResultsRDD = None, fireAlerts = fireAlertsRdd, - saveIntermediateResults = identity, kwargs = Map("config" -> GfwConfig.get())) }