From 94529c4a33f339ee11d6c7e1b11132c7cc85eafd Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Mon, 7 Oct 2024 13:18:35 -0600 Subject: [PATCH 01/10] Updated MODIS reader to allow for a list of filenames, renamed swath dims. --- monetio/sat/_modis_l2_mm.py | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/monetio/sat/_modis_l2_mm.py b/monetio/sat/_modis_l2_mm.py index 0bf35b11..15f98c62 100644 --- a/monetio/sat/_modis_l2_mm.py +++ b/monetio/sat/_modis_l2_mm.py @@ -2,6 +2,7 @@ import sys from collections import OrderedDict from glob import glob +from datetime import datetime, timezone import numpy as np import xarray as xr @@ -10,28 +11,33 @@ def read_dataset(fname, variable_dict): """ Parameters - ---------- + __________ fname : str Input file path. Returns - ------- + _______ xarray.Dataset """ from monetio.sat.hdfio import hdf_close, hdf_list, hdf_open, hdf_read + epoch_1993 = int(datetime(1993, 1, 1, tzinfo=timezone.utc).timestamp()) + print("reading " + fname) ds = xr.Dataset() f = hdf_open(fname) hdf_list(f) - latitude = hdf_read(f, "Latitude") # noqa: F841 - longitude = hdf_read(f, "Longitude") # noqa: F841 - start_time = hdf_read(f, "Scan_Start_Time") # noqa: F841 + # Geolocation and Time Parameters + latitude = hdf_read(f, "Latitude") + longitude = hdf_read(f, "Longitude") + start_time = hdf_read(f, "Scan_Start_Time") \ + + epoch_1993 # convert seconds since 1993 to since 1970 for varname in variable_dict: print(varname) values = hdf_read(f, varname) + print('min, max: ', values.min(), ' ', values.max()) if "scale" in variable_dict[varname]: values = variable_dict[varname]["scale"] * values if "minimum" in variable_dict[varname]: @@ -46,13 +52,17 @@ def read_dataset(fname, variable_dict): ds.attrs["quality_thresh"] = variable_dict[varname]["quality_flag"] hdf_close(f) + ds = ds.assign_coords({'lon': (['dim_0', 'dim_1'], longitude), + 'lat': (['dim_0', 'dim_1'], latitude)}) + ds = ds.rename_dims({'dim_0': 'Cell_Along_Swath', 'dim_1': 'Cell_Across_Swath'}) + return ds def apply_quality_flag(ds): """ Parameters - ---------- + __________ ds : xarray.Dataset """ if "quality_flag" in ds.attrs: @@ -69,19 +79,24 @@ def apply_quality_flag(ds): def read_mfdataset(fnames, variable_dict, debug=False): """ Parameters - ---------- + __________ fnames : str Regular expression for input file paths. Returns - ------- + _______ xarray.Dataset """ if debug: logging_level = logging.DEBUG - logging.basicConfig(stream=sys.stdout, level=logging_level) - - files = sorted(glob(fnames)) + else: + logging_level = logging.INFO + logging.basicConfig(stream=sys.stdout, level=logging_level) + + if isinstance(fnames, list): + files = fnames + else: + files = sorted(glob(fnames)) granules = OrderedDict() From 1dabef02881c6a99b373653ad5d4bb4781dd7967 Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Mon, 7 Oct 2024 13:59:04 -0600 Subject: [PATCH 02/10] Minor changes. --- monetio/sat/_modis_l2_mm.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/monetio/sat/_modis_l2_mm.py b/monetio/sat/_modis_l2_mm.py index 15f98c62..fc23331f 100644 --- a/monetio/sat/_modis_l2_mm.py +++ b/monetio/sat/_modis_l2_mm.py @@ -11,12 +11,12 @@ def read_dataset(fname, variable_dict): """ Parameters - __________ + ---------- fname : str Input file path. Returns - _______ + ------- xarray.Dataset """ from monetio.sat.hdfio import hdf_close, hdf_list, hdf_open, hdf_read @@ -62,7 +62,7 @@ def read_dataset(fname, variable_dict): def apply_quality_flag(ds): """ Parameters - __________ + ---------- ds : xarray.Dataset """ if "quality_flag" in ds.attrs: @@ -79,19 +79,19 @@ def apply_quality_flag(ds): def read_mfdataset(fnames, variable_dict, debug=False): """ Parameters - __________ + ---------- fnames : str Regular expression for input file paths. Returns - _______ + ------- xarray.Dataset """ if debug: logging_level = logging.DEBUG + logging.basicConfig(stream=sys.stdout, level=logging_level) else: logging_level = logging.INFO - logging.basicConfig(stream=sys.stdout, level=logging_level) if isinstance(fnames, list): files = fnames From 0baae4041f45cd930bea126e9c641b526461fa49 Mon Sep 17 00:00:00 2001 From: David Fillmore <1524012+dwfncar@users.noreply.github.com> Date: Mon, 7 Oct 2024 14:39:03 -0600 Subject: [PATCH 03/10] Update monetio/sat/_modis_l2_mm.py Co-authored-by: Zachary Moon --- monetio/sat/_modis_l2_mm.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/monetio/sat/_modis_l2_mm.py b/monetio/sat/_modis_l2_mm.py index fc23331f..e460495a 100644 --- a/monetio/sat/_modis_l2_mm.py +++ b/monetio/sat/_modis_l2_mm.py @@ -90,8 +90,6 @@ def read_mfdataset(fnames, variable_dict, debug=False): if debug: logging_level = logging.DEBUG logging.basicConfig(stream=sys.stdout, level=logging_level) - else: - logging_level = logging.INFO if isinstance(fnames, list): files = fnames From 1899980395856f67b9dbfebff9e41f5a578e0f4f Mon Sep 17 00:00:00 2001 From: David Fillmore <1524012+dwfncar@users.noreply.github.com> Date: Mon, 7 Oct 2024 14:39:24 -0600 Subject: [PATCH 04/10] Update monetio/sat/_modis_l2_mm.py Co-authored-by: Zachary Moon --- monetio/sat/_modis_l2_mm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monetio/sat/_modis_l2_mm.py b/monetio/sat/_modis_l2_mm.py index e460495a..90a9cdbb 100644 --- a/monetio/sat/_modis_l2_mm.py +++ b/monetio/sat/_modis_l2_mm.py @@ -32,8 +32,8 @@ def read_dataset(fname, variable_dict): # Geolocation and Time Parameters latitude = hdf_read(f, "Latitude") longitude = hdf_read(f, "Longitude") - start_time = hdf_read(f, "Scan_Start_Time") \ - + epoch_1993 # convert seconds since 1993 to since 1970 + # convert seconds since 1993 to since 1970 + start_time = hdf_read(f, "Scan_Start_Time") + epoch_1993 for varname in variable_dict: print(varname) values = hdf_read(f, varname) From 39a6a92d04d4041d8e0be51853e04cb58a627938 Mon Sep 17 00:00:00 2001 From: David Fillmore <1524012+dwfncar@users.noreply.github.com> Date: Mon, 7 Oct 2024 14:39:44 -0600 Subject: [PATCH 05/10] Update monetio/sat/_modis_l2_mm.py Co-authored-by: Zachary Moon --- monetio/sat/_modis_l2_mm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/monetio/sat/_modis_l2_mm.py b/monetio/sat/_modis_l2_mm.py index 90a9cdbb..e3beb6d9 100644 --- a/monetio/sat/_modis_l2_mm.py +++ b/monetio/sat/_modis_l2_mm.py @@ -91,10 +91,10 @@ def read_mfdataset(fnames, variable_dict, debug=False): logging_level = logging.DEBUG logging.basicConfig(stream=sys.stdout, level=logging_level) - if isinstance(fnames, list): - files = fnames - else: + if isinstance(fnames, str): files = sorted(glob(fnames)) + else: + files = fnames granules = OrderedDict() From 82b57460dc932c88e9c02d478861059a0645352f Mon Sep 17 00:00:00 2001 From: David Fillmore <1524012+dwfncar@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:53:51 -0600 Subject: [PATCH 06/10] Update monetio/sat/_modis_l2_mm.py Co-authored-by: Zachary Moon --- monetio/sat/_modis_l2_mm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monetio/sat/_modis_l2_mm.py b/monetio/sat/_modis_l2_mm.py index e3beb6d9..8276bafc 100644 --- a/monetio/sat/_modis_l2_mm.py +++ b/monetio/sat/_modis_l2_mm.py @@ -37,7 +37,7 @@ def read_dataset(fname, variable_dict): for varname in variable_dict: print(varname) values = hdf_read(f, varname) - print('min, max: ', values.min(), ' ', values.max()) + print("min, max: ", values.min(), " ", values.max()) if "scale" in variable_dict[varname]: values = variable_dict[varname]["scale"] * values if "minimum" in variable_dict[varname]: From 407de0606199912a248913124c7568986e0edac4 Mon Sep 17 00:00:00 2001 From: David Fillmore <1524012+dwfncar@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:54:04 -0600 Subject: [PATCH 07/10] Update monetio/sat/_modis_l2_mm.py Co-authored-by: Zachary Moon --- monetio/sat/_modis_l2_mm.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/monetio/sat/_modis_l2_mm.py b/monetio/sat/_modis_l2_mm.py index 8276bafc..9dc3bc41 100644 --- a/monetio/sat/_modis_l2_mm.py +++ b/monetio/sat/_modis_l2_mm.py @@ -52,8 +52,12 @@ def read_dataset(fname, variable_dict): ds.attrs["quality_thresh"] = variable_dict[varname]["quality_flag"] hdf_close(f) - ds = ds.assign_coords({'lon': (['dim_0', 'dim_1'], longitude), - 'lat': (['dim_0', 'dim_1'], latitude)}) + ds = ds.assign_coords( + { + 'lon': (['dim_0', 'dim_1'], longitude), + 'lat': (['dim_0', 'dim_1'], latitude), + } + ) ds = ds.rename_dims({'dim_0': 'Cell_Along_Swath', 'dim_1': 'Cell_Across_Swath'}) return ds From cb1215bf98492ddd4408bcb6ca6ccd6d46bb3b14 Mon Sep 17 00:00:00 2001 From: David Fillmore <1524012+dwfncar@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:54:15 -0600 Subject: [PATCH 08/10] Update monetio/sat/_modis_l2_mm.py Co-authored-by: Zachary Moon --- monetio/sat/_modis_l2_mm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monetio/sat/_modis_l2_mm.py b/monetio/sat/_modis_l2_mm.py index 9dc3bc41..f067a8eb 100644 --- a/monetio/sat/_modis_l2_mm.py +++ b/monetio/sat/_modis_l2_mm.py @@ -1,8 +1,8 @@ import logging import sys from collections import OrderedDict -from glob import glob from datetime import datetime, timezone +from glob import glob import numpy as np import xarray as xr From 69f6bc7a4f45c1f6fff4f396b724fbb0a1d705fc Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Mon, 7 Oct 2024 16:02:15 -0600 Subject: [PATCH 09/10] Added scan time to granule xarray. --- monetio/sat/_modis_l2_mm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/monetio/sat/_modis_l2_mm.py b/monetio/sat/_modis_l2_mm.py index f067a8eb..6fa29599 100644 --- a/monetio/sat/_modis_l2_mm.py +++ b/monetio/sat/_modis_l2_mm.py @@ -56,6 +56,7 @@ def read_dataset(fname, variable_dict): { 'lon': (['dim_0', 'dim_1'], longitude), 'lat': (['dim_0', 'dim_1'], latitude), + 'time': (['dim_0', 'dim_1'], start_time), } ) ds = ds.rename_dims({'dim_0': 'Cell_Along_Swath', 'dim_1': 'Cell_Across_Swath'}) From 3af095f3832a3bf3097a34b70923347da71c3116 Mon Sep 17 00:00:00 2001 From: zmoon Date: Mon, 7 Oct 2024 17:20:33 -0500 Subject: [PATCH 10/10] fmt --- monetio/sat/_modis_l2_mm.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/monetio/sat/_modis_l2_mm.py b/monetio/sat/_modis_l2_mm.py index 6fa29599..b09c5c09 100644 --- a/monetio/sat/_modis_l2_mm.py +++ b/monetio/sat/_modis_l2_mm.py @@ -54,12 +54,12 @@ def read_dataset(fname, variable_dict): ds = ds.assign_coords( { - 'lon': (['dim_0', 'dim_1'], longitude), - 'lat': (['dim_0', 'dim_1'], latitude), - 'time': (['dim_0', 'dim_1'], start_time), - } - ) - ds = ds.rename_dims({'dim_0': 'Cell_Along_Swath', 'dim_1': 'Cell_Across_Swath'}) + "lon": (["dim_0", "dim_1"], longitude), + "lat": (["dim_0", "dim_1"], latitude), + "time": (["dim_0", "dim_1"], start_time), + } + ) + ds = ds.rename_dims({"dim_0": "Cell_Along_Swath", "dim_1": "Cell_Across_Swath"}) return ds