diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7daf21c5..4bf40ccc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,12 @@ exclude: 'experiments' repos: - repo: https://github.com/ambv/black - rev: 23.1.0 + rev: 24.1.1 hooks: - id: black language_version: python3.10 - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.0 hooks: - id: isort name: isort (python) @@ -17,6 +17,6 @@ repos: name: isort (pyi) types: [pyi] - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 + rev: 7.0.0 hooks: - id: flake8 diff --git a/climetlab/indexing/cube.py b/climetlab/indexing/cube.py index 2efb73b5..176ff42d 100644 --- a/climetlab/indexing/cube.py +++ b/climetlab/indexing/cube.py @@ -67,8 +67,22 @@ def __init__( self._field_shape = None + # Sort the source according to their + # internal_args = reduce(operator.add, [Separator.split(a) for a in args], []) + # for i in ds: + # print(i) + # print("before") + # print(ds[0]) + # print(ds[1]) + # print(ds[2]) + # print(ds[3]) self.source = ds.order_by(*args, remapping=remapping, patches=patches) del ds + # print("after") + # print(self.source[0]) + # print(self.source[1]) + # print(self.source[2]) + # print(self.source[3]) # Get a mapping of user names to unique values # With possible reduce dimentionality if the user use 'level+param' @@ -238,7 +252,6 @@ def __init__(self, cube, coords, coords_names=None): self.owner = cube assert all(isinstance(_, int) for _ in coords), coords self.coords = coords - self.extended_icoords = self.coords self.flatten_values = cube.flatten_values def __repr__(self): @@ -246,6 +259,10 @@ def __repr__(self): f"{self.__class__.__name__}({self.coords},index_names={self._coords_names})" ) + @property + def extended_icoords(self): + return self.coords + def to_numpy(self, **kwargs): return self.owner[self.coords].to_numpy( reshape=not self.flatten_values, **kwargs diff --git a/climetlab/indexing/fieldset.py b/climetlab/indexing/fieldset.py index cd7e9c7c..dc68c22c 100644 --- a/climetlab/indexing/fieldset.py +++ b/climetlab/indexing/fieldset.py @@ -10,12 +10,12 @@ from climetlab.core import Base from climetlab.core.index import Index, MaskIndex, MultiIndex + class Field(Base): pass -class FieldSet(Index): - +class FieldSet(Index): @classmethod def new_mask_index(self, *args, **kwargs): return MaskFieldSet(*args, **kwargs) diff --git a/climetlab/readers/grib/codes.py b/climetlab/readers/grib/codes.py index 9547ff9d..fe5a7202 100644 --- a/climetlab/readers/grib/codes.py +++ b/climetlab/readers/grib/codes.py @@ -18,7 +18,6 @@ from climetlab.core import Base from climetlab.core.constants import DATETIME -from climetlab.indexing.fieldset import Field from climetlab.profiling import call_counter from climetlab.utils.bbox import BoundingBox @@ -322,7 +321,7 @@ def at_offset(self, offset): # count = defaultdict(int) -class GribField(Field): +class GribField(Base): def __init__(self, path, offset, length, handle_cache=None): self.path = path self._offset = offset diff --git a/climetlab/readers/grib/index/__init__.py b/climetlab/readers/grib/index/__init__.py index 2cb2a69f..dc0811c1 100644 --- a/climetlab/readers/grib/index/__init__.py +++ b/climetlab/readers/grib/index/__init__.py @@ -14,6 +14,7 @@ from lru import LRU +from climetlab.core.index import Index, MaskIndex, MultiIndex from climetlab.decorators import normalize_grib_key_values, normalize_grib_keys from climetlab.indexing.database import ( FILEPARTS_KEY_NAMES, @@ -25,12 +26,11 @@ from climetlab.readers.grib.fieldset import FieldSetMixin from climetlab.utils import progress_bar from climetlab.utils.availability import Availability -from climetlab.indexing.fieldset import FieldSet LOG = logging.getLogger(__name__) -class GribFieldSet(FieldSetMixin, FieldSet): +class FieldSet(FieldSetMixin, Index): _availability = None def __init__(self, *args, **kwargs): @@ -39,12 +39,21 @@ def __init__(self, *args, **kwargs): ): self._availability = Availability(self.availability_path) - FieldSet.__init__(self, *args, **kwargs) + Index.__init__(self, *args, **kwargs) + + @classmethod + def new_mask_index(self, *args, **kwargs): + return MaskFieldSet(*args, **kwargs) @property def availability_path(self): return None + @classmethod + def merge(cls, sources): + assert all(isinstance(_, FieldSet) for _ in sources) + return MultiFieldSet(sources) + def available(self, request, as_list_of_dicts=False): from climetlab.utils.availability import Availability @@ -141,7 +150,18 @@ def is_full_hypercube(self): def _normalize_kwargs_names(self, **kwargs): return kwargs -class FieldSetInFiles(GribFieldSet): + +class MaskFieldSet(FieldSet, MaskIndex): + def __init__(self, *args, **kwargs): + MaskIndex.__init__(self, *args, **kwargs) + + +class MultiFieldSet(FieldSet, MultiIndex): + def __init__(self, *args, **kwargs): + MultiIndex.__init__(self, *args, **kwargs) + + +class FieldSetInFiles(FieldSet): # Remote Fieldsets (with urls) are also here, # as the actual fieldset is accessed on a file in cache. # This class changes the interface (_getitem__ and __len__) diff --git a/climetlab/readers/grib/output.py b/climetlab/readers/grib/output.py index 2714f722..a2499f51 100644 --- a/climetlab/readers/grib/output.py +++ b/climetlab/readers/grib/output.py @@ -199,9 +199,9 @@ def update_metadata(self, handle, metadata, compulsary): if "number" in metadata: compulsary += ("numberOfForecastsInEnsemble",) productDefinitionTemplateNumber = {"tp": 11} - metadata[ - "productDefinitionTemplateNumber" - ] = productDefinitionTemplateNumber.get(handle.get("shortName"), 1) + metadata["productDefinitionTemplateNumber"] = ( + productDefinitionTemplateNumber.get(handle.get("shortName"), 1) + ) if metadata.get("type") in ("pf", "cf"): metadata.setdefault("typeOfGeneratingProcess", 4) diff --git a/climetlab/readers/grib/reader.py b/climetlab/readers/grib/reader.py index 11ed5b34..4b4bb4b0 100644 --- a/climetlab/readers/grib/reader.py +++ b/climetlab/readers/grib/reader.py @@ -10,7 +10,7 @@ import logging from climetlab.readers import Reader -from climetlab.indexing.fieldset import MultiFieldSet +from climetlab.readers.grib.index import MultiFieldSet from climetlab.readers.grib.index.file import FieldSetInOneFile LOG = logging.getLogger(__name__) diff --git a/climetlab/readers/netcdf/__init__.py b/climetlab/readers/netcdf/__init__.py index 3f8751a1..c445d2e5 100644 --- a/climetlab/readers/netcdf/__init__.py +++ b/climetlab/readers/netcdf/__init__.py @@ -15,7 +15,6 @@ import numpy as np -from climetlab.core import Base from climetlab.indexing.fieldset import Field from climetlab.readers.netcdf.flavours import get_flavour from climetlab.utils.bbox import BoundingBox diff --git a/climetlab/readers/netcdf/flavours.py b/climetlab/readers/netcdf/flavours.py index f91646b4..aaaf5077 100644 --- a/climetlab/readers/netcdf/flavours.py +++ b/climetlab/readers/netcdf/flavours.py @@ -32,5 +32,6 @@ def get_levelist(self, field): def get_number(self, field): return None + def get_flavour(reader, flavour): return Flavour("default", reader) diff --git a/climetlab/scripts/availability.py b/climetlab/scripts/availability.py old mode 100755 new mode 100644 diff --git a/climetlab/scripts/grib.py b/climetlab/scripts/grib.py old mode 100755 new mode 100644 diff --git a/climetlab/sources/constants.py b/climetlab/sources/constants.py index 716b2af8..8a93dbc5 100644 --- a/climetlab/sources/constants.py +++ b/climetlab/sources/constants.py @@ -16,7 +16,7 @@ from climetlab.decorators import cached_method, normalize from climetlab.indexing.cube import index_to_coords -from climetlab.readers.grib.index import GribFieldSet +from climetlab.readers.grib.index import FieldSet from climetlab.utils.dates import to_datetime LOG = logging.getLogger(__name__) @@ -239,7 +239,7 @@ def make_datetime(date, time): return datetime.datetime(date.year, date.month, date.day, time) -class Constants(GribFieldSet): +class Constants(FieldSet): def __init__(self, source_or_dataset, request={}, **kwargs): request = dict(**request) request.update(kwargs) diff --git a/climetlab/sources/era5_accumulations.py b/climetlab/sources/era5_accumulations.py index 7837ebe8..bdb58079 100644 --- a/climetlab/sources/era5_accumulations.py +++ b/climetlab/sources/era5_accumulations.py @@ -95,7 +95,7 @@ class Era5Accumulations(Source): Note 2 : There are no overlap due to the way the date+time+step are computed """ - def __init__(self, *args, accumulation_period=6, **kwargs): + def __init__(self, *args, **kwargs): request = {} for a in args: request.update(self.requests(**a)) @@ -111,7 +111,7 @@ def __init__(self, *args, accumulation_period=6, **kwargs): number = request.get("number", [0]) assert isinstance(number, (list, tuple)) - user_step = accumulation_period # For now, we only support 6h accumulation + user_step = 6 # For now, we only support 6h accumulation user_dates = request["date"] user_times = request["time"] diff --git a/climetlab/sources/indexed_urls.py b/climetlab/sources/indexed_urls.py index 591e6c5f..ef7420d9 100644 --- a/climetlab/sources/indexed_urls.py +++ b/climetlab/sources/indexed_urls.py @@ -10,7 +10,7 @@ import warnings from climetlab.indexing import PerUrlIndex -from climetlab.indexing.fieldset import MultiFieldSet +from climetlab.readers.grib.index import MultiFieldSet from climetlab.readers.grib.index.sql import FieldsetInFilesWithSqlIndex from climetlab.sources.indexed import IndexedSource from climetlab.utils.patterns import Pattern diff --git a/climetlab/sources/loader.py b/climetlab/sources/loader.py index 96b8142f..a005c73b 100644 --- a/climetlab/sources/loader.py +++ b/climetlab/sources/loader.py @@ -124,7 +124,7 @@ def mutate(self): source = sources[0] for s in sources[1:]: source += s - from climetlab.indexing.fieldset import FieldSet + from climetlab.readers.grib.index import FieldSet assert isinstance(source, FieldSet), type(source) diff --git a/climetlab/sources/opendap.py b/climetlab/sources/opendap.py index 92eaa852..4c0c1c73 100644 --- a/climetlab/sources/opendap.py +++ b/climetlab/sources/opendap.py @@ -8,32 +8,19 @@ # -from climetlab.core.index import FieldSet -from climetlab.readers.netcdf import NetCDFReader -from climetlab.sources import Source +import xarray as xr +from .base import Source -class OpenDAP(FieldSet): - def __init__(self, url, flavour=None, **kwargs): - super().__init__() - - self._url = url - self._reader = NetCDFReader(self, url, opendap=True, flavour=flavour) - self._kwargs = kwargs - def __iter__(self): - return iter(self._reader) - - def __len__(self): - return len(self._reader) +class OpenDAP(Source): + def __init__(self, url): + super().__init__() - def __getitem__(self, n): - return self._reader[n] + self.url = url - def mutate(self): - if self._kwargs: - return self.sel(**self._kwargs) - return super().mutate() + def to_xarray(self): + return xr.open_dataset(self.url) source = OpenDAP diff --git a/climetlab/sources/virtual.py b/climetlab/sources/virtual.py index a414cb59..3f470003 100644 --- a/climetlab/sources/virtual.py +++ b/climetlab/sources/virtual.py @@ -13,7 +13,7 @@ import threading import climetlab as cml -from climetlab.readers.grib.index import GribFieldSet +from climetlab.readers.grib.index import FieldSet from climetlab.utils.serialise import register_serialisation LOG = logging.getLogger(__name__) @@ -71,7 +71,7 @@ def __getitem__(self, key): return self[key] -class Virtual(GribFieldSet): +class Virtual(FieldSet): SIZE = int(365.25 * 24 * (2022 - 1959)) # SIZE = 100 diff --git a/climetlab/sphinxext/generate_gallery_rst.py b/climetlab/sphinxext/generate_gallery_rst.py old mode 100755 new mode 100644 diff --git a/climetlab/sphinxext/generate_settings_rst.py b/climetlab/sphinxext/generate_settings_rst.py old mode 100755 new mode 100644 diff --git a/climetlab/utils/config.py b/climetlab/utils/config.py index 27154c2b..85a39dd9 100644 --- a/climetlab/utils/config.py +++ b/climetlab/utils/config.py @@ -108,9 +108,9 @@ def do_load(self, partial=False): else: out += ds - from climetlab.readers.grib.index import GribFieldSet + from climetlab.readers.grib.index import FieldSet - assert isinstance(out, GribFieldSet), type(out) + assert isinstance(out, FieldSet), type(out) self._do_load = (out, partial) return self._do_load[0] @@ -385,9 +385,11 @@ def __init__(self, loops, input, output, partial=False): inputs = Inputs(input) self.output = output self.loops = [ - c - if isinstance(c, Loop) and c.inputs == inputs - else Loop(c, inputs, parent=self, partial=partial) + ( + c + if isinstance(c, Loop) and c.inputs == inputs + else Loop(c, inputs, parent=self, partial=partial) + ) for c in loops ] if not self.loops: diff --git a/climetlab/version b/climetlab/version index 727d97b9..144996ed 100644 --- a/climetlab/version +++ b/climetlab/version @@ -1 +1 @@ -0.20.2 +0.20.3 diff --git a/tests/sources/test_merge.py b/tests/sources/test_merge.py index 82386234..3a0dad37 100644 --- a/tests/sources/test_merge.py +++ b/tests/sources/test_merge.py @@ -15,7 +15,6 @@ import xarray as xr from climetlab import load_source -from climetlab.testing import IN_GITHUB # These functionalities are variations around # http://xarray.pydata.org/en/stable/user-guide/combining.html#combining-multi