Merge branch 'release/0.6.3'

.github/ci-config.yml

@@ -1,8 +1,9 @@
+cmake_options: -DENABLE_FDB_BUILD_TOOLS=ON
 dependencies: |
   ecmwf/ecbuild
   MathisRosenhauer/libaec@master
   ecmwf/eccodes
-  ecmwf/eckit
+  ecmwf/eckit@develop
   ecmwf/metkit
   ecmwf/fdb
 dependency_branch: develop

.github/ci-hpc-config.yml

@@ -1,4 +1,6 @@
 build:
+  cmake_options:
+    - "-DENABLE_FDB_BUILD_TOOLS=ON"
   modules:
     - ninja
     - aec

VERSION

@@ -1 +1 @@
-0.6.2
+0.6.3

pygribjump/src/pygribjump/VERSION

@@ -0,0 +1 @@
+../../../VERSION

pygribjump/src/pygribjump/__init__.py

@@ -1 +1,2 @@
 from .pygribjump import *
+from ._version import __version__

pygribjump/src/pygribjump/_version.py

@@ -0,0 +1,5 @@
+from pathlib import Path
+from .pygribjump import *
+
+with open(Path(__file__).parent / "VERSION") as f:
+    __version__ = f.read().strip()

pygribjump/src/pygribjump/gribjump_c.h

@@ -26,11 +26,13 @@ int gribjump_result_values(gribjump_extraction_result_t* result, double*** value
 int gribjump_result_values_nocopy(gribjump_extraction_result_t* result, double*** values, unsigned long* nrange, unsigned long** nvalues);
 int gribjump_result_mask(gribjump_extraction_result_t* result,  unsigned long long*** masks, unsigned long* nrange, unsigned long** nmasks);
 int gribjump_delete_result(gribjump_extraction_result_t* result);
-int gribjump_new_axes(gj_axes_t** axes, const char* reqstr, gribjump_handle_t* gj);
+int gribjump_new_axes(gj_axes_t** axes, const char* reqstr, int* level, const char* ctx, gribjump_handle_t* gj);
 int gribjump_axes_keys(gj_axes_t* axes, const char*** keys_out, unsigned long* size);
 int gribjump_axes_values(gj_axes_t* axes, const char* key, const char*** values_out, unsigned long* size);
 int gribjump_delete_axes(gj_axes_t* axes);
 
 int gribjump_initialise();
+int gribjump_version_c(const char** version);
+int gribjump_git_sha1_c(const char** sha1);
 
 const char* gribjump_error_string(int err);

pygribjump/src/pygribjump/pygribjump.py

@@ -17,6 +17,8 @@
 import cffi
 import os
 import findlibs
+import warnings
+from ._version import __version__
 
 ffi = cffi.FFI()
 
@@ -44,8 +46,7 @@ def __init__(self):
         self.__lib = ffi.dlopen(libName)
 
         # All of the executable members of the CFFI-loaded library are functions in the GribJump
-        # C API. These should be wrapped with the correct error handling. Otherwise forward
-        # these on directly.
+        # C API. These should be wrapped with the correct error handling.
 
         for f in dir(self.__lib):
             try:
@@ -59,6 +60,14 @@ def __init__(self):
         # Initialise the library, and set it up for python-appropriate behaviour
         self.gribjump_initialise()
 
+        # Check versions
+        tmp_str = ffi.new('char**')
+        self.gribjump_version_c(tmp_str)
+        versionstr = ffi.string(tmp_str[0]).decode('utf-8')
+
+        if versionstr != __version__:
+            warnings.warn(f"GribJump library version {versionstr} does not match pygribjump version {__version__}")
+
     def __read_header(self, hdr_path):
         with open(hdr_path, 'r') as f:
             return f.read()
@@ -117,10 +126,7 @@ def extract(self, polyrequest, ctx=None, dump=True):
         nfields = ffi.new('unsigned long**')
         nrequests = len(requests)
         c_requests = ffi.new('gribjump_extraction_request_t*[]', [r.ctype for r in requests])
-        if (ctx):
-            logctx=str(ctx)
-        else:
-            logctx=""
+        logctx=str(ctx) if ctx else "pygribjump_extract"
 
         logctx_c = ffi.new('const char[]', logctx.encode('ascii'))
         lib.extract(self.__gribjump, c_requests, nrequests, results_array, nfields, logctx_c)
@@ -210,12 +216,17 @@ def extract_single(self, request):
         ]
         return res
 
-    def axes(self, req):
+
+    def axes(self, req, level=3, ctx=None):
         # note old axes used a dict in. This is now a string.
+        logctx=str(ctx) if ctx else "pygribjump_axes"
+        ctx_c = ffi.new('const char[]', logctx.encode('ascii'))
         requeststr = dic_to_request(req)
         newaxes = ffi.new('gj_axes_t**')
         reqstr = ffi.new('const char[]', requeststr.encode('ascii'))
-        lib.gribjump_new_axes(newaxes, reqstr, self.__gribjump)
+        level_c = ffi.new('int*', level)
+        lib.gribjump_new_axes(newaxes, reqstr, level_c, ctx_c, self.__gribjump)
+
         # TODO want to return a dict like:
         # {key: [value1, value2, ...], ...}
         # each key and value is a string
@@ -349,3 +360,10 @@ def dic_to_request(dic):
     # e.g. {"class":"od", "expver":"0001", "levtype":"pl"} -> "class=od,expver=0001,levtype=pl"
     return ','.join(['='.join([k, v]) for k, v in dic.items()])
 
+def version():
+    return __version__
+
+def library_version():
+    tmp_str = ffi.new('char**')
+    lib.gribjump_version_c(tmp_str)
+    return ffi.string(tmp_str[0]).decode('utf-8')

pygribjump/tests/test_pygribjump.py

@@ -83,3 +83,16 @@ def test_extract_simple_sunshine_case(read_only_fdb_setup) -> None:
 
     actual = grib_jump.extract(polyrequest)
     assert numpy.array_equal(expected, actual[0][0][0][0], equal_nan=True)
+
+def test_axes(read_only_fdb_setup) -> None:
+    gribjump = pygj.GribJump()
+    req = {
+        "date": "20230508",
+    }
+    ax1 = gribjump.axes(req, level=1) # {'class': ['od'], 'date': ['20230508'], 'domain': ['g'], 'expver': ['0001'], 'stream': ['oper'], 'time': ['1200']}
+    ax2 = gribjump.axes(req, level=2) # {'class': ['od'], 'date': ['20230508'], 'domain': ['g'], 'expver': ['0001'], 'levtype': ['sfc'], 'stream': ['oper'], 'time': ['1200'], 'type': ['fc']}
+    ax3 = gribjump.axes(req, level=3) # {'class': ['od'], 'date': ['20230508'], 'domain': ['g'], 'expver': ['0001'], 'levelist': [''], 'levtype': ['sfc'], 'param': ['151130'], 'step': ['1'], 'stream': ['oper'], 'time': ['1200'], 'type': ['fc']}
+
+    assert len(ax1.keys()) == 6
+    assert len(ax2.keys()) == 8
+    assert len(ax3.keys()) == 11

pygribjump/tests/test_version.py

@@ -0,0 +1,5 @@
+from importlib.metadata import version
+import pygribjump
+
+def test_version():
+    assert pygribjump.__version__ == version("pygribjump")

pyproject.toml

@@ -4,3 +4,45 @@ addopts = "-vv -s"
 testpaths = [
     "pygribjump/tests"
 ]
+
+# pyproject.toml
+
+[build-system]
+requires = ["setuptools", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "pygribjump"
+description = "Python interface to GribJump"
+readme = "README.md"
+requires-python = ">=3.8"
+dynamic = ["version"]
+authors = [
+    { name = "ECMWF", email = "[email protected]" }
+]
+urls = { "Home" = "https://github.com/ecmwf/gribjump" }
+dependencies = [
+    "cffi~=1.17",
+    "numpy~=2.1",
+    "pytest~=8.3",
+    "setuptools~=75.1",
+    "findlibs~=0.0.5",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pyyaml",
+    "pyfdb",
+]
+
+[tool.setuptools.dynamic]
+version = { file = ["VERSION"] }
+
+[tool.setuptools]
+packages = ["pygribjump"]
+package-dir = { "pygribjump" = "./pygribjump/src/pygribjump" }
+include-package-data = true
+zip-safe = false
+
+[tool.setuptools.package-data]
+"pygribjump" = ["VERSION", "pygribjump/src/pygribjump/gribjump_c.h"]

src/gribjump/CMakeLists.txt

@@ -29,9 +29,11 @@ list( APPEND gribjump_srcs
     Stats.cc
     Stats.h
 
+    tools/GribJumpTool.h
+    tools/GribJumpTool.cc
     tools/ToolUtils.h
     tools/ToolUtils.cc
-    
+
     remote/RemoteGribJump.cc
     remote/RemoteGribJump.h
 
@@ -45,6 +47,7 @@ list( APPEND gribjump_srcs
     ExtractionData.cc
     ExtractionData.h
     Metrics.h
+    Metrics.cc
     Types.h
 )
 

src/gribjump/Config.cc

@@ -37,7 +37,8 @@ Config::Config() {
 
 Config::Config(const eckit::PathName path) :
     eckit::LocalConfiguration(eckit::YAMLConfiguration(path)),
-    serverMap_{loadServerMap()} {
+    serverMap_{loadServerMap()},
+    path_{path} {
 }
 
 std::map<std::string, std::string> Config::loadServerMap() const {

src/gribjump/Config.h

@@ -23,12 +23,16 @@ class Config : public eckit::LocalConfiguration {
     Config(const eckit::PathName);
 
     const std::map<std::string, std::string>& serverMap() const { return serverMap_; }
+
+    ///@note : Will be empty if default config is used
+    const std::string& path() const { return path_; }
 
 private:
     std::map<std::string, std::string> loadServerMap() const;
 
 private:
     std::map<std::string, std::string> serverMap_;
+    std::string path_;
 };
 
 } // namespace gribjump

src/gribjump/Engine.cc

@@ -159,6 +159,7 @@ filemap_t Engine::buildFileMap(const ExtractionRequests& requests, ExItemMap& ke
     }
 
     const metkit::mars::MarsRequest req = unionRequest(marsrequests);
+    MetricsManager::instance().set("union_request", req.asString());
     timer.reset("Gribjump Engine: Flattened requests and constructed union request");
 
     filemap_t filemap = FDBLister::instance().fileMap(req, keyToExtractionItem);
@@ -248,14 +249,19 @@ void Engine::scheduleTasks(filemap_t& filemap){
 
 ResultsMap Engine::extract(const ExtractionRequests& requests, bool flatten) {
 
+    eckit::Timer timer("Engine::extract");
     ExItemMap keyToExtractionItem = buildKeyToExtractionItem(requests, flatten); // Owns the ExtractionItems
     filemap_t filemap = buildFileMap(requests, keyToExtractionItem);
-    eckit::Timer timer("Engine::extract");
+    MetricsManager::instance().set("elapsed_build_filemap", timer.elapsed());
+    timer.reset("Gribjump Engine: Built file map");
 
     scheduleTasks(filemap);
+    MetricsManager::instance().set("elapsed_tasks", timer.elapsed());
     timer.reset("Gribjump Engine: All tasks finished");
 
     ResultsMap results = collectResults(keyToExtractionItem);
+    MetricsManager::instance().set("elapsed_collect_results", timer.elapsed());
+
     timer.reset("Gribjump Engine: Repackaged results");
 
     return results;
@@ -275,29 +281,43 @@ ResultsMap Engine::collectResults(ExItemMap& keyToExtractionItem) {
 
 size_t Engine::scan(const MarsRequests& requests, bool byfiles) {
 
-    const std::map< eckit::PathName, eckit::OffsetList > files = FDBLister::instance().filesOffsets(requests);
-
-    size_t counter = 0;
+    const std::map< eckit::PathName, eckit::OffsetList > filemap = FDBLister::instance().filesOffsets(requests);
 
-    if (byfiles) {
-        for (auto& [fname, offsets] : files) {
-            taskGroup_.enqueueTask(new FileScanTask(taskGroup_, counter++, fname, offsets));
+    if (byfiles) { // ignore offsets and scan entire file
+        std::vector<eckit::PathName> files;
+        for (auto& [fname, offsets] : filemap) {
+            files.push_back(fname);
         }
+
+        return scan(files);
+
     }
-    else {
-        for (auto& [fname, offsets] : files) {
-            taskGroup_.enqueueTask(new FileScanTask(taskGroup_, counter++, fname, {}));
-        }
+
+    size_t counter = 0;
+
+    for (auto& [fname, offsets] : filemap) {
+        taskGroup_.enqueueTask(new FileScanTask(taskGroup_, counter++, fname, offsets));
+    }
+    taskGroup_.waitForTasks();
+
+    return filemap.size();
+}
+
+size_t Engine::scan(std::vector<eckit::PathName> files) {
+    size_t counter = 0;
+
+    for (auto& fname : files) {
+        taskGroup_.enqueueTask(new FileScanTask(taskGroup_, counter++, fname, {}));
     }
 
-    // Wait for the tasks to complete
     taskGroup_.waitForTasks();
 
     return files.size();
 }
 
-std::map<std::string, std::unordered_set<std::string> > Engine::axes(const std::string& request) {
-    return FDBLister::instance().axes(request);
+std::map<std::string, std::unordered_set<std::string> > Engine::axes(const std::string& request, int level) {
+    MetricsManager::instance().set("request", request);
+    return FDBLister::instance().axes(request, level);
 }
 
 void Engine::reportErrors(eckit::Stream& client) {
@@ -307,10 +327,6 @@ void Engine::reportErrors(eckit::Stream& client) {
 void Engine::raiseErrors() {
     taskGroup_.raiseErrors();
 }
-void Engine::updateMetrics(Metrics& metrics) {
-    metrics.nTasks = taskGroup_.nTasks();
-    metrics.nFailedTasks = taskGroup_.nErrors();
-}
 //----------------------------------------------------------------------------------------------------------------------
 
 } // namespace gribjump

src/gribjump/Engine.h

@@ -32,11 +32,11 @@ class Engine {
 
     // byfiles: scan entire file, not just fields matching request
     size_t scan(const MarsRequests& requests, bool byfiles = false);
+    size_t scan(std::vector<eckit::PathName> files);
 
-    std::map<std::string, std::unordered_set<std::string> > axes(const std::string& request);
+    std::map<std::string, std::unordered_set<std::string> > axes(const std::string& request, int level=3);
 
     void scheduleTasks(filemap_t& filemap);
-    void updateMetrics(Metrics& metrics);
 
     void reportErrors(eckit::Stream& client_);
     void raiseErrors();