update

.github/workflows/pytest.yml

@@ -12,6 +12,10 @@ jobs:
       matrix:
         python-version: [3.9, '3.10', '3.11', '3.12']
     runs-on: ubuntu-latest
+    env:
+      PYTHON_VERSION: ${{ matrix.python-version }}
+      OS: ${{ matrix.os }}
+
     steps:
       - name: Python Setup
         uses: actions/setup-python@v2
@@ -20,16 +24,35 @@ jobs:
           architecture: x64
       - name: Checkout Source
         uses: actions/checkout@v2
-      - name: Install Dependencies
+      - name: Setup Conda Environment
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-version: latest
+          python-version: ${{ matrix.python-version }}
+          activate-environment: test-environment
+          channels: conda-forge
+          conda-remove-defaults: true
+          channel-priority: strict
+      - name: Set cache environment variables
+        shell: bash -l {0}
         run: |
-          pip install --upgrade pip
-          pip install -r requirements.txt
-          pip install pytest-timeout
-          pip install pytest-cov
+          echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV
+          CONDA_PREFIX=$(python -c "import sys; print(sys.prefix)")
+          echo "CONDA_PREFIX=$CONDA_PREFIX" >> $GITHUB_ENV
+      - name: Update environment
+        run: mamba env update -n test-environment -f continuous_integration/environment.yaml
+        if: steps.cache.outputs.cache-hit != 'true'
+
+      - name: Install mapgen
+        shell: bash -l {0}
+        run: |
+          python -m pip install --no-deps -e .;
       - name: Run Tests
-        run: python -m pytest -v --cov=./ --cov-report=xml --timeout=60
+        shell: bash -l {0}
+        run: |
+          pytest -v --cov=./ --cov-report=xml --timeout=60
       - name: Upload to Codecov
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v4
         with:
-          fail_ci_if_error: true
-          token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
+          file: ./coverage.xml
+          token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos

continuous_integration/environment.yaml

@@ -0,0 +1,28 @@
+name: test-environment
+channels:
+  - conda-forge
+dependencies:
+  - jinja2
+  - pyyaml
+  - pytest-mock
+  - uvicorn
+  - requests
+  - boto3
+  - rasterio
+  - satpy
+  - mapserver
+  - gdal
+  - libgdal
+  - cmocean
+  - lxml
+  - cartopy
+  - metpy
+  - pandas
+  - pip
+  - pip:
+    - pytest-lazy-fixtures
+    - pytest-timeout
+    - pytest
+    - pytest-cov
+    - WebTest
+

mapgen/modules/helpers.py

@@ -160,6 +160,7 @@ def handle_request(map_object, full_request):
             logger.debug("STYLES not in the request. Nothing to reset.")
             pass
         try:
+            logger.debug(f"PWD {os.getcwd()}")
             map_object.OWSDispatch( ows_req )
         except Exception as e:
             logger.error(f"status_code=500, mapscript fails to parse query parameters: {str(full_request)}, with error: {str(e)}")

robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /

tests/test_arome_arctic.py

@@ -31,12 +31,14 @@ def _create_arome_arctic_dataset_from_orig_data_without_reference_time():
     ds_trimmed.to_netcdf("test_arome_arctic_without_forecast_time_20241111T06Z.nc")
 
 @patch('mapgen.modules.get_quicklook.find_config_for_this_netcdf')
-def test_read_dataset(mock_read_config, tmpdir, caplog):
+@patch('mapgen.modules.helpers._find_summary_from_csw')
+def test_read_dataset(mock_csw, mock_read_config, tmpdir, caplog):
     """Test reading the dataset"""
     netcdf_path = "tests/data/test_arome_arctic.nc"
     query_string = ""
     http_host = "localhost"
     url_scheme = "http"
+    mock_csw.return_value = "TEST_CSW"
     mock_read_config.return_value =  {'pattern': '^(.*data/test_arome_arctic.nc)$',
                                  'base_netcdf_directory': '.',
                                  'module': 'mapgen.modules.arome_arctic_quicklook',
@@ -63,12 +65,14 @@ def test_read_dataset(mock_read_config, tmpdir, caplog):
     tmpdir.remove()
 
 @patch('mapgen.modules.get_quicklook.find_config_for_this_netcdf')
-def test_get_quicklook_arome_arctic_no_reference_time(mock_read_config, tmpdir, caplog):
+@patch('mapgen.modules.helpers._find_summary_from_csw')
+def test_get_quicklook_arome_arctic_no_reference_time(mock_csw, mock_read_config, tmpdir, caplog):
     """Test for a none existing netcdf file"""
     netcdf_path = "tests/data/test_arome_arctic_without_forecast_time_20241111T06Z.nc"
     query_string = ""
     http_host = "localhost"
     url_scheme = "http"
+    mock_csw.return_value = "TEST CSW"
     mock_read_config.return_value =  {'pattern': '^(.*data/test_arome_arctic_without_forecast_time_(\d{8}T\d{2})Z.nc)$',
                                       'base_netcdf_directory': '.',
                                       'module': 'mapgen.modules.arome_arctic_quicklook',
@@ -133,16 +137,18 @@ def test_read_dataset_base_path_missing_base(mock_read_config, tmpdir):
     tmpdir.remove()
 
 @patch('mapgen.modules.get_quicklook.find_config_for_this_netcdf')
-def test_read_dataset_request_getmap(mock_read_config, tmpdir, caplog):
+@patch('mapgen.modules.helpers._find_summary_from_csw')
+def test_read_dataset_request_getmap(mock_csw, mock_read_config, tmpdir, caplog):
     """Test reading the dataset"""
     netcdf_path = "tests/data/test_arome_arctic.nc"
     query_string = ("?&SERVICE=WMS&VERSION=1.3.0&REQUEST=GetMap&LAYERS=air_temperature_2m&WIDTH=767&HEIGHT=880&CRS=EPSG%3A3857&"
                     "BBOX=-4822584.097826986,7566329.44660393,10215292.880334288,24819695.471091438&STYLES=raster&FORMAT=image/png&"
                     "TRANSPARENT=TRUE&&TIME=2024-11-11T07%3A00%3A00Z")
     http_host = "localhost"
     url_scheme = "http"
+    mock_csw.return_value = "TEST CSW"
     mock_read_config.return_value =  {'pattern': '^(.*data/test_arome_arctic.nc)$',
-                                 'base_netcdf_directory': '.',
+                                 'base_netcdf_directory': os.getcwd(),
                                  'module': 'mapgen.modules.arome_arctic_quicklook',
                                  'module_function': 'arome_arctic_quicklook',
                                  'mapfile_template': os.path.join(tmpdir, 'test.map'),
@@ -171,14 +177,16 @@ def test_read_dataset_request_getmap(mock_read_config, tmpdir, caplog):
     tmpdir.remove()
 
 @patch('mapgen.modules.get_quicklook.find_config_for_this_netcdf')
-def test_read_dataset_request_getmap_contour_style(mock_read_config, tmpdir, caplog):
+@patch('mapgen.modules.helpers._find_summary_from_csw')
+def test_read_dataset_request_getmap_contour_style(mock_csw, mock_read_config, tmpdir, caplog):
     """Test reading the dataset"""
     netcdf_path = "tests/data/test_arome_arctic.nc"
     query_string = ("?&SERVICE=WMS&VERSION=1.3.0&REQUEST=GetMap&LAYERS=air_temperature_2m&WIDTH=767&HEIGHT=880&CRS=EPSG%3A3857&"
                     "BBOX=-4822584.097826986,7566329.44660393,10215292.880334288,24819695.471091438&STYLES=contour&FORMAT=image/png&"
                     "TRANSPARENT=TRUE&&TIME=2024-11-11T07%3A00%3A00Z")
     http_host = "localhost"
     url_scheme = "http"
+    mock_csw.return_value = "TEST CSW"
     mock_read_config.return_value =  {'pattern': '^(.*data/test_arome_arctic.nc)$',
                                  'base_netcdf_directory': '.',
                                  'module': 'mapgen.modules.arome_arctic_quicklook',
@@ -199,14 +207,16 @@ def test_read_dataset_request_getmap_contour_style(mock_read_config, tmpdir, cap
     assert "STYLES: contour" in caplog.text
 
 @patch('mapgen.modules.get_quicklook.find_config_for_this_netcdf')
-def test_read_dataset_request_getmap_vector_barbs(mock_read_config, tmpdir, caplog):
+@patch('mapgen.modules.helpers._find_summary_from_csw')
+def test_read_dataset_request_getmap_vector_barbs(mock_csw, mock_read_config, tmpdir, caplog):
     """Test reading the dataset"""
     netcdf_path = "tests/data/test_arome_arctic.nc"
     query_string = ("?&SERVICE=WMS&VERSION=1.3.0&REQUEST=GetMap&LAYERS=wind_10m_vector&WIDTH=767&HEIGHT=880&CRS=EPSG%3A3857&"
                     "BBOX=-4891698.287598623,6944679.416003374,10026224.553483257,24060418.79038676&STYLES=Wind_Barbs&"
                     "FORMAT=image/png&TRANSPARENT=TRUE&&TIME=2024-11-11T07%3A00%3A00Z&DIM_SPACING=32&DIM_COLOUR=light-green")
     http_host = "localhost"
     url_scheme = "http"
+    mock_csw.return_value = "TEST CSW"
     mock_read_config.return_value =  {'pattern': '^(.*data/test_arome_arctic.nc)$',
                                  'base_netcdf_directory': '.',
                                  'module': 'mapgen.modules.arome_arctic_quicklook',
@@ -225,14 +235,16 @@ def test_read_dataset_request_getmap_vector_barbs(mock_read_config, tmpdir, capl
     assert 'VECTOR wind_10m_vector x_wind_10m y_wind_10m' in caplog.text
 
 @patch('mapgen.modules.get_quicklook.find_config_for_this_netcdf')
-def test_read_dataset_request_getmap_vector_vector(mock_read_config, tmpdir, caplog):
+@patch('mapgen.modules.helpers._find_summary_from_csw')
+def test_read_dataset_request_getmap_vector_vector(mock_csw, mock_read_config, tmpdir, caplog):
     """Test reading the dataset"""
     netcdf_path = "tests/data/test_arome_arctic.nc"
     query_string = ("?&SERVICE=WMS&VERSION=1.3.0&REQUEST=GetMap&LAYERS=wind_10m_vector&WIDTH=767&HEIGHT=880&CRS=EPSG%3A3857&"
                     "BBOX=-4891698.287598623,6944679.416003374,10026224.553483257,24060418.79038676&STYLES=vector&"
                     "FORMAT=image/png&TRANSPARENT=TRUE&&TIME=2024-11-11T07%3A00%3A00Z&DIM_SPACING=32&DIM_COLOUR=light-green")
     http_host = "localhost"
     url_scheme = "http"
+    mock_csw.return_value = "TEST CSW"
     mock_read_config.return_value =  {'pattern': '^(.*data/test_arome_arctic.nc)$',
                                  'base_netcdf_directory': '.',
                                  'module': 'mapgen.modules.arome_arctic_quicklook',