Merge branch 'master' into longterm-monitoring

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.5.3
+current_version = 2.6.0
 commit = True
 tag = False
 tag_name = {new_version}
@@ -30,11 +30,11 @@ search = {current_version}
 replace = {new_version}
 
 [bumpversion:file:RELEASE.txt]
-search = {current_version} 2024-09-11T22:57:09Z
+search = {current_version} 2024-11-19T13:53:14Z
 replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ}
 
 [bumpversion:part:releaseTime]
-values = 2024-09-11T22:57:09Z
+values = 2024-11-19T13:53:14Z
 
 [bumpversion:file(version):birdhouse/components/canarie-api/docker_configuration.py.template]
 search = 'version': '{current_version}'

CHANGES.md

@@ -46,8 +46,80 @@
   * ``/thanos-query``: a prometheus-like query interface to inspect the data stored by thanos
   * ``/thanos-minio``: a minio web console to inspect the data stored by minio.
 
-- Update the prometheus version from `v2.19.0` to the current latest `v2.52.0`. This is required to support the interaction between 
-  prometheus and thanos.
+[2.6.0](https://github.com/bird-house/birdhouse-deploy/tree/2.6.0) (2024-11-19)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+
+- Add the `prometheus-log-parser` optional component
+
+  This component parses log files from other components and converts their logs to prometheus
+  metrics that are then ingested by the monitoring Prometheus instance (the one created by the
+  `components/monitoring` component).
+
+  For more information on how this component reads log files and converts them to prometheus components see
+  the [log-parser](https://github.com/DACCS-Climate/log-parser/) documentation.
+
+  To configure this component:
+
+  * set the `PROMETHEUS_LOG_PARSER_POLL_DELAY` variable to a number of seconds to set how often the log parser
+    checks if new lines have been added to log files (default: 1)
+  * set the `PROMETHEUS_LOG_PARSER_TAIL` variable to `"true"` to only parse new lines in log files. If unset,
+    this will parse all existing lines in the log file as well (default: `"true"`)
+
+  To view all metrics exported by the log parser:
+
+  * Navigate to the `https://<BIRDHOUSE_FQDN>/prometheus/graph` search page
+  * Put `{job="log_parser"}` in the search bar and click the "Execute" button
+
+- Update the prometheus version to the current latest `v2.53.3`. This is required to support   
+  loading multiple prometheus scrape configuration files with the `scrape_config_files`
+  configuration option.
+
+[2.5.5](https://github.com/bird-house/birdhouse-deploy/tree/2.5.5) (2024-11-14)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+- Jupyter env: new full build with latest of everything
+
+  See [Ouranosinc/PAVICS-e2e-workflow-tests#137](https://github.com/Ouranosinc/PAVICS-e2e-workflow-tests/pull/137)
+  for more info.
+
+
+[2.5.4](https://github.com/bird-house/birdhouse-deploy/tree/2.5.4) (2024-10-31)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+
+- THREDDS: add more options to configure `catalog.xml`
+  - The default THREDDS configuration creates two default datasets, the *Service Data* dataset and the 
+    *Main* dataset. The *Service Data* dataset is used internally and hosts WPS outputs. The *Main* dataset is the
+    place where users can access data served by THREDDS. Both of these are configured to serve files with the following 
+    extensions: .nc .ncml .txt .md .rst .csv
+
+  - In order to allow the THREDDS server to serve files with additional extensions, this introduces two new
+    variables:
+    - `THREDDS_SERVICE_DATA_EXTRA_FILE_FILTERS`: this allows users to specify additional [filter 
+        elements](https://docs.unidata.ucar.edu/tds/current/userguide/tds_dataset_scan_ref.html#including-only-desired-files) to the *Service Data* dataset. This is especially useful if a WPS 
+        outputs files with an extension other than the default (eg: .h5) to the `wps_outputs/` directory.
+    - `THREDDS_DATASET_DATASETSCAN_BODY`: this allows users to specify the whole body of the *Main* dataset's 
+       [`<datasetScan>`](https://docs.unidata.ucar.edu/tds/current/userguide/tds_dataset_scan_ref.html) element.
+       This allows users to fully customize how this dataset serves files.
+
+  - We limit the configuration options for the *Service Data* dataset more than the *Main* dataset because the *Service
+    Data* dataset requires a basic configuration in order to properly serve WPS outputs. Making significant changes
+    to this configuration could have unexpected negative impacts on WPS usage.
+
+  - In order to allow customization of the Magpie THREDDS configuration in case new file extensions are added we introduce
+    two additional variables:
+    - `THREDDS_MAGPIE_EXTRA_METADATA_PREFIXES`: additional file prefixes (ie. regular expression match patterns) that Magpie
+      should treat as metadata (accessible with "browse" permissions).
+    - `THREDDS_MAGPIE_EXTRA_DATA_PREFIXES`: additional file prefixes (ie. regular expression match patterns) that Magpie
+      should treat as data (accessible with "read" permissions).
+
+  - The defaults for these new variables are fully backwards compatible. Without changing these variables, the THREDDS
+    server should behave exactly the same as before except that .md files and .rst files are now considered metadata
+    files according to the Magpie configuration, meaning that they can now be viewed with "browse" permissions.
 
 [2.5.3](https://github.com/bird-house/birdhouse-deploy/tree/2.5.3) (2024-09-11)
 ------------------------------------------------------------------------------------------------------------------

Makefile

@@ -1,7 +1,7 @@
 # Generic variables
 override SHELL       := bash
 override APP_NAME    := birdhouse-deploy
-override APP_VERSION := 2.5.3
+override APP_VERSION := 2.6.0
 
 # utility to remove comments after value of an option variable
 override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g")

README.rst

@@ -18,13 +18,13 @@ for a full-fledged production platform.
     * - citation
       - | |citation|
 
-.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/2.5.3.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/2.6.0.svg
     :alt: Commits since latest release
-    :target: https://github.com/bird-house/birdhouse-deploy/compare/2.5.3...master
+    :target: https://github.com/bird-house/birdhouse-deploy/compare/2.6.0...master
 
-.. |latest-version| image:: https://img.shields.io/badge/tag-2.5.3-blue.svg?style=flat
+.. |latest-version| image:: https://img.shields.io/badge/tag-2.6.0-blue.svg?style=flat
     :alt: Latest Tag
-    :target: https://github.com/bird-house/birdhouse-deploy/tree/2.5.3
+    :target: https://github.com/bird-house/birdhouse-deploy/tree/2.6.0
 
 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest
     :alt: ReadTheDocs Build Status (latest version)

RELEASE.txt

@@ -1 +1 @@
-2.5.3 2024-09-11T22:57:09Z
+2.6.0 2024-11-19T13:53:14Z

birdhouse/components/README.rst

@@ -276,6 +276,7 @@ Here is a sample setup to test autodeploy:
   # then remove the running scheduler container
   birdhouse compose up -d --remove-orphans
 
+.. _Monitoring:
 
 Monitoring
 ==========

birdhouse/components/canarie-api/docker_configuration.py.template

@@ -108,8 +108,8 @@ SERVICES = {
             # NOTE:
             #   Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
-            'version': '2.5.3',
-            'releaseTime': '2024-09-11T22:57:09Z',
+            'version': '2.6.0',
+            'releaseTime': '2024-11-19T13:53:14Z',
             'institution': '${BIRDHOUSE_INSTITUTION}',
             'researchSubject': '${BIRDHOUSE_SUBJECT}',
             'supportEmail': '${BIRDHOUSE_SUPPORT_EMAIL}',
@@ -141,8 +141,8 @@ PLATFORMS = {
             # NOTE:
             #   Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
-            'version': '2.5.3',
-            'releaseTime': '2024-09-11T22:57:09Z',
+            'version': '2.6.0',
+            'releaseTime': '2024-11-19T13:53:14Z',
             'institution': '${BIRDHOUSE_INSTITUTION}',
             'researchSubject': '${BIRDHOUSE_SUBJECT}',
             'supportEmail': '${BIRDHOUSE_SUPPORT_EMAIL}',

birdhouse/components/jupyterhub/default.env

@@ -10,7 +10,7 @@ export JUPYTERHUB_IMAGE='${JUPYTERHUB_DOCKER}:${JUPYTERHUB_VERSION}'
 export JUPYTERHUB_IMAGE_URI='registry.hub.docker.com/${JUPYTERHUB_IMAGE}'
 
 # Jupyter single-user server images, can be overriden in env.local to have a space separated list of multiple images
-export JUPYTERHUB_DOCKER_NOTEBOOK_IMAGES="pavics/workflow-tests:py311-240506-update240508"
+export JUPYTERHUB_DOCKER_NOTEBOOK_IMAGES="pavics/workflow-tests:py311-241111"
 
 # Name of the image displayed on the JupyterHub image selection page
 # Can be overriden in env.local to have a space separated list of multiple images, the name order must correspond

birdhouse/components/monitoring/default.env

@@ -8,7 +8,7 @@ export GRAFANA_VERSION="7.0.3"
 export GRAFANA_DOCKER=grafana/grafana
 export GRAFANA_IMAGE='${GRAFANA_DOCKER}:${GRAFANA_VERSION}'
 
-export PROMETHEUS_VERSION="v2.52.0"
+export PROMETHEUS_VERSION="v2.53.3"
 export PROMETHEUS_DOCKER=prom/prometheus
 export PROMETHEUS_IMAGE='${PROMETHEUS_DOCKER}:${PROMETHEUS_VERSION}'
 

birdhouse/components/monitoring/prometheus.yml.template

@@ -20,6 +20,9 @@ scrape_configs:
   - targets:
     - ${BIRDHOUSE_FQDN}:9100
 
+scrape_config_files:
+  - "/etc/prometheus/scrape_config.d/*.yml"
+
 rule_files:
 - "/etc/prometheus/*.rules"
 

birdhouse/components/proxy/nginx.conf.template

@@ -15,6 +15,8 @@ http {
     include       /etc/nginx/mime.types;
     default_type  application/octet-stream;
 
+    # NOTE: if this log_format ever changes, make sure to update the relevant code in the
+    #       prometheus-log-parser component as well to match the change.
     log_format  main  '$remote_addr - $remote_user [$time_iso8601] "$request" '
                       '$status $body_bytes_sent "$http_referer" '
                       '"$http_user_agent" "$http_x_forwarded_for"';

birdhouse/components/thredds/catalog.xml.template

@@ -4,14 +4,14 @@
          xmlns:xlink="http://www.w3.org/1999/xlink" >
 
     <service name="all" serviceType="Compound" base="" >
-        <service name="http" serviceType="HTTPServer" base="/twitcher/ows/proxy/thredds/fileServer/" />
-        <service name="odap" serviceType="OpenDAP" base="/twitcher/ows/proxy/thredds/dodsC/" />
-        <service name="ncml" serviceType="NCML" base="/twitcher/ows/proxy/thredds/ncml/"/>
-        <service name="uddc" serviceType="UDDC" base="/twitcher/ows/proxy/thredds/uddc/"/>
-        <service name="iso" serviceType="ISO" base="/twitcher/ows/proxy/thredds/iso/"/>
-        <service name="wcs" serviceType="WCS" base="/twitcher/ows/proxy/thredds/wcs/" />
-        <service name="wms" serviceType="WMS" base="/twitcher/ows/proxy/thredds/wms/" />
-        <service name="subsetServer" serviceType="NetcdfSubset" base="/twitcher/ows/proxy/thredds/ncss/" />
+        <service name="http" serviceType="HTTPServer" base="${TWITCHER_PROTECTED_PATH}/thredds/fileServer/" />
+        <service name="odap" serviceType="OpenDAP" base="${TWITCHER_PROTECTED_PATH}/thredds/dodsC/" />
+        <service name="ncml" serviceType="NCML" base="${TWITCHER_PROTECTED_PATH}/thredds/ncml/"/>
+        <service name="uddc" serviceType="UDDC" base="${TWITCHER_PROTECTED_PATH}/thredds/uddc/"/>
+        <service name="iso" serviceType="ISO" base="${TWITCHER_PROTECTED_PATH}/thredds/iso/"/>
+        <service name="wcs" serviceType="WCS" base="${TWITCHER_PROTECTED_PATH}/thredds/wcs/" />
+        <service name="wms" serviceType="WMS" base="${TWITCHER_PROTECTED_PATH}/thredds/wms/" />
+        <service name="subsetServer" serviceType="NetcdfSubset" base="${TWITCHER_PROTECTED_PATH}/thredds/ncss/" />
     </service>
 
     <datasetScan name="${THREDDS_SERVICE_DATA_LOCATION_NAME}" ID="${THREDDS_SERVICE_DATA_URL_PATH}" path="${THREDDS_SERVICE_DATA_URL_PATH}" location="${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER}">
@@ -27,24 +27,14 @@
         <include wildcard="*.md" />
         <include wildcard="*.rst" />
         <include wildcard="*.csv" />
+        ${THREDDS_SERVICE_DATA_EXTRA_FILE_FILTERS}
       </filter>
 
     </datasetScan>
 
     <datasetScan name="${THREDDS_DATASET_LOCATION_NAME}" ID="${THREDDS_DATASET_URL_PATH}" path="${THREDDS_DATASET_URL_PATH}" location="${THREDDS_DATASET_LOCATION_ON_CONTAINER}">
 
-      <metadata inherited="true">
-        <serviceName>all</serviceName>
-      </metadata>
-
-      <filter>
-        <include wildcard="*.nc" />
-        <include wildcard="*.ncml" />
-        <include wildcard="*.txt" />
-        <include wildcard="*.md" />
-        <include wildcard="*.rst" />
-        <include wildcard="*.csv" />
-      </filter>
+      ${THREDDS_DATASET_DATASETSCAN_BODY}
 
     </datasetScan>
 

birdhouse/components/thredds/config/magpie/providers.cfg.template

@@ -15,21 +15,24 @@ providers:
         - ".+\\.ncml"  # match longest extension first to avoid tuncating it by match of sorter '.nc'
         - ".+\\.nc"
       metadata_type:
-        prefixes:
-          - null  # note: special YAML value evaluated as `no-prefix`, use quotes if literal value is needed
-          - "\\w+\\.gif"  # threddsIcon, folder icon, etc.
-          - "\\w+\\.ico"  # favicon
-          - "\\w+\\.txt"  # licence
-          - "\\w+\\.css"  # tds.css
-          - "catalog\\.\\w+"  # note: special case for `THREDDS` top-level directory (root) accessed for `BROWSE`
-          - catalog
-          - ncml
-          - uddc
-          - iso
+        prefixes: [
+          null,  # note: special YAML value evaluated as `no-prefix`, use quotes if literal value is needed
+          "\\w+\\.gif",  # threddsIcon, folder icon, etc.
+          "\\w+\\.ico",  # favicon
+          "\\w+\\.css",  # tds.css
+          "catalog\\.\\w+",  # note: special case for `THREDDS` top-level directory (root) accessed for `BROWSE`
+          catalog,
+          ncml,
+          uddc,
+          iso,
+          ${THREDDS_MAGPIE_EXTRA_METADATA_PREFIXES}
+        ]
       data_type:
-        prefixes:
-          - fileServer
-          - dodsC
-          - wcs
-          - wms
-          - ncss
+        prefixes: [
+          fileServer,
+          dodsC,
+          wcs,
+          wms,
+          ncss,
+          ${THREDDS_MAGPIE_EXTRA_DATA_PREFIXES}
+        ]

birdhouse/components/thredds/default.env

@@ -17,7 +17,25 @@ export THREDDS_SERVICE_DATA_LOCATION_NAME='Birdhouse'
 export THREDDS_DATASET_URL_PATH='datasets'
 export THREDDS_SERVICE_DATA_URL_PATH='birdhouse'
 
+export THREDDS_MAGPIE_EXTRA_METADATA_PREFIXES='".+\\.txt", ".+\\.md", ".+\\.rst"'
+export THREDDS_MAGPIE_EXTRA_DATA_PREFIXES=''
 
+export THREDDS_SERVICE_DATA_EXTRA_FILE_FILTERS=''
+
+export THREDDS_DATASET_DATASETSCAN_BODY='
+      <metadata inherited="true">
+        <serviceName>all</serviceName>
+      </metadata>
+
+      <filter>
+        <include wildcard="*.nc" />
+        <include wildcard="*.ncml" />
+        <include wildcard="*.txt" />
+        <include wildcard="*.md" />
+        <include wildcard="*.rst" />
+        <include wildcard="*.csv" />
+      </filter>
+'
 
 # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here
 VARS="
@@ -28,6 +46,7 @@ VARS="
   \$THREDDS_DATASET_LOCATION_NAME
   \$THREDDS_DATASET_URL_PATH
   \$THREDDS_DATASET_LOCATION_ON_CONTAINER
+  \$THREDDS_DATASET_DATASETSCAN_BODY
 "
 
 OPTIONAL_VARS="
@@ -39,6 +58,9 @@ OPTIONAL_VARS="
   \$THREDDS_IMAGE
   \$THREDDS_IMAGE_URI
   \$THREDDS_ADDITIONAL_CATALOG
+  \$THREDDS_SERVICE_DATA_EXTRA_FILE_FILTERS
+  \$THREDDS_MAGPIE_EXTRA_METADATA_PREFIXES
+  \$THREDDS_MAGPIE_EXTRA_DATA_PREFIXES
 "
 
 export DELAYED_EVAL="