diff --git a/.bumpversion.cfg b/.bumpversion.cfg index bf3a149c9..342de4d60 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.38.0 +current_version = 1.39.0 commit = True tag = False tag_name = {new_version} @@ -30,11 +30,11 @@ search = {current_version} replace = {new_version} [bumpversion:file:RELEASE.txt] -search = {current_version} 2023-11-21T16:50:24Z +search = {current_version} 2023-11-27T17:57:33Z replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ} [bumpversion:part:releaseTime] -values = 2023-11-21T16:50:24Z +values = 2023-11-27T17:57:33Z [bumpversion:file(version):birdhouse/config/canarie-api/docker_configuration.py.template] search = 'version': '{current_version}' diff --git a/CHANGES.md b/CHANGES.md index be484d243..6593da126 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,19 @@ [//]: # (list changes here, using '-' for each new entry, remove this when items are added) +[1.39.0](https://github.com/bird-house/birdhouse-deploy/tree/1.39.0) (2023-11-27) +------------------------------------------------------------------------------------------------------------------ + +## Changes + +- Add a Magpie Webhook to create the Magpie resources corresponding to the STAC-API path elements when a `STAC-API` + `POST /collections/{collection_id}` or `POST /collections/{collection_id}/items/{item_id}` request is accomplished. + - When creating the STAC `Item`, the `source` entry in `links` corresponding to a `THREDDS` file on the same instance + is used to define the Magpie `resource_display_name` corresponding to a file to be mapped later on + (eg: a NetCDF `birdhouse/test-data/tc_Anon[...].nc`). + - Checking same instance `source` path is necessary because `STAC` could refer to external assets, and we do not want + to inject Magpie resource that are not part of the active instance where the hook is running. + [1.38.0](https://github.com/bird-house/birdhouse-deploy/tree/1.38.0) (2023-11-21) ------------------------------------------------------------------------------------------------------------------ diff --git a/Makefile b/Makefile index 63c4f86a4..516eb238b 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Generic variables override SHELL := bash override APP_NAME := birdhouse-deploy -override APP_VERSION := 1.38.0 +override APP_VERSION := 1.39.0 # utility to remove comments after value of an option variable override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g") diff --git a/README.rst b/README.rst index 3c67559cd..30f8de2e3 100644 --- a/README.rst +++ b/README.rst @@ -14,13 +14,13 @@ for a full-fledged production platform. * - releases - | |latest-version| |commits-since| -.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.38.0.svg +.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.39.0.svg :alt: Commits since latest release - :target: https://github.com/bird-house/birdhouse-deploy/compare/1.38.0...master + :target: https://github.com/bird-house/birdhouse-deploy/compare/1.39.0...master -.. |latest-version| image:: https://img.shields.io/badge/tag-1.38.0-blue.svg?style=flat +.. |latest-version| image:: https://img.shields.io/badge/tag-1.39.0-blue.svg?style=flat :alt: Latest Tag - :target: https://github.com/bird-house/birdhouse-deploy/tree/1.38.0 + :target: https://github.com/bird-house/birdhouse-deploy/tree/1.39.0 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest :alt: ReadTheDocs Build Status (latest version) diff --git a/RELEASE.txt b/RELEASE.txt index e00b91d6c..dbe32d625 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -1 +1 @@ -1.38.0 2023-11-21T16:50:24Z +1.39.0 2023-11-27T17:57:33Z diff --git a/birdhouse/components/stac/config/magpie/config.yml.template b/birdhouse/components/stac/config/magpie/config.yml.template index f8157469f..8351be5c3 100644 --- a/birdhouse/components/stac/config/magpie/config.yml.template +++ b/birdhouse/components/stac/config/magpie/config.yml.template @@ -7,6 +7,15 @@ providers: c4i: false type: api sync_type: api + hooks: + - type: response + path: "/stac/collections/?" + method: POST + target: /opt/birdhouse/src/magpie/hooks/stac_hooks.py:create_collection_resource + - type: response + path: "/stac/collections/[\\w-]+/items/?" + method: POST + target: /opt/birdhouse/src/magpie/hooks/stac_hooks.py:create_item_resource permissions: # create a default 'stac' resource under 'stac' service diff --git a/birdhouse/components/stac/config/magpie/stac_hooks.py b/birdhouse/components/stac/config/magpie/stac_hooks.py new file mode 100644 index 000000000..c6b150a06 --- /dev/null +++ b/birdhouse/components/stac/config/magpie/stac_hooks.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +These hooks will be running within Twitcher, using MagpieAdapter context, applied for STAC requests. + +The code below can make use of any package that is installed by Magpie/Twitcher. + +.. seealso:: + Documentation about Magpie/Twitcher request/response hooks is available here: + https://pavics-magpie.readthedocs.io/en/latest/configuration.html#service-hooks +""" + +import re +from typing import TYPE_CHECKING, List, Dict + +from magpie.api.management.resource import resource_utils as ru +from magpie.api.requests import get_service_matchdict_checked +from magpie.models import Route +from magpie.utils import get_logger +from magpie.db import get_session_from_other +from ziggurat_foundations.models.services.resource import ResourceService + +if TYPE_CHECKING: + from pyramid.response import Response + from sqlalchemy.orm.session import Session + +LOGGER = get_logger("magpie.stac") + +def create_collection_resource(response): + # type: (Response) -> Response + """ + Create the stac collection resource + """ + request = response.request + body = request.json + collection_id = body["id"] + try: + display_name = extract_display_name(body["links"]) + except Exception as exc: + LOGGER.error("Error when extracting display_name from links %s %s", body["links"], str(exc), exc_info=exc) + return response + + # note: matchdict reference of Twitcher owsproxy view is used, just so happens to be same name as Magpie + service = get_service_matchdict_checked(request) + # Getting a new session from the request, since the current session found in the request is already handled with his own transaction manager. + session = get_session_from_other(request.db) + try: + # Create the resource tree + create_resource_tree(f"stac/collections/{collection_id}", 0, service.resource_id , session, display_name) + session.commit() + + except Exception as exc: + LOGGER.error("Unexpected error while creating the collection %s %s", display_name, str(exc), exc_info=exc) + session.rollback() + + return response + +def create_item_resource(response): + # type: (Response) -> Response + """ + Create the stac item resource + """ + request = response.request + body = request.json + item_id = body["id"] + try: + display_name = extract_display_name(body["links"]) + except Exception as exc: + LOGGER.error("Error when extracting display_name from links %s %s", body["links"], str(exc), exc_info=exc) + return response + + # Get the from url -> /collections/{collection_id}/items + collection_id = re.search(r'(?<=collections/)[0-9a-zA-Z_.-]+?(?=/items)', request.url).group() + + # note: matchdict reference of Twitcher owsproxy view is used, just so happens to be same name as Magpie + service = get_service_matchdict_checked(request) + # Getting a new session from the request, since the current session found in the request is already handled with his own transaction manager. + session = get_session_from_other(request.db) + try: + # Create the resource tree + create_resource_tree(f"stac/collections/{collection_id}/items/{item_id}", 0, service.resource_id, session, display_name) + session.commit() + + except Exception as exc: + LOGGER.error("Unexpected error while creating the item %s %s", display_name, str(exc), exc_info=exc) + session.rollback() + + return response + +def extract_display_name(links): + # type: (List[Dict[str, str]]) -> str + """ + Extract THREDD path from a STAC links + """ + display_name = None + for link in links: + if link["rel"] == "source": + # Example of title `thredds:birdhouse/CMIP6` + display_name = link["title"] + break + if not display_name: + raise ValueError("The display name was not extracted properly") + + return display_name + +def create_resource_tree(resource_tree, current_depth, parent_id, session, display_name): + # type: (str, int, int, session, str) -> None + """ + Create the resource tree on Magpie + """ + tree = resource_tree.split("/") + # We are at the max depth of the tree. + if current_depth > len(tree) - 1: + return + + resource_name = tree[current_depth] + query = session.query(ResourceService.model).filter(ResourceService.model.resource_name == resource_name, ResourceService.model.parent_id == parent_id) + resource = query.first() + + if resource is not None: + # Since the resource exists, we can use its id to create the next resource. + parent_id = resource.resource_id + next_depth = current_depth + 1 + create_resource_tree(resource_tree, next_depth, parent_id, session, display_name) + + # The resource wasn't found in the current depth, we need to create it. + else: + # Creating the last resource in the tree, we need to use the display_name. + if current_depth == len(tree) - 1: + ru.create_resource(resource_name, display_name, Route.resource_type_name, parent_id, db_session=session) + else: + # Creating the resource somewhere in the middle of the tree before using its id. + node = ru.create_resource(resource_name, None, Route.resource_type_name, parent_id, db_session=session) + parent_id = node.json["resource"]["resource_id"] + next_depth = current_depth + 1 + create_resource_tree(resource_tree, next_depth, parent_id, session, display_name) diff --git a/birdhouse/components/stac/config/twitcher/docker-compose-extra.yml b/birdhouse/components/stac/config/twitcher/docker-compose-extra.yml new file mode 100644 index 000000000..b84f09b74 --- /dev/null +++ b/birdhouse/components/stac/config/twitcher/docker-compose-extra.yml @@ -0,0 +1,10 @@ +version: "3.4" + +services: + # extend twitcher with MagpieAdapter hooks employed for STAC proxied requests + twitcher: + volumes: + # NOTE: MagpieAdapter hooks are defined within Magpie config, but it is actually Twitcher proxy that runs them + # target mount location depends on 'MAGPIE_PROVIDERS_CONFIG_PATH' environment variable that is found under `birdhouse/config/twitcher/docker-compose-extra.yml` + - ./components/stac/config/magpie/config.yml:/opt/birdhouse/src/magpie/config/stac-config.yml:ro + - ./components/stac/config/magpie/stac_hooks.py:/opt/birdhouse/src/magpie/hooks/stac_hooks.py:ro diff --git a/birdhouse/components/weaver/config/twitcher/docker-compose-extra.yml b/birdhouse/components/weaver/config/twitcher/docker-compose-extra.yml index e1a5fe6d8..fca74d5dc 100644 --- a/birdhouse/components/weaver/config/twitcher/docker-compose-extra.yml +++ b/birdhouse/components/weaver/config/twitcher/docker-compose-extra.yml @@ -5,6 +5,6 @@ services: twitcher: volumes: # NOTE: MagpieAdapter hooks are defined within Magpie config, but it is actually Twitcher proxy that runs them - # target mount location depends on main docker-compose 'MAGPIE_PROVIDERS_CONFIG_PATH' environment variable + # target mount location depends on 'MAGPIE_PROVIDERS_CONFIG_PATH' environment variable that is found under `birdhouse/config/twitcher/docker-compose-extra.yml` - ./components/weaver/config/magpie/config.yml:/opt/birdhouse/src/magpie/config/weaver-config.yml:ro - ./components/weaver/config/magpie/weaver_hooks.py:/opt/birdhouse/src/magpie/hooks/weaver_hooks.py:ro diff --git a/birdhouse/config/canarie-api/docker_configuration.py.template b/birdhouse/config/canarie-api/docker_configuration.py.template index c8de1a026..78874a790 100644 --- a/birdhouse/config/canarie-api/docker_configuration.py.template +++ b/birdhouse/config/canarie-api/docker_configuration.py.template @@ -109,8 +109,8 @@ SERVICES = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.38.0', - 'releaseTime': '2023-11-21T16:50:24Z', + 'version': '1.39.0', + 'releaseTime': '2023-11-27T17:57:33Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', @@ -142,8 +142,8 @@ PLATFORMS = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.38.0', - 'releaseTime': '2023-11-21T16:50:24Z', + 'version': '1.39.0', + 'releaseTime': '2023-11-27T17:57:33Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', diff --git a/docs/source/conf.py b/docs/source/conf.py index dbdf8be04..775b7d315 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -69,9 +69,9 @@ # built documents. # # The short X.Y version. -version = '1.38.0' +version = '1.39.0' # The full version, including alpha/beta/rc tags. -release = '1.38.0' +release = '1.39.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages.