Merge pull request #40 from Ouranosinc/DAC-570-mount-public-wpsoutputs

Dac 570 mount  wpsoutputs data

AUTHORS.rst

@@ -7,5 +7,6 @@ Contributors
 .. add yourselves to the list when submitting a PR to be in docs
 
 * Francis Charette Migneault <[email protected]> [`@fmigneault <https://github.com/fmigneault>`_]
+* Charles-William Cummings <[email protected]> [`@ChaamC <https://github.com/ChaamC>`_]
 * David Byrns <[email protected]> [`@dbyrns <https://github.com/dbyrns>`_]
 * Francis Pelletier <[email protected]> [`@f-PLT <https://github.com/f-PLT>`_]

CHANGES.rst

@@ -7,7 +7,12 @@ Changes
 `Unreleased <https://github.com/Ouranosinc/cowbird/tree/master>`_ (latest)
 ------------------------------------------------------------------------------------
 
-* Nothing yet.
+Features / Changes
+~~~~~~~~~~~~~~~~~~~~~
+* Add monitoring to the ``FileSystem`` handler to watch WPS outputs data.
+* Synchronize both public and user WPS outputs data to the workspace folder with hardlinks for user access.
+* Add resync endpoint ``/handlers/{handler_name}/resync`` to trigger a handler's resync operations. Only the
+  ``FileSystem`` handler is implemented for now, regenerating hardlinks associated to WPS outputs public data.
 
 `2.0.0 <https://github.com/Ouranosinc/cowbird/tree/2.0.0>`_ (2023-07-21)
 ------------------------------------------------------------------------------------

config/config.example.yml

@@ -45,6 +45,12 @@ handlers:
     priority: 1
     workspace_dir: ${WORKSPACE_DIR}
     jupyterhub_user_data_dir: ${JUPYTERHUB_USER_DATA_DIR}
+    wps_outputs_dir: ${WPS_OUTPUTS_DIR}
+    secure_data_proxy_name: ${SECURE_DATA_PROXY_NAME}
+    wps_outputs_res_name: ${WPS_OUTPUTS_RES_NAME}
+    public_workspace_wps_outputs_subpath: ${PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBPATH}
+    notebooks_dir_name: ${NOTEBOOKS_DIR_NAME}
+    user_wps_outputs_dir_name: ${USER_WPS_OUTPUTS_DIR_NAME}
 
 # [Required] This section defines how to synchronize permissions between Magpie services when they share resources
 sync_permissions:

cowbird/api/handlers/__init__.py

@@ -9,4 +9,5 @@ def includeme(config: Configurator) -> None:
     logger.info("Adding API base routes...")
     config.add_route(**s.service_api_route_info(s.HandlersAPI))
     config.add_route(**s.service_api_route_info(s.HandlerAPI))
+    config.add_route(**s.service_api_route_info(s.HandlerResyncAPI))
     config.scan()

cowbird/api/handlers/views.py

@@ -6,6 +6,7 @@
 from cowbird.api import requests as ar
 from cowbird.api import schemas as s
 from cowbird.handlers import get_handlers
+from cowbird.handlers.handler import Handler
 from cowbird.typedefs import JSON, AnyResponseType
 
 
@@ -20,17 +21,35 @@ def get_handlers_view(request: Request) -> AnyResponseType:
     return ax.valid_http(HTTPOk, content=data, detail=s.Handlers_GET_OkResponseSchema.description)
 
 
+def get_handler(request: Request) -> Handler:
+    """
+    Get a handler object from the parameters of a request.
+    """
+    handler_name = ar.get_path_param(request, "handler_name", http_error=HTTPBadRequest,
+                                     msg_on_fail=s.Handlers_GET_BadRequestResponseSchema.description)
+    handlers = list(filter(lambda handler: handler.name == handler_name, get_handlers(request)))
+    ax.verify_param(len(handlers), is_equal=True, param_compare=1, param_name="handler_name",
+                    http_error=HTTPNotFound, msg_on_fail=s.Handler_Check_NotFoundResponseSchema.description)
+    return handlers[0]
+
+
 @s.HandlerAPI.get(schema=s.Handler_GET_RequestSchema, tags=[s.HandlersTag],
                   response_schemas=s.Handler_GET_responses)
 @view_config(route_name=s.HandlerAPI.name, request_method="GET")
 def get_handler_view(request: Request) -> AnyResponseType:
     """
     Get handler details.
     """
-    handler_name = ar.get_path_param(request, "handler_name", http_error=HTTPBadRequest,
-                                     msg_on_fail=s.Handlers_GET_BadRequestResponseSchema.description)
-    handlers = list(filter(lambda handler: handler.name == handler_name, get_handlers(request)))
-    ax.verify_param(len(handlers), is_equal=True, param_compare=1, param_name="handler_name",
-                    http_error=HTTPNotFound, msg_on_fail=s.Handler_GET_NotFoundResponseSchema.description)
-    data: JSON = {"handler": handlers[0].json()}
+    data: JSON = {"handler": get_handler(request).json()}
     return ax.valid_http(HTTPOk, content=data, detail=s.Handlers_GET_OkResponseSchema.description)
+
+
+@s.HandlerResyncAPI.put(schema=s.HandlerResync_PUT_RequestSchema, tags=[s.HandlersTag],
+                        response_schemas=s.HandlerResync_PUT_responses)
+@view_config(route_name=s.HandlerResyncAPI.name, request_method="PUT")
+def resync_handler_view(request: Request) -> AnyResponseType:
+    """
+    Resync handler operation.
+    """
+    get_handler(request).resync()
+    return ax.valid_http(HTTPOk, detail=s.HandlerResync_PUT_OkResponseSchema.description)

cowbird/api/schemas.py

@@ -128,6 +128,9 @@ def generate_api_schema(swagger_base_spec: JSON) -> JSON:
 HandlerAPI = Service(
     path="/handlers/{handler_name}",
     name="handler_detail")
+HandlerResyncAPI = Service(
+    path="/handlers/{handler_name}/resync",
+    name="handler_resync")
 UserWebhookAPI = Service(
     path="/webhooks/users",
     name="user_webhook")
@@ -489,11 +492,20 @@ class Handler_GET_OkResponseSchema(BaseResponseSchemaAPI):
     body = Handler_GET_ResponseBodySchema(code=HTTPOk.code, description=description)
 
 
-class Handler_GET_NotFoundResponseSchema(BaseResponseSchemaAPI):
+class Handler_Check_NotFoundResponseSchema(BaseResponseSchemaAPI):
     description = "Could not find specified handler."
     body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description)
 
 
+class HandlerResync_PUT_RequestSchema(BaseRequestSchemaAPI):
+    path = Handler_RequestPathSchema()
+
+
+class HandlerResync_PUT_OkResponseSchema(BaseResponseSchemaAPI):
+    description = "Handler resync successful."
+    body = BaseResponseBodySchema(code=HTTPOk.code, description=description)
+
+
 class Handlers_POST_ForbiddenResponseSchema(BaseResponseSchemaAPI):
     description = "Handler registration forbidden."
     body = ErrorResponseBodySchema(code=HTTPForbidden.code, description=description)
@@ -708,7 +720,7 @@ class SwaggerAPI_GET_OkResponseSchema(colander.MappingSchema):
 Handler_GET_responses = {
     "200": Handler_GET_OkResponseSchema(),
     "401": UnauthorizedResponseSchema(),
-    "404": Handler_GET_NotFoundResponseSchema(),
+    "404": Handler_Check_NotFoundResponseSchema(),
     "406": NotAcceptableResponseSchema(),
     "500": InternalServerErrorResponseSchema(),
 }
@@ -721,6 +733,12 @@ class SwaggerAPI_GET_OkResponseSchema(colander.MappingSchema):
     "422": Handler_PATCH_UnprocessableEntityResponseSchema(),
     "500": InternalServerErrorResponseSchema(),
 }
+HandlerResync_PUT_responses = {
+    "200": HandlerResync_PUT_OkResponseSchema(),
+    "404": Handler_Check_NotFoundResponseSchema(),
+    "406": NotAcceptableResponseSchema(),
+    "500": InternalServerErrorResponseSchema(),
+}
 UserWebhook_POST_responses = {
     "200": UserWebhook_POST_OkResponseSchema(),
     "400": UserWebhook_POST_BadRequestResponseSchema(),

cowbird/config.py

@@ -4,7 +4,7 @@
 from typing import Any, Dict, List, Literal, Tuple, Union, cast, overload
 
 import yaml
-from schema import Optional, Regex, Schema
+from schema import And, Optional, Regex, Schema
 
 from cowbird.typedefs import (
     ConfigDict,
@@ -160,12 +160,23 @@ def validate_handlers_config_schema(handlers_cfg: Dict[str, HandlerConfig]) -> N
     """
     Validates the schema of the `handlers` section found in the config.
     """
+    str_not_empty_validator = And(str, lambda s: len(s) > 0)
     schema = Schema({
         str: {  # Handler name
+            # parameters common to all handlers
             Optional("active"): bool,
             Optional("priority"): int,
             Optional("url"): str,
             Optional("workspace_dir"): str,
+
+            # parameters for specific handlers
+            Optional("jupyterhub_user_data_dir"): str_not_empty_validator,
+            Optional("wps_outputs_dir"): str_not_empty_validator,
+            Optional("secure_data_proxy_name"): str_not_empty_validator,
+            Optional("wps_outputs_res_name"): str_not_empty_validator,
+            Optional("notebooks_dir_name"): str_not_empty_validator,
+            Optional("public_workspace_wps_outputs_subpath"): str_not_empty_validator,
+            Optional("user_wps_outputs_dir_name"): str_not_empty_validator,
         }
     }, ignore_extra_keys=True)
     schema.validate(handlers_cfg)

cowbird/handlers/handler.py

@@ -86,22 +86,6 @@ def json(self) -> JSON:
     def _user_workspace_dir(self, user_name: str) -> str:
         return os.path.join(self.workspace_dir, user_name)
 
-    @abc.abstractmethod
-    def get_resource_id(self, resource_full_name: str) -> int:
-        """
-        Each handler must provide this implementation required by the permission synchronizer.
-
-        The function needs to find the resource id in Magpie from the resource full name using its knowledge of the
-        service. If the resource doesn't already exist, the function needs to create it, again using its knowledge of
-        resource type and parent resource type if required.
-
-        .. todo: Could be moved to another abstract class (SynchronizableHandler) that some Handler could implement
-                 Permissions_synchronizer would then check instance type of handler while loading sync config
-
-                 TODO: Check if this TODO is still relevant considering latest Magpie implementation?
-        """
-        raise NotImplementedError
-
     @abc.abstractmethod
     def user_created(self, user_name: str) -> None:
         raise NotImplementedError
@@ -117,3 +101,7 @@ def permission_created(self, permission: Permission) -> None:
     @abc.abstractmethod
     def permission_deleted(self, permission: Permission) -> None:
         raise NotImplementedError
+
+    @abc.abstractmethod
+    def resync(self) -> None:
+        raise NotImplementedError

cowbird/handlers/impl/catalog.py

@@ -26,9 +26,6 @@ def __init__(self, settings: SettingsType, name: str, **kwargs: Any) -> None:
         super(Catalog, self).__init__(settings, name, **kwargs)
         # TODO: Need to monitor data directory
 
-    def get_resource_id(self, resource_full_name: str) -> int:
-        raise NotImplementedError
-
     def user_created(self, user_name: str) -> None:
         LOGGER.info("Start monitoring workspace of created user [%s]", user_name)
         Monitoring().register(self._user_workspace_dir(user_name), True, Catalog)
@@ -74,3 +71,7 @@ def on_modified(self, path: str) -> None:
         :param path: Absolute path of a new file/directory
         """
         LOGGER.info("The following path [%s] has just been modified", path)
+
+    def resync(self) -> None:
+        # FIXME: this should be implemented in the eventual task addressing the resync mechanism.
+        raise NotImplementedError