From 9f64eb80057b2fe65efbf91daf11a1f99e4cbddc Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Fri, 22 Nov 2024 15:14:23 -0500 Subject: [PATCH 01/18] enable log redirects --- bin/birdhouse | 26 +++++- birdhouse/birdhouse-compose.sh | 4 +- birdhouse/read-configs.include.sh | 6 +- .../scripts/clear-running-wps-jobs-in-db.sh | 2 +- birdhouse/scripts/logging.include.sh | 89 ++++++++++++------- birdhouse/scripts/sync-data | 2 +- 6 files changed, 89 insertions(+), 40 deletions(-) diff --git a/bin/birdhouse b/bin/birdhouse index ba5654be7..cd3480116 100755 --- a/bin/birdhouse +++ b/bin/birdhouse @@ -7,7 +7,7 @@ COMPOSE_DIR="$(dirname "${THIS_DIR}")/birdhouse" export BIRDHOUSE_COMPOSE="${BIRDHOUSE_COMPOSE:-"${COMPOSE_DIR}/birdhouse-compose.sh"}" export __BIRDHOUSE_SUPPORTED_INTERFACE=True -USAGE="USAGE: $0 [-h|--help] [-b|--backwards-compatible] [-e|--env-file local-env-file] {info|compose|configs}" +USAGE="USAGE: $0 [-h|--help] [-b|--backwards-compatible] [-e|--env-file local-env-file] {[--log-stdout] | [--log-file log-file-path]} {info|compose|configs}" HELP="$USAGE Manage the Birdhouse software stack. @@ -21,6 +21,8 @@ Options: -h, --help Print this message and exit -b, --backwards-compatible Run in backwards compatible mode -e, --env-file string Override the local environment file, default is ${COMPOSE_DIR}/env.local + --log-stdout Write logs to stdout, default is to write to stderr + --log-file string Write logs to this file path, default is to write to stderr " CONFIGS_USAGE="USAGE: $0 configs [-h|--help] [-d|--default] {[-p|--print-config-command] | [-c|--command command]}" @@ -194,6 +196,26 @@ parse_args() { shift parse_args "$@" ;; + --log-stdout) + shift + [ "${LOG_FILE}" = 'True' ] && parse_args 'invalid arg that triggers usage message' + export BIRDHOUSE_LOG_FD=1 # The argument here takes precedence over the env variable + LOG_STDOUT=True + parse_args "$@" + ;; + --log-file=*) + arg_value="${1#*=}" + shift + parse_args --log-file "${arg_value}" "$@" + ;; + --log-file) + shift + [ "${LOG_STDOUT}" = 'True' ] && parse_args 'invalid arg that triggers usage message' + export BIRDHOUSE_LOG_FILE=$(realpath "$1") # The argument here takes precedence over the env variable + LOG_FILE=True + shift + parse_args "$@" + ;; info) shift "${BIRDHOUSE_COMPOSE}" info "$@" @@ -212,7 +234,7 @@ parse_args() { echo "$HELP" ;; -??*) - parse_multiple_short_flags parse_configs_args "$@" + parse_multiple_short_flags parse_args "$@" ;; *) >&2 echo "$USAGE" diff --git a/birdhouse/birdhouse-compose.sh b/birdhouse/birdhouse-compose.sh index 04475a12b..d53e73c81 100755 --- a/birdhouse/birdhouse-compose.sh +++ b/birdhouse/birdhouse-compose.sh @@ -105,8 +105,8 @@ fi create_compose_conf_list # this sets COMPOSE_CONF_LIST log INFO "Displaying resolved compose configurations:" -echo "COMPOSE_CONF_LIST=" -echo ${COMPOSE_CONF_LIST} | tr ' ' '\n' | grep -v '^-f' +log INFO "COMPOSE_CONF_LIST=" +log INFO ${COMPOSE_CONF_LIST} | tr ' ' '\n' | grep -v '^-f' if [ x"$1" = x"info" ]; then log INFO "Stopping before execution of docker-compose command." diff --git a/birdhouse/read-configs.include.sh b/birdhouse/read-configs.include.sh index bad87dcfa..64e337c30 100644 --- a/birdhouse/read-configs.include.sh +++ b/birdhouse/read-configs.include.sh @@ -103,7 +103,7 @@ read_default_env() { . "${COMPOSE_DIR}/default.env" else - log WARN "'${COMPOSE_DIR}/default.env' not found" 1>&2 + log WARN "'${COMPOSE_DIR}/default.env' not found" fi } @@ -123,7 +123,7 @@ read_env_local() { eval "${saved_shell_options}" else - log WARN "'${BIRDHOUSE_LOCAL_ENV}' not found" 1>&2 + log WARN "'${BIRDHOUSE_LOCAL_ENV}' not found" fi } @@ -155,7 +155,7 @@ source_conf_files() { # corresponding PR are merged and old component names can be removed # after the corresponding PR are merge without any impact on the # autodeploy process. - log WARN "'${adir}' in ${conf_locations} does not exist" 1>&2 + log WARN "'${adir}' in ${conf_locations} does not exist" fi if [ -f "${adir}/default.env" ]; then # Source config settings of dependencies first if they haven't been sourced previously. diff --git a/birdhouse/scripts/clear-running-wps-jobs-in-db.sh b/birdhouse/scripts/clear-running-wps-jobs-in-db.sh index 897e3deb1..1cdfae0ef 100755 --- a/birdhouse/scripts/clear-running-wps-jobs-in-db.sh +++ b/birdhouse/scripts/clear-running-wps-jobs-in-db.sh @@ -11,7 +11,7 @@ fi # eg: DB_NAME=finch DB_NAME="$1" if [ -z "$DB_NAME" ]; then - log ERROR "please provide a database name, ex: finch" 1>&2 + log ERROR "please provide a database name, ex: finch" exit 2 fi shift diff --git a/birdhouse/scripts/logging.include.sh b/birdhouse/scripts/logging.include.sh index fcce426bd..f57ed2a94 100644 --- a/birdhouse/scripts/logging.include.sh +++ b/birdhouse/scripts/logging.include.sh @@ -14,48 +14,75 @@ if [ "${BIRDHOUSE_COLOR}" -eq "1" ]; then fi BIRDHOUSE_LOG_LEVEL=${BIRDHOUSE_LOG_LEVEL:-INFO} +if [ "${__BIRDHOUSE_SUPPORTED_INTERFACE}" = 'True' ]; then + __DEFAULT_BIRDHOUSE_LOG_FD=2 +else + # logs were previously written to stdout + # (this supports backwards compatible scripts that don't use the interface) + __DEFAULT_BIRDHOUSE_LOG_FD=1 +fi export LOG_DEBUG="${GRAY}DEBUG${NORMAL}: " export LOG_INFO="${BLUE}INFO${NORMAL}: " export LOG_WARN="${YELLOW}WARNING${NORMAL}: " export LOG_ERROR="${RED}ERROR${NORMAL}: " export LOG_CRITICAL="${REG_BG_BOLD}CRITICAL${NORMAL}: " # to report misuse of functions +if [ -n "${BIRDHOUSE_LOG_FD}" ]; then + if [ -z "${BIRDHOUSE_LOG_FD##*[!0-9]*}" ]; then + echo "${LOG_CRITICAL}Invalid log file descriptor setting (not an integer): [BIRDHOUSE_LOG_FD=${BIRDHOUSE_LOG_FD}]." + exit 2 + fi +fi + + +log_dest() { + read log_line + if [ -n "${log_line}" ]; then + if [ -n "${BIRDHOUSE_LOG_FILE}" ]; then + echo "$log_line" >> "${BIRDHOUSE_LOG_FILE}" + else + echo "$log_line" 1>&"${BIRDHOUSE_LOG_FD:-${__DEFAULT_BIRDHOUSE_LOG_FD}}" + fi + fi +} # Usage: log {LEVEL} "{message}" [...] # Any amount of messages can be passed to the function. log() { - if [ "${BIRDHOUSE_LOG_LEVEL}" != DEBUG ] \ - && [ "${BIRDHOUSE_LOG_LEVEL}" != INFO ] \ - && [ "${BIRDHOUSE_LOG_LEVEL}" != WARN ] \ - && [ "${BIRDHOUSE_LOG_LEVEL}" != ERROR ]; then - echo "${LOG_CRITICAL}Invalid log level setting: [BIRDHOUSE_LOG_LEVEL=${BIRDHOUSE_LOG_LEVEL}]." - exit 2 - fi - level="$1" - shift - if [ "$*" = "" ]; then - echo "${LOG_CRITICAL}Invalid log message is missing." - exit 2 - fi - if [ "${level}" = "DEBUG" ]; then - if [ "${BIRDHOUSE_LOG_LEVEL}" = DEBUG ]; then - echo "${LOG_DEBUG}$*" + { + if [ "${BIRDHOUSE_LOG_LEVEL}" != DEBUG ] \ + && [ "${BIRDHOUSE_LOG_LEVEL}" != INFO ] \ + && [ "${BIRDHOUSE_LOG_LEVEL}" != WARN ] \ + && [ "${BIRDHOUSE_LOG_LEVEL}" != ERROR ]; then + echo "${LOG_CRITICAL}Invalid log level setting: [BIRDHOUSE_LOG_LEVEL=${BIRDHOUSE_LOG_LEVEL}]." + exit 2 fi - elif [ "${level}" = "INFO" ]; then - if [ "${BIRDHOUSE_LOG_LEVEL}" = DEBUG ] \ - || [ "${BIRDHOUSE_LOG_LEVEL}" = INFO ]; then - echo "${LOG_INFO}$*" + level="$1" + shift + if [ "$*" = "" ]; then + echo "${LOG_CRITICAL}Invalid log message is missing." + exit 2 fi - elif [ "${level}" = "WARN" ]; then - if [ "${BIRDHOUSE_LOG_LEVEL}" = DEBUG ] \ - || [ "${BIRDHOUSE_LOG_LEVEL}" = INFO ] \ - || [ "${BIRDHOUSE_LOG_LEVEL}" = WARN ]; then - echo "${LOG_WARN}$*" + if [ "${level}" = "DEBUG" ]; then + if [ "${BIRDHOUSE_LOG_LEVEL}" = DEBUG ]; then + echo "${LOG_DEBUG}$*" + fi + elif [ "${level}" = "INFO" ]; then + if [ "${BIRDHOUSE_LOG_LEVEL}" = DEBUG ] \ + || [ "${BIRDHOUSE_LOG_LEVEL}" = INFO ]; then + echo "${LOG_INFO}$*" + fi + elif [ "${level}" = "WARN" ]; then + if [ "${BIRDHOUSE_LOG_LEVEL}" = DEBUG ] \ + || [ "${BIRDHOUSE_LOG_LEVEL}" = INFO ] \ + || [ "${BIRDHOUSE_LOG_LEVEL}" = WARN ]; then + echo "${LOG_WARN}$*" + fi + elif [ "${level}" = "ERROR" ]; then + echo "${LOG_ERROR}$*" + else + echo "${LOG_CRITICAL}Invalid log level: [${level}]" + exit 2 fi - elif [ "${level}" = "ERROR" ]; then - echo "${LOG_ERROR}$*" - else - echo "${LOG_CRITICAL}Invalid log level: [${level}]" - exit 2 - fi + } | log_dest } diff --git a/birdhouse/scripts/sync-data b/birdhouse/scripts/sync-data index e4024833b..dbffa6b38 100755 --- a/birdhouse/scripts/sync-data +++ b/birdhouse/scripts/sync-data @@ -28,7 +28,7 @@ SOURCE_HOST="$1"; shift FORCE_MODE="$1" if [ -z "$SOURCE_HOST" ]; then - log ERROR "no source host provided" 1>&2 + log ERROR "no source host provided" exit 2 fi From d271818c44ff4002f4b14a3d0d6f170570ebec0a Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Tue, 26 Nov 2024 10:15:22 -0500 Subject: [PATCH 02/18] give containers access to host.docker.internal in dev mode --- birdhouse/birdhouse-compose.sh | 5 +++++ .../monitoring/docker-compose-extra.yml | 2 +- .../local-dev-test/.gitignore | 1 + .../pre-docker-compose-up.include | 19 +++++++++++++++++++ 4 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 birdhouse/optional-components/local-dev-test/.gitignore create mode 100644 birdhouse/optional-components/local-dev-test/pre-docker-compose-up.include diff --git a/birdhouse/birdhouse-compose.sh b/birdhouse/birdhouse-compose.sh index d53e73c81..1bbbebc68 100755 --- a/birdhouse/birdhouse-compose.sh +++ b/birdhouse/birdhouse-compose.sh @@ -123,6 +123,11 @@ if [ x"$1" = x"up" ]; then log INFO "Executing '$COMPONENT_PRE_COMPOSE_UP'" sh ${SHELL_EXEC_FLAGS} "$COMPONENT_PRE_COMPOSE_UP" fi + COMPONENT_PRE_COMPOSE_UP_INCLUDE="$adir/pre-docker-compose-up.include" + if [ -f "$COMPONENT_PRE_COMPOSE_UP_INCLUDE" ]; then + log INFO "Sourcing '$COMPONENT_PRE_COMPOSE_UP_INCLUDE'" + . "$COMPONENT_PRE_COMPOSE_UP_INCLUDE" + fi done fi diff --git a/birdhouse/components/monitoring/docker-compose-extra.yml b/birdhouse/components/monitoring/docker-compose-extra.yml index 29f6f46e9..24f1f1d48 100644 --- a/birdhouse/components/monitoring/docker-compose-extra.yml +++ b/birdhouse/components/monitoring/docker-compose-extra.yml @@ -21,7 +21,7 @@ services: image: ${NODE_EXPORTER_IMAGE} container_name: node-exporter volumes: - - /:/host:ro,rslave + - /:/host:ro network_mode: "host" pid: "host" command: --path.rootfs=/host diff --git a/birdhouse/optional-components/local-dev-test/.gitignore b/birdhouse/optional-components/local-dev-test/.gitignore new file mode 100644 index 000000000..9b6cadaa8 --- /dev/null +++ b/birdhouse/optional-components/local-dev-test/.gitignore @@ -0,0 +1 @@ +docker-compose-extra-ignore.yml diff --git a/birdhouse/optional-components/local-dev-test/pre-docker-compose-up.include b/birdhouse/optional-components/local-dev-test/pre-docker-compose-up.include new file mode 100644 index 000000000..3d11b1f83 --- /dev/null +++ b/birdhouse/optional-components/local-dev-test/pre-docker-compose-up.include @@ -0,0 +1,19 @@ +#!/usr/bin/env sh + +# Note: filename is not docker-compose-extra.yml so that it won't get added prematurely to COMPOSE_CONF_LIST +THIS_COMPOSE_FILE="${COMPOSE_DIR}/optional-components/local-dev-test/docker-compose-extra-ignore.yml" +echo "services:" > "${THIS_COMPOSE_FILE}" + +for service in $(PROXY_SECURE_PORT=443 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${COMPOSE_CONF_LIST} config --services); do + printf ' %s:\n extra_hosts:\n - "host.docker.internal:host-gateway"\n' $service >> "${THIS_COMPOSE_FILE}" +done + +if [ "${BIRDHOUSE_FQDN}" != "host.docker.internal" ]; then + log WARN 'BIRDHOUSE_FQDN should be set to 'host.docker.internal' when the local-dev-test optional component is enabled' +fi + +COMPOSE_CONF_LIST="${COMPOSE_CONF_LIST} -f ${THIS_COMPOSE_FILE}" + +log INFO "adding ${THIS_COMPOSE_FILE} to COMPOSE_CONF_LIST" + +unset THIS_COMPOSE_FILE From a01c3f62771a3b73e3aaf44fdf2e4d294f5ae2a0 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Tue, 26 Nov 2024 10:46:33 -0500 Subject: [PATCH 03/18] add short options for log file --- bin/birdhouse | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/birdhouse b/bin/birdhouse index cd3480116..50c58e62d 100755 --- a/bin/birdhouse +++ b/bin/birdhouse @@ -203,12 +203,12 @@ parse_args() { LOG_STDOUT=True parse_args "$@" ;; - --log-file=*) + -l=|--log-file=*) arg_value="${1#*=}" shift parse_args --log-file "${arg_value}" "$@" ;; - --log-file) + -l|--log-file) shift [ "${LOG_STDOUT}" = 'True' ] && parse_args 'invalid arg that triggers usage message' export BIRDHOUSE_LOG_FILE=$(realpath "$1") # The argument here takes precedence over the env variable From 79445d893d825951c80fea49e8c14ed5849f8b8d Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Wed, 27 Nov 2024 10:47:32 -0500 Subject: [PATCH 04/18] allow stack to work through http only --- .gitignore | 1 + birdhouse/birdhouse-compose.sh | 8 +++---- birdhouse/components/README.rst | 20 ++++++++--------- .../docker_configuration.py.template | 6 ++--- .../canarie_api_monitoring.py.template | 2 +- .../config/cowbird/config.yml.template | 2 +- .../config/cowbird/cowbird.ini.template | 2 +- .../cowbird.conf.template | 2 +- .../canarie_api_monitoring.py.template | 2 +- .../finch/service-config.json.template | 4 ++-- birdhouse/components/finch/wps.cfg.template | 2 +- .../canarie_api_monitoring.py.template | 4 ++-- .../geoserver.conf.template | 2 +- .../geoserver/docker-compose-extra.yml | 2 +- .../geoserver/service-config.json.template | 2 +- .../canarie_api_monitoring.py.template | 2 +- .../hummingbird/service-config.json.template | 4 ++-- .../canarie_api_monitoring.py.template | 4 ++-- .../jupyterhub/jupyterhub_config.py.template | 4 ++-- .../jupyterhub/service-config.json.template | 2 +- .../canarie_api_monitoring.py.template | 4 ++-- .../magpie/docker-compose-extra.yml | 2 +- .../components/magpie/magpie.ini.template | 2 +- .../monitoring.conf.template | 6 ++--- .../monitoring/docker-compose-extra.yml | 6 ++--- .../proxy/conf.d/frontend.conf.template | 22 ++----------------- .../components/proxy/conf.d/https.include | 19 ++++++++++++++++ birdhouse/components/proxy/default.env | 18 +++++++++++---- .../components/proxy/docker-compose-extra.yml | 4 +--- .../proxy/docker-compose-ssl-cert.yml | 8 +++++++ .../proxy/pre-docker-compose-up.include | 14 ++++++++++++ birdhouse/components/raven/default.env | 2 +- .../raven/service-config.json.template | 4 ++-- birdhouse/components/raven/wps.cfg.template | 2 +- .../canarie_api_monitoring.py.template | 2 +- .../conf.extra-service.d/stac.conf.template | 4 ++-- .../components/stac/docker-compose-extra.yml | 2 +- .../stac/service-config.json.template | 8 +++---- .../canarie_api_monitoring.py.template | 4 ++-- .../thredds.conf.template | 2 +- .../thredds/service-config.json.template | 4 ++-- .../canarie_api_monitoring.py.template | 4 ++-- .../components/twitcher/twitcher.ini.template | 4 ++-- .../canarie_api_monitoring.py.template | 2 +- .../conf.extra-service.d/weaver.conf.template | 4 ++-- .../config/weaver/data_sources.yml.template | 4 ++-- .../weaver/config/weaver/weaver.ini.template | 6 ++--- .../components/weaver/post-docker-compose-up | 8 +++---- .../weaver/service-config.json.template | 6 ++--- .../catalog/catalog.cfg.template | 8 +++---- .../canarie_api_monitoring.py.template | 2 +- .../flyingpigeon/service-config.json.template | 4 ++-- .../flyingpigeon/wps.cfg.template | 2 +- .../canarie_api_monitoring.py.template | 2 +- .../frontend/frontend.env.template | 6 ++--- .../canarie_api_monitoring.py.template | 2 +- .../canarie_api_monitoring.py.template | 4 ++-- .../canarie_api_monitoring.py.template | 2 +- birdhouse/env.local.example | 6 ++--- birdhouse/optional-components/README.rst | 8 +++++++ .../canarie-api/a_demo_override_precedence.py | 2 +- ...z_demo_only_py_file_are_loaded.wrongsuffix | 2 +- .../canarie_api_full_monitoring.py.template | 2 +- .../canarie_api_full_monitoring.py.template | 2 +- .../canarie_api_full_monitoring.py.template | 2 +- .../canarie_api_full_monitoring.py.template | 2 +- .../canarie_api_full_monitoring.py.template | 2 +- .../canarie_api_full_monitoring.py.template | 2 +- .../canarie_api_full_monitoring.py.template | 2 +- .../canarie_api_full_monitoring.py.template | 2 +- .../canarie_api_full_monitoring.py.template | 2 +- .../canarie_api_monitoring.py.template | 2 +- .../optional-components/emu/wps.cfg.template | 2 +- .../canarie_api_monitoring.py.template | 2 +- .../generic_bird/wps.cfg.template | 2 +- .../pre-docker-compose-up.include | 9 ++++++-- .../secure-data-auth.conf.template | 2 +- .../docker-compose-extra.yml | 2 +- .../weaver/request_options.yml.template | 2 +- .../canarie_api_monitoring.py.template | 2 +- birdhouse/scripts/check-instance-ready | 6 ++--- 81 files changed, 199 insertions(+), 154 deletions(-) create mode 100644 birdhouse/components/proxy/conf.d/https.include create mode 100644 birdhouse/components/proxy/docker-compose-ssl-cert.yml create mode 100644 birdhouse/components/proxy/pre-docker-compose-up.include diff --git a/.gitignore b/.gitignore index 128ede050..16f981110 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,4 @@ venv/ ## Testing .pytest_cache/ *.log +birdhouse/data/ \ No newline at end of file diff --git a/birdhouse/birdhouse-compose.sh b/birdhouse/birdhouse-compose.sh index 1bbbebc68..0731e97c8 100755 --- a/birdhouse/birdhouse-compose.sh +++ b/birdhouse/birdhouse-compose.sh @@ -132,8 +132,8 @@ if [ x"$1" = x"up" ]; then fi log INFO "Executing docker-compose with extra options: $* ${COMPOSE_EXTRA_OPTS}" -# the PROXY_SECURE_PORT is a little trick to make the compose file invalid without the usage of this wrapper script -PROXY_SECURE_PORT=443 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${COMPOSE_CONF_LIST} $* ${COMPOSE_EXTRA_OPTS} +# the PROXY_HTTP_PORT is a little trick to make the compose file invalid without the usage of this wrapper script +PROXY_HTTP_PORT=80 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${COMPOSE_CONF_LIST} $* ${COMPOSE_EXTRA_OPTS} ERR=$? if [ ${ERR} -gt 0 ]; then log ERROR "docker-compose error, exit code ${ERR}" @@ -153,11 +153,11 @@ while [ $# -gt 0 ] do if [ x"$1" = x"up" ]; then # we restart the proxy after an up to make sure nginx continue to work if any container IP address changes - PROXY_SECURE_PORT=443 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${COMPOSE_CONF_LIST} restart proxy + PROXY_HTTP_PORT=80 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${COMPOSE_CONF_LIST} restart proxy # run postgres post-startup setup script # Note: this must run before the post-docker-compose-up scripts since some may expect postgres databases to exist - postgres_id=$(PROXY_SECURE_PORT=443 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${COMPOSE_CONF_LIST} ps -q postgres 2> /dev/null) + postgres_id=$(PROXY_HTTP_PORT=80 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${COMPOSE_CONF_LIST} ps -q postgres 2> /dev/null) if [ ! -z "$postgres_id" ]; then docker exec ${postgres_id} /postgres-setup.sh fi diff --git a/birdhouse/components/README.rst b/birdhouse/components/README.rst index c9c1ffebe..c3733ce96 100644 --- a/birdhouse/components/README.rst +++ b/birdhouse/components/README.rst @@ -567,7 +567,7 @@ An endpoint monitoring tool that shows the current status of other components in Usage ----- -The service is available at ``https://${BIRDHOUSE_FQDN_PUBLIC}/canarie`` +The service is available at ``${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/canarie`` How to Enable the Component --------------------------- @@ -599,7 +599,7 @@ degree-days of cooling, the duration of heatwaves, etc. This returns annual valu Usage ----- -The service is available at ``https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch`` +The service is available at ``${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch`` How to Enable the Component --------------------------- @@ -618,7 +618,7 @@ Geospatial Web. Usage ----- -The service is available at ``https://${BIRDHOUSE_FQDN_PUBLIC}/geoserver``. For usage and +The service is available at ``${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/geoserver``. For usage and configuration options please refer to the `Geoserver documentation`_. .. _Geoserver documentation: https://docs.geoserver.org @@ -637,7 +637,7 @@ A Web Processing Service for compliance checks used in the climate science commu Usage ----- -The service is available at ``https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird`` +The service is available at ``${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird`` How to Enable the Component --------------------------- @@ -654,7 +654,7 @@ end-users. Usage ----- -The service is available at ``https://${BIRDHOUSE_FQDN_PUBLIC}/jupyter``. Users are able to log in to Jupyterhub using the +The service is available at ``${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/jupyter``. Users are able to log in to Jupyterhub using the same user name and password as Magpie. They will then be able to launch a personal jupyterlab server. How to Enable the Component @@ -673,7 +673,7 @@ User/Group/Service/Resource/Permission management and integrates with Twitcher. Usage ----- -The service is available at ``https://${BIRDHOUSE_FQDN_PUBLIC}/magpie``. For usage and configuration options please +The service is available at ``${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/magpie``. For usage and configuration options please refer to the `Magpie documentation`_. .. _Magpie documentation: https://pavics-magpie.readthedocs.io @@ -706,7 +706,7 @@ A web based container deployment and management tool. Usage ----- -The service is available at ``https://${BIRDHOUSE_FQDN_PUBLIC}/portainer/``. For usage and configuration options please +The service is available at ``${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/portainer/``. For usage and configuration options please refer to the `portainer documentation`_. How to Enable the Component @@ -757,7 +757,7 @@ processing as well as time series analysis. Usage ----- -The service is available at ``https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven`` +The service is available at ``${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven`` How to Enable the Component --------------------------- @@ -775,7 +775,7 @@ Climate Data Catalog and Format Renderers. See the `Thredds documentation`_ for Usage ----- -The catalog is available at the ``https://${BIRDHOUSE_FQDN_PUBLIC}/thredds`` endpoint. +The catalog is available at the ``${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/thredds`` endpoint. How to Enable the Component --------------------------- @@ -811,7 +811,7 @@ of all processes executed by these services. Usage ----- -All outputs from these processes will become available at the ``https://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs`` endpoint. +All outputs from these processes will become available at the ``${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs`` endpoint. By default, this endpoint is not protected. To secure access to this endpoint it is highly recommended to enable the `./optional-components/secure-data-proxy` component as well. diff --git a/birdhouse/components/canarie-api/docker_configuration.py.template b/birdhouse/components/canarie-api/docker_configuration.py.template index 22f6cb7b5..74b1df238 100644 --- a/birdhouse/components/canarie-api/docker_configuration.py.template +++ b/birdhouse/components/canarie-api/docker_configuration.py.template @@ -10,7 +10,7 @@ import requests_cache # see entrypoint script logger = logging.getLogger("canarie-api-config") -MY_SERVER_NAME = 'https://${BIRDHOUSE_FQDN_PUBLIC}/canarie' +MY_SERVER_NAME = '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/canarie' DATABASE = { 'filename': '/data/stats.db', @@ -125,7 +125,7 @@ SERVICES = { 'releasenotes': '${BIRDHOUSE_RELEASE_NOTES_URL}', 'support': '${BIRDHOUSE_SUPPORT_URL}', 'source': 'https://github.com/bird-house/birdhouse-deploy', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}', 'licence': '${BIRDHOUSE_LICENSE_URL}', 'provenance': 'https://pavics-sdi.readthedocs.io/en/latest/provenance/index.html' }, @@ -157,7 +157,7 @@ PLATFORMS = { 'releasenotes': 'https://github.com/bird-house/birdhouse-deploy/releases', 'support': 'https://github.com/Ouranosinc/pavics-sdi/issues', 'source': 'https://github.com/Ouranosinc/pavics-sdi', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}', 'licence': 'https://pavics-sdi.readthedocs.io/en/latest/license.html', 'provenance': 'https://pavics-sdi.readthedocs.io/en/latest/provenance/index.html', 'factsheet': 'http://www.canarie.ca/software/pavics' diff --git a/birdhouse/components/cowbird/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/cowbird/config/canarie-api/canarie_api_monitoring.py.template index 1d693d201..55aa4539f 100644 --- a/birdhouse/components/cowbird/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/cowbird/config/canarie-api/canarie_api_monitoring.py.template @@ -19,7 +19,7 @@ SERVICES['Cowbird'] = { 'releasenotes': 'https://github.com/Ouranosinc/cowbird//blob/master/CHANGES.rst', 'support': 'https://github.com/Ouranosinc/cowbird//issues', 'source': 'https://github.com/Ouranosinc/cowbird/', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}/cowbird/', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/cowbird/', 'licence': 'https://github.com/Ouranosinc/cowbird//blob/${COWBIRD_VERSION}/LICENSE', 'provenance': 'https://github.com/Ouranosinc/cowbird/' }, diff --git a/birdhouse/components/cowbird/config/cowbird/config.yml.template b/birdhouse/components/cowbird/config/cowbird/config.yml.template index 096dcbc61..bc1bf583c 100644 --- a/birdhouse/components/cowbird/config/cowbird/config.yml.template +++ b/birdhouse/components/cowbird/config/cowbird/config.yml.template @@ -21,7 +21,7 @@ handlers: admin_password: ${GEOSERVER_ADMIN_PASSWORD} Catalog: active: true - url: https://${BIRDHOUSE_FQDN_PUBLIC}/twitcher/ows/proxy/catalog + url: ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/twitcher/ows/proxy/catalog workspace_dir: ${WORKSPACE_DIR} Thredds: active: true diff --git a/birdhouse/components/cowbird/config/cowbird/cowbird.ini.template b/birdhouse/components/cowbird/config/cowbird/cowbird.ini.template index a1b3daca9..1336f4916 100644 --- a/birdhouse/components/cowbird/config/cowbird/cowbird.ini.template +++ b/birdhouse/components/cowbird/config/cowbird/cowbird.ini.template @@ -32,7 +32,7 @@ mongo_uri = mongodb://${COWBIRD_MONGODB_HOST}:${COWBIRD_MONGODB_PORT}/cowbird # below values are for the external definitions after proxy resolution # internal app access is defined in [server:main] section cowbird.port = -cowbird.url = https://${BIRDHOUSE_FQDN_PUBLIC}/cowbird +cowbird.url = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/cowbird [app:api_app] use = egg:Paste#static diff --git a/birdhouse/components/cowbird/config/proxy/conf.extra-service.d/cowbird.conf.template b/birdhouse/components/cowbird/config/proxy/conf.extra-service.d/cowbird.conf.template index 00d297a06..2c2439001 100644 --- a/birdhouse/components/cowbird/config/proxy/conf.extra-service.d/cowbird.conf.template +++ b/birdhouse/components/cowbird/config/proxy/conf.extra-service.d/cowbird.conf.template @@ -1,6 +1,6 @@ location /cowbird { - proxy_pass https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/cowbird; + proxy_pass ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/cowbird; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Host $host:$server_port; diff --git a/birdhouse/components/finch/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/finch/config/canarie-api/canarie_api_monitoring.py.template index 58b113166..576f4b78f 100644 --- a/birdhouse/components/finch/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/finch/config/canarie-api/canarie_api_monitoring.py.template @@ -34,7 +34,7 @@ SERVICES['indices'] = { 'monitoring': { 'Finch': { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&version=1.0.0&request=GetCapabilities' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&version=1.0.0&request=GetCapabilities' } }, } diff --git a/birdhouse/components/finch/service-config.json.template b/birdhouse/components/finch/service-config.json.template index 37642e732..537afa70d 100644 --- a/birdhouse/components/finch/service-config.json.template +++ b/birdhouse/components/finch/service-config.json.template @@ -14,7 +14,7 @@ { "rel": "service", "type": "text/xml", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&request=GetCapabilities" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&request=GetCapabilities" }, { "rel": "service-doc", @@ -24,7 +24,7 @@ { "rel": "service-desc", "type": "text/xml", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&request=GetCapabilities" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&request=GetCapabilities" }, { "rel": "service-meta", diff --git a/birdhouse/components/finch/wps.cfg.template b/birdhouse/components/finch/wps.cfg.template index 19aee28f1..3da46e9bd 100644 --- a/birdhouse/components/finch/wps.cfg.template +++ b/birdhouse/components/finch/wps.cfg.template @@ -1,5 +1,5 @@ [server] -outputurl = https://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs/finch +outputurl = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs/finch outputpath = /data/wpsoutputs/finch # default 3mb, fix "Broken pipe" between the proxy and the wps service diff --git a/birdhouse/components/geoserver/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/geoserver/config/canarie-api/canarie_api_monitoring.py.template index 876128992..1dcd39ad8 100644 --- a/birdhouse/components/geoserver/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/geoserver/config/canarie-api/canarie_api_monitoring.py.template @@ -24,14 +24,14 @@ SERVICES['GeoServer'] = { 'releasenotes': 'https://geoserver.org/release/${GEOSERVER_VERSION}/', 'support': 'https://github.com/kartoza/docker-geoserver/issues', 'source': 'https://github.com/kartoza/docker-geoserver', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}/geoserver/', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/geoserver/', 'licence': 'https://github.com/geoserver/geoserver/blob/${GEOSERVER_VERSION}/LICENSE.txt', 'provenance': 'https://github.com/kartoza/docker-geoserver' }, "monitoring": { "GeoServer": { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}/geoserver/web/' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/geoserver/web/' } } } diff --git a/birdhouse/components/geoserver/config/proxy/conf.extra-service.d/geoserver.conf.template b/birdhouse/components/geoserver/config/proxy/conf.extra-service.d/geoserver.conf.template index 8b3e0de6b..d2b4fd955 100644 --- a/birdhouse/components/geoserver/config/proxy/conf.extra-service.d/geoserver.conf.template +++ b/birdhouse/components/geoserver/config/proxy/conf.extra-service.d/geoserver.conf.template @@ -19,7 +19,7 @@ # If GEOSERVER_SKIP_AUTH is "True" then the following section is skipped and this # location block will always return 200 (which means that the /geoserver/ location, above, # will be publicly available. - proxy_pass https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/geoserver$request_uri; + proxy_pass ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/geoserver$request_uri; proxy_pass_request_body off; proxy_set_header Host $host; proxy_set_header Content-Length ""; diff --git a/birdhouse/components/geoserver/docker-compose-extra.yml b/birdhouse/components/geoserver/docker-compose-extra.yml index b51c7e395..c78a56a8d 100644 --- a/birdhouse/components/geoserver/docker-compose-extra.yml +++ b/birdhouse/components/geoserver/docker-compose-extra.yml @@ -21,7 +21,7 @@ services: MAXIMUM_MEMORY: 8G # https://github.com/kartoza/docker-geoserver#proxy-base-url HTTP_PROXY_NAME: ${BIRDHOUSE_FQDN_PUBLIC} - HTTP_SCHEME: https + HTTP_SCHEME: ${BIRDHOUSE_PROXY_SCHEME} volumes: # run deployment/fix-geoserver-data-dir-perm on existing # GEOSERVER_DATA_DIR to match user geoserveruser inside docker image diff --git a/birdhouse/components/geoserver/service-config.json.template b/birdhouse/components/geoserver/service-config.json.template index acf7c38aa..bacfb541b 100644 --- a/birdhouse/components/geoserver/service-config.json.template +++ b/birdhouse/components/geoserver/service-config.json.template @@ -20,7 +20,7 @@ { "rel": "service", "type": "text/html", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}/geoserver/" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/geoserver/" }, { "rel": "service-doc", diff --git a/birdhouse/components/hummingbird/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/hummingbird/config/canarie-api/canarie_api_monitoring.py.template index cfc5ffc83..7d8867b9f 100644 --- a/birdhouse/components/hummingbird/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/hummingbird/config/canarie-api/canarie_api_monitoring.py.template @@ -25,7 +25,7 @@ SERVICES['hummingbird'] = { 'releasenotes': 'https://github.com/bird-house/hummingbird/blob/master/CHANGES.rst', 'support': 'https://github.com/bird-house/hummingbird/issues', 'source': 'https://github.com/bird-house/hummingbird', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird/wps?service=WPS&version=1.0.0&request=GetCapabilities', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird/wps?service=WPS&version=1.0.0&request=GetCapabilities', 'licence': 'https://github.com/bird-house/hummingbird/blob/master/LICENSE.txt', 'provenance': 'https://github.com/bird-house/hummingbird' }, diff --git a/birdhouse/components/hummingbird/service-config.json.template b/birdhouse/components/hummingbird/service-config.json.template index 7e36d64c2..4cc5e453a 100644 --- a/birdhouse/components/hummingbird/service-config.json.template +++ b/birdhouse/components/hummingbird/service-config.json.template @@ -14,7 +14,7 @@ { "rel": "service", "type": "text/xml", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird?service=WPS&request=GetCapabilities" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird?service=WPS&request=GetCapabilities" }, { "rel": "service-doc", @@ -24,7 +24,7 @@ { "rel": "service-desc", "type": "text/xml", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird?service=WPS&request=GetCapabilities" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird?service=WPS&request=GetCapabilities" }, { "rel": "service-meta", diff --git a/birdhouse/components/jupyterhub/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/jupyterhub/config/canarie-api/canarie_api_monitoring.py.template index 1681c8804..ba9abdc56 100644 --- a/birdhouse/components/jupyterhub/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/jupyterhub/config/canarie-api/canarie_api_monitoring.py.template @@ -19,14 +19,14 @@ SERVICES['Jupyter'] = { 'releasenotes': 'https://github.com/Ouranosinc/jupyterhub/tags', # no CHANGES file available 'support': 'https://github.com/Ouranosinc/jupyterhub/issues', 'source': 'https://github.com/Ouranosinc/jupyterhub', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}/jupyter/', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/jupyter/', 'licence': 'https://github.com/Ouranosinc/jupyterhub/blob/${JUPYTERHUB_VERSION}/LICENSE', 'provenance': '' }, "monitoring": { "Jupyter": { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}/jupyter/hub/login' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/jupyter/hub/login' }, } } diff --git a/birdhouse/components/jupyterhub/jupyterhub_config.py.template b/birdhouse/components/jupyterhub/jupyterhub_config.py.template index 0fce675b0..1b2434f82 100644 --- a/birdhouse/components/jupyterhub/jupyterhub_config.py.template +++ b/birdhouse/components/jupyterhub/jupyterhub_config.py.template @@ -107,10 +107,10 @@ c.DockerSpawner.environment = { # Post on Panel forum: # https://discourse.holoviz.org/t/how-to-customize-the-display-url-from-panel-serve-for-use-behind-jupyterhub-with-jupyter-server-proxy/3571 # Issue about Panel Preview: https://github.com/holoviz/panel/issues/3440 - "BIRDHOUSE_HOST_URL": "https://${BIRDHOUSE_FQDN_PUBLIC}", + "BIRDHOUSE_HOST_URL": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}", # https://docs.dask.org/en/stable/configuration.html # https://jupyterhub-on-hadoop.readthedocs.io/en/latest/dask.html - "DASK_DISTRIBUTED__DASHBOARD__LINK": "https://${BIRDHOUSE_FQDN_PUBLIC}{JUPYTERHUB_SERVICE_PREFIX}proxy/{port}/status" + "DASK_DISTRIBUTED__DASHBOARD__LINK": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}{JUPYTERHUB_SERVICE_PREFIX}proxy/{port}/status" } host_user_data_dir = join(os.environ['WORKSPACE_DIR'], "{username}") diff --git a/birdhouse/components/jupyterhub/service-config.json.template b/birdhouse/components/jupyterhub/service-config.json.template index 59850d054..0d6fcfff2 100644 --- a/birdhouse/components/jupyterhub/service-config.json.template +++ b/birdhouse/components/jupyterhub/service-config.json.template @@ -14,7 +14,7 @@ { "rel": "service", "type": "text/html", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}/jupyter" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/jupyter" }, { "rel": "service-doc", diff --git a/birdhouse/components/magpie/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/magpie/config/canarie-api/canarie_api_monitoring.py.template index 0f9e4b23f..b4eee3c44 100644 --- a/birdhouse/components/magpie/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/magpie/config/canarie-api/canarie_api_monitoring.py.template @@ -23,14 +23,14 @@ SERVICES['Magpie'] = { 'releasenotes': 'https://github.com/Ouranosinc/Magpie/blob/master/CHANGES.rst', 'support': 'https://github.com/Ouranosinc/Magpie/issues', 'source': 'https://github.com/Ouranosinc/Magpie', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}/magpie/', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/magpie/', 'licence': 'https://github.com/Ouranosinc/Magpie/blob/${MAGPIE_VERSION}/LICENSE', 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' }, "monitoring": { "Magpie": { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}/magpie/version' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/magpie/version' }, 'response': { 'text': r'\{.*"code": 200.*"type": "application/json".*\}' diff --git a/birdhouse/components/magpie/docker-compose-extra.yml b/birdhouse/components/magpie/docker-compose-extra.yml index 4c464fd3c..737cd080a 100644 --- a/birdhouse/components/magpie/docker-compose-extra.yml +++ b/birdhouse/components/magpie/docker-compose-extra.yml @@ -12,7 +12,7 @@ services: image: pavics/magpie:${MAGPIE_VERSION} container_name: magpie environment: - TWITCHER_PROTECTED_URL: https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH} + TWITCHER_PROTECTED_URL: ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH} # target directories to allow loading multiple config files of corresponding category # each compose override should volume mount its files inside the matching directories # (note: DO NOT use 'MAGPIE_CONFIG_PATH' that would disable multi-config loading capability) diff --git a/birdhouse/components/magpie/magpie.ini.template b/birdhouse/components/magpie/magpie.ini.template index f1c0ed93d..0af08d080 100644 --- a/birdhouse/components/magpie/magpie.ini.template +++ b/birdhouse/components/magpie/magpie.ini.template @@ -28,7 +28,7 @@ pyramid.includes = pyramid_tm ziggurat_foundations.ext.pyramid.sign_in ziggurat_ # other overridable variables available in magpie/constants.py # magpie.port = 2001 -magpie.url = https://${BIRDHOUSE_FQDN}/magpie +magpie.url = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}/magpie magpie.max_restart = 5 magpie.push_phoenix = true # This secret should be the same in Twitcher ! diff --git a/birdhouse/components/monitoring/config/proxy/conf.extra-service.d/monitoring.conf.template b/birdhouse/components/monitoring/config/proxy/conf.extra-service.d/monitoring.conf.template index 07a62c224..95ac63d8b 100644 --- a/birdhouse/components/monitoring/config/proxy/conf.extra-service.d/monitoring.conf.template +++ b/birdhouse/components/monitoring/config/proxy/conf.extra-service.d/monitoring.conf.template @@ -22,7 +22,7 @@ location = /secure-grafana-auth { internal; - proxy_pass https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/grafana$request_uri; + proxy_pass ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/grafana$request_uri; proxy_pass_request_body off; proxy_set_header Host $host; proxy_set_header Content-Length ""; @@ -34,7 +34,7 @@ location = /secure-prometheus-auth { internal; - proxy_pass https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/prometheus$request_uri; + proxy_pass ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/prometheus$request_uri; proxy_pass_request_body off; proxy_set_header Host $host; proxy_set_header Content-Length ""; @@ -46,7 +46,7 @@ location = /secure-alertmanager-auth { internal; - proxy_pass https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/alertmanager$request_uri; + proxy_pass ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/alertmanager$request_uri; proxy_pass_request_body off; proxy_set_header Host $host; proxy_set_header Content-Length ""; diff --git a/birdhouse/components/monitoring/docker-compose-extra.yml b/birdhouse/components/monitoring/docker-compose-extra.yml index 24f1f1d48..4a99f7555 100644 --- a/birdhouse/components/monitoring/docker-compose-extra.yml +++ b/birdhouse/components/monitoring/docker-compose-extra.yml @@ -45,7 +45,7 @@ services: # https://prometheus.io/docs/prometheus/latest/storage/ - --storage.tsdb.retention.time=90d # wrong default was http://container-hash:9090/ - - --web.external-url=https://${BIRDHOUSE_FQDN_PUBLIC}/prometheus/ + - --web.external-url=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/prometheus/ restart: always # https://grafana.com/docs/grafana/latest/installation/docker/ @@ -61,7 +61,7 @@ services: - grafana_persistence:/var/lib/grafana:rw environment: GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD} - GF_SERVER_ROOT_URL: https://${BIRDHOUSE_FQDN_PUBLIC}/grafana + GF_SERVER_ROOT_URL: ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/grafana GF_SERVER_SERVE_FROM_SUB_PATH: 'true' GF_SERVER_DOMAIN: ${BIRDHOUSE_FQDN_PUBLIC} restart: always @@ -83,7 +83,7 @@ services: # enable debug logging - --log.level=debug # wrong default was http://container-hash:9093/ - - --web.external-url=https://${BIRDHOUSE_FQDN_PUBLIC}/alertmanager + - --web.external-url=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/alertmanager restart: always volumes: diff --git a/birdhouse/components/proxy/conf.d/frontend.conf.template b/birdhouse/components/proxy/conf.d/frontend.conf.template index 391f7902c..4eb872aae 100644 --- a/birdhouse/components/proxy/conf.d/frontend.conf.template +++ b/birdhouse/components/proxy/conf.d/frontend.conf.template @@ -18,7 +18,7 @@ server { listen 80; server_name localhost; - ${INCLUDE_FOR_PORT_80} + ${PROXY_INCLUDE_FOR_PORT_80} # redirect server error pages to the static page /50x.html error_page 500 502 503 504 /50x.html; @@ -27,22 +27,4 @@ server { } } -server { - listen 443; - server_name localhost; - proxy_buffering off; - - resolver 127.0.0.11; - - ssl on; - ssl_certificate /etc/nginx/cert.pem; - ssl_certificate_key /etc/nginx/cert.pem; - - include /etc/nginx/conf.d/all-services.include; - - # redirect server error pages to the static page /50x.html - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } -} +${PROXY_INCLUDE_HTTPS} diff --git a/birdhouse/components/proxy/conf.d/https.include b/birdhouse/components/proxy/conf.d/https.include new file mode 100644 index 000000000..992fd2458 --- /dev/null +++ b/birdhouse/components/proxy/conf.d/https.include @@ -0,0 +1,19 @@ +server { + listen 443; + server_name localhost; + proxy_buffering off; + + resolver 127.0.0.11; + + ssl on; + ssl_certificate /etc/nginx/cert.pem; + ssl_certificate_key /etc/nginx/cert.pem; + + include /etc/nginx/conf.d/all-services.include; + + # redirect server error pages to the static page /50x.html + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} \ No newline at end of file diff --git a/birdhouse/components/proxy/default.env b/birdhouse/components/proxy/default.env index 08f9dde89..1b28892ec 100644 --- a/birdhouse/components/proxy/default.env +++ b/birdhouse/components/proxy/default.env @@ -6,14 +6,18 @@ export PROXY_IMAGE="nginx:1.23.4" # Any WPS processes taking longer than this should use async mode. export PROXY_READ_TIMEOUT_VALUE="240s" +export BIRDHOUSE_PROXY_SCHEME='$([ x"$BIRDHOUSE_HTTP_ONLY" = x"True" ] && echo http || echo https )' +export BIRDHOUSE_ALLOW_UNSECURE_HTTP='$([ x"$BIRDHOUSE_HTTP_ONLY" = x"True" ] && echo True || echo )' +export PROXY_INCLUDE_HTTPS='$([ x"$BIRDHOUSE_HTTP_ONLY" = x"True" ] || echo "include /etc/nginx/conf.d/https.include;" )' + # Content of "location /" in file config/proxy/conf.d/all-services.include.template # Useful to have a custom homepage. # Note that the default homepage will become the jupyterhub login page if the jupyterhub component is enabled. # If the jupyterhub component is not enabled, it is highly recommended to create a custom homepage since the magpie # landing page is not the most user-friendly option. -export BIRDHOUSE_PROXY_ROOT_LOCATION="return 302 https://\$host/jupyter/hub/login;" +export BIRDHOUSE_PROXY_ROOT_LOCATION='return 302 ${BIRDHOUSE_PROXY_SCHEME}://\$host/jupyter/hub/login;' -export INCLUDE_FOR_PORT_80='$([ x"$BIRDHOUSE_ALLOW_UNSECURE_HTTP" = x"True" ] && echo "include /etc/nginx/conf.d/all-services.include;" || echo "include /etc/nginx/conf.d/redirect-to-https.include;")' +export PROXY_INCLUDE_FOR_PORT_80='$([ x"$BIRDHOUSE_ALLOW_UNSECURE_HTTP" = x"True" ] && echo "include /etc/nginx/conf.d/all-services.include;" || echo "include /etc/nginx/conf.d/redirect-to-https.include;")' export PROXY_LOG_DIR="/var/log/nginx/" export PROXY_LOG_FILE="access_file.log" @@ -21,8 +25,12 @@ export PROXY_LOG_PATH='${PROXY_LOG_DIR}/${PROXY_LOG_FILE}' export DELAYED_EVAL=" $DELAYED_EVAL - INCLUDE_FOR_PORT_80 PROXY_LOG_PATH + BIRDHOUSE_PROXY_SCHEME + BIRDHOUSE_ALLOW_UNSECURE_HTTP + PROXY_INCLUDE_HTTPS + PROXY_INCLUDE_FOR_PORT_80 + BIRDHOUSE_PROXY_ROOT_LOCATION " # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here @@ -31,13 +39,15 @@ export VARS=" \$BIRDHOUSE_DEPLOY_COMPONENTS_JSON \$BIRDHOUSE_DEPLOY_SERVICES_JSON \$BIRDHOUSE_VERSION_JSON + \$BIRDHOUSE_PROXY_SCHEME " export OPTIONAL_VARS=" $OPTIONAL_VARS - \$INCLUDE_FOR_PORT_80 + \$PROXY_INCLUDE_FOR_PORT_80 \$PROXY_READ_TIMEOUT_VALUE \$BIRDHOUSE_PROXY_ROOT_LOCATION \$PROXY_LOG_FILE \$PROXY_LOG_PATH + \$PROXY_INCLUDE_HTTPS " diff --git a/birdhouse/components/proxy/docker-compose-extra.yml b/birdhouse/components/proxy/docker-compose-extra.yml index 66ce09ef4..86852670b 100644 --- a/birdhouse/components/proxy/docker-compose-extra.yml +++ b/birdhouse/components/proxy/docker-compose-extra.yml @@ -12,12 +12,10 @@ services: image: ${PROXY_IMAGE} container_name: proxy ports: - - "80:80" - - "443:${PROXY_SECURE_PORT}" + - "80:${PROXY_HTTP_PORT}" volumes: - ./components/proxy/conf.d:/etc/nginx/conf.d - ./components/proxy/nginx.conf:/etc/nginx/nginx.conf - - ${BIRDHOUSE_SSL_CERTIFICATE}:/etc/nginx/cert.pem - ./components/proxy/static:/static environment: # https://github.com/bird-house/birdhouse-deploy/issues/198 diff --git a/birdhouse/components/proxy/docker-compose-ssl-cert.yml b/birdhouse/components/proxy/docker-compose-ssl-cert.yml new file mode 100644 index 000000000..abd2a4a95 --- /dev/null +++ b/birdhouse/components/proxy/docker-compose-ssl-cert.yml @@ -0,0 +1,8 @@ +version: "3.4" + +services: + proxy: + ports: + - "443:443" + volumes: + - ${BIRDHOUSE_SSL_CERTIFICATE}:/etc/nginx/cert.pem diff --git a/birdhouse/components/proxy/pre-docker-compose-up.include b/birdhouse/components/proxy/pre-docker-compose-up.include new file mode 100644 index 000000000..b9bac8221 --- /dev/null +++ b/birdhouse/components/proxy/pre-docker-compose-up.include @@ -0,0 +1,14 @@ +#!/usr/bin/env sh + +# Note: filename is not docker-compose-extra.yml so that it won't get added prematurely to COMPOSE_CONF_LIST + +if [ x"$BIRDHOUSE_HTTP_ONLY" != x"True" ]; then + THIS_COMPOSE_FILE="${COMPOSE_DIR}/optional-components/local-dev-test/docker-compose-extra-ssl-cert.yml" + echo "services:" > "${THIS_COMPOSE_FILE}" + + COMPOSE_CONF_LIST="${COMPOSE_CONF_LIST} -f ${THIS_COMPOSE_FILE}" + + log INFO "adding ${THIS_COMPOSE_FILE} to COMPOSE_CONF_LIST" + + unset THIS_COMPOSE_FILE +fi diff --git a/birdhouse/components/raven/default.env b/birdhouse/components/raven/default.env index 24f115a56..43c4047f0 100644 --- a/birdhouse/components/raven/default.env +++ b/birdhouse/components/raven/default.env @@ -4,7 +4,7 @@ # This is the production Geoserver that is always available with appropriate data. # For site that want to run your own Geoserver with your own data, please # override this variable with your own Geoserver instance. -# Ex: RAVEN_GEO_URL="https://${BIRDHOUSE_FQDN}/geoserver/" +# Ex: RAVEN_GEO_URL="${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}/geoserver/" __DEFAULT__RAVEN_GEO_URL="https://pavics.ouranos.ca/geoserver/" export RAVEN_GEO_URL='${__DEFAULT__RAVEN_GEO_URL}' diff --git a/birdhouse/components/raven/service-config.json.template b/birdhouse/components/raven/service-config.json.template index 96517aea4..40e00b616 100644 --- a/birdhouse/components/raven/service-config.json.template +++ b/birdhouse/components/raven/service-config.json.template @@ -14,7 +14,7 @@ { "rel": "service", "type": "text/xml", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven?service=WPS&request=GetCapabilities" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven?service=WPS&request=GetCapabilities" }, { "rel": "service-doc", @@ -24,7 +24,7 @@ { "rel": "service-desc", "type": "text/xml", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven?service=WPS&request=GetCapabilities" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven?service=WPS&request=GetCapabilities" }, { "rel": "service-meta", diff --git a/birdhouse/components/raven/wps.cfg.template b/birdhouse/components/raven/wps.cfg.template index 822129ee5..4596a5c09 100644 --- a/birdhouse/components/raven/wps.cfg.template +++ b/birdhouse/components/raven/wps.cfg.template @@ -1,5 +1,5 @@ [server] -outputurl = https://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs/raven +outputurl = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs/raven outputpath = /data/wpsoutputs/raven # default 3mb, fix "Broken pipe" between the proxy and the wps service diff --git a/birdhouse/components/stac/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/stac/config/canarie-api/canarie_api_monitoring.py.template index 360fd0387..d3e280cae 100644 --- a/birdhouse/components/stac/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/stac/config/canarie-api/canarie_api_monitoring.py.template @@ -19,7 +19,7 @@ SERVICES['STAC'] = { 'releasenotes': 'https://github.com/crim-ca/sac-app/blob/master/CHANGES.rst', 'support': 'https://github.com/crim-ca/stac-app/issues', 'source': 'https://github.com/crim-ca/stac-app', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}/stac/', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/stac/', 'licence': 'https://github.com/crim-ca/stac-app/blob/master/LICENSE', 'provenance': 'https://github.com/crim-ca/stac-app' }, diff --git a/birdhouse/components/stac/config/proxy/conf.extra-service.d/stac.conf.template b/birdhouse/components/stac/config/proxy/conf.extra-service.d/stac.conf.template index cdaee3575..8db59fcc5 100644 --- a/birdhouse/components/stac/config/proxy/conf.extra-service.d/stac.conf.template +++ b/birdhouse/components/stac/config/proxy/conf.extra-service.d/stac.conf.template @@ -4,7 +4,7 @@ # We need the second `/stac` for API redirect in STAC (see `root-path` and `ROUTER_PREFIX`). # See https://github.com/stac-utils/stac-fastapi/issues/427 # See https://github.com/crim-ca/stac-app/blob/main/stac_app.py#L60 - proxy_pass https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/stac/stac; + proxy_pass ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/stac/stac/; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Host $host:$server_port; @@ -14,7 +14,7 @@ # Automatically redirect to /stac/stac and exclude redirect when already using /stac location ~ ^${TWITCHER_PROTECTED_PATH}/stac(?!/stac) { - return 302 ${TWITCHER_PROTECTED_PATH}/stac/stac; + return 302 ${TWITCHER_PROTECTED_PATH}/stac/stac/; } location /stac-browser/ { diff --git a/birdhouse/components/stac/docker-compose-extra.yml b/birdhouse/components/stac/docker-compose-extra.yml index 2362aa967..7e62a290c 100644 --- a/birdhouse/components/stac/docker-compose-extra.yml +++ b/birdhouse/components/stac/docker-compose-extra.yml @@ -30,7 +30,7 @@ services: container_name: stac-browser image: ${STAC_BROWSER_IMAGE} environment: - - CATALOG_URL=https://${BIRDHOUSE_FQDN_PUBLIC}/stac/ + - CATALOG_URL=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/stac/ - ROOT_PATH=/stac-browser/ stac-db: diff --git a/birdhouse/components/stac/service-config.json.template b/birdhouse/components/stac/service-config.json.template index 5334fbc36..7508e3b22 100644 --- a/birdhouse/components/stac/service-config.json.template +++ b/birdhouse/components/stac/service-config.json.template @@ -15,7 +15,7 @@ { "rel": "service", "type": "application/json", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}/stac/" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/stac/" }, { "rel": "service-doc", @@ -30,7 +30,7 @@ { "rel": "alternate", "type": "text/html", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}/stac-browser/" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/stac-browser/" }, { "rel": "service-meta", @@ -55,7 +55,7 @@ { "rel": "service", "type": "text/html", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}/stac-browser/" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/stac-browser/" }, { "rel": "service-doc", @@ -65,7 +65,7 @@ { "rel": "alternate", "type": "application/json", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}/stac/" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/stac/" }, { "rel": "service-meta", diff --git a/birdhouse/components/thredds/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/thredds/config/canarie-api/canarie_api_monitoring.py.template index f0357e880..98c37357e 100644 --- a/birdhouse/components/thredds/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/thredds/config/canarie-api/canarie_api_monitoring.py.template @@ -29,7 +29,7 @@ SERVICES['renderer'] = { 'monitoring': { 'ncWMS': { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds/wms/${THREDDS_SERVICE_DATA_URL_PATH}/testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc?service=WMS&version=1.3.0&request=GetCapabilities' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds/wms/${THREDDS_SERVICE_DATA_URL_PATH}/testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc?service=WMS&version=1.3.0&request=GetCapabilities' } }, } @@ -56,7 +56,7 @@ SERVICES['Thredds'] = { 'releasenotes': 'https://docs.unidata.ucar.edu/tds/current/userguide/upgrade.html', 'support': 'https://www.unidata.ucar.edu/software/tds/#help', 'source': 'https://github.com/Unidata/tds', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds/', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds/', 'licence': 'https://github.com/Unidata/tds/blob/main/LICENSE', 'provenance': 'https://downloads.unidata.ucar.edu/tds/' }, diff --git a/birdhouse/components/thredds/config/proxy/conf.extra-service.d/thredds.conf.template b/birdhouse/components/thredds/config/proxy/conf.extra-service.d/thredds.conf.template index 671da15ab..b78c0276f 100644 --- a/birdhouse/components/thredds/config/proxy/conf.extra-service.d/thredds.conf.template +++ b/birdhouse/components/thredds/config/proxy/conf.extra-service.d/thredds.conf.template @@ -1,7 +1,7 @@ location /thredds/ { #return 302 /twitcher/ows/proxy$request_uri; - proxy_pass https://${BIRDHOUSE_FQDN}${TWITCHER_PROTECTED_PATH}/thredds/; + proxy_pass ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}${TWITCHER_PROTECTED_PATH}/thredds/; # direct hit Thredds, bypassing twitcher, for debugging only # proxy_pass http://thredds:8080${TWITCHER_PROTECTED_PATH}/thredds/; proxy_set_header Host $host; diff --git a/birdhouse/components/thredds/service-config.json.template b/birdhouse/components/thredds/service-config.json.template index 9fe03f9f6..004fe099a 100644 --- a/birdhouse/components/thredds/service-config.json.template +++ b/birdhouse/components/thredds/service-config.json.template @@ -16,7 +16,7 @@ { "rel": "service", "type": "text/html", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}/thredds/" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/thredds/" }, { "rel": "service-doc", @@ -26,7 +26,7 @@ { "rel": "service-desc", "type": "text/xml", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}/thredds/catalog.xml" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/thredds/catalog.xml" }, { "rel": "service-meta", diff --git a/birdhouse/components/twitcher/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/twitcher/config/canarie-api/canarie_api_monitoring.py.template index b5576e122..f7addaa7a 100644 --- a/birdhouse/components/twitcher/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/twitcher/config/canarie-api/canarie_api_monitoring.py.template @@ -42,14 +42,14 @@ SERVICES['Twitcher'] = { 'releasenotes': 'https://github.com/bird-house/twitcher/blob/master/CHANGES.rst', 'support': 'https://github.com/bird-house/twitcher/issues', 'source': 'https://github.com/bird-house/twitcher', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}/twitcher/', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/twitcher/', 'licence': 'https://github.com/bird-house/twitcher/blob/master/LICENSE.txt', 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' }, "monitoring": { "Twitcher": { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}/twitcher/' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/twitcher/' } } } diff --git a/birdhouse/components/twitcher/twitcher.ini.template b/birdhouse/components/twitcher/twitcher.ini.template index e00479012..02da6b4cb 100644 --- a/birdhouse/components/twitcher/twitcher.ini.template +++ b/birdhouse/components/twitcher/twitcher.ini.template @@ -51,7 +51,7 @@ cache.service.expire = 60 # debugtoolbar.hosts = 127.0.0.1 ::1 # twitcher -twitcher.url = https://${BIRDHOUSE_FQDN}/twitcher +twitcher.url = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}/twitcher twitcher.adapter = magpie.adapter.MagpieAdapter twitcher.rpcinterface = false twitcher.username = @@ -65,7 +65,7 @@ twitcher.workdir = twitcher.prefix = # magpie (for twitcher.adapter) -magpie.url = https://${BIRDHOUSE_FQDN}/magpie +magpie.url = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}/magpie magpie.secret = ${MAGPIE_SECRET} magpie.admin_user = ${MAGPIE_ADMIN_USERNAME} magpie.admin_password = ${MAGPIE_ADMIN_PASSWORD} diff --git a/birdhouse/components/weaver/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/weaver/config/canarie-api/canarie_api_monitoring.py.template index 21f8a8a35..94ee359e3 100644 --- a/birdhouse/components/weaver/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/weaver/config/canarie-api/canarie_api_monitoring.py.template @@ -19,7 +19,7 @@ SERVICES['Weaver'] = { 'releasenotes': 'https://github.com/crim-ca/weaver/blob/master/CHANGES.rst', 'support': 'https://github.com/crim-ca/weaver/issues', 'source': 'https://github.com/crim-ca/weaver', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}/weaver/', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/weaver/', 'licence': 'https://github.com/crim-ca/weaver/blob/${WEAVER_VERSION}/LICENSE.txt', 'provenance': 'https://github.com/crim-ca/weaver' }, diff --git a/birdhouse/components/weaver/config/proxy/conf.extra-service.d/weaver.conf.template b/birdhouse/components/weaver/config/proxy/conf.extra-service.d/weaver.conf.template index 6a0d99820..9a3d5b91e 100644 --- a/birdhouse/components/weaver/config/proxy/conf.extra-service.d/weaver.conf.template +++ b/birdhouse/components/weaver/config/proxy/conf.extra-service.d/weaver.conf.template @@ -6,11 +6,11 @@ # whether the *shortcut* Weaver endpoint, the alias or the explicit 'twitcher' proxy route is used. # redirect EMS/ADES to actual secured Weaver path #location /${WEAVER_CONFIG} { - # return 302 https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${WEAVER_MANAGER_NAME}; + # return 302 ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${WEAVER_MANAGER_NAME}; #} location /${WEAVER_MANAGER_NAME} { - proxy_pass https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${WEAVER_MANAGER_NAME}; + proxy_pass ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${WEAVER_MANAGER_NAME}; proxy_set_header Host $host; proxy_buffering off; include /etc/nginx/conf.d/cors.include; diff --git a/birdhouse/components/weaver/config/weaver/data_sources.yml.template b/birdhouse/components/weaver/config/weaver/data_sources.yml.template index 7ad9dbb13..6e8e8c520 100644 --- a/birdhouse/components/weaver/config/weaver/data_sources.yml.template +++ b/birdhouse/components/weaver/config/weaver/data_sources.yml.template @@ -2,13 +2,13 @@ # Employed by default for looking at files available directly (e.g.: pre-fetched) localhost: netloc: "localhost" - ades: "https://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}" + ades: "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}" default: true # Weaver self-reference, but using the exposed endpoint on the birdhouse instance public: netloc: "${BIRDHOUSE_FQDN_PUBLIC}" - ades: "https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${WEAVER_MANAGER_NAME}" + ades: "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${WEAVER_MANAGER_NAME}" # Weaver self-reference with specifically 'opensearch' parameters as process input. # Since none are available in the stack, there is no collection ID. diff --git a/birdhouse/components/weaver/config/weaver/weaver.ini.template b/birdhouse/components/weaver/config/weaver/weaver.ini.template index 19e3b2647..91236355d 100644 --- a/birdhouse/components/weaver/config/weaver/weaver.ini.template +++ b/birdhouse/components/weaver/config/weaver/weaver.ini.template @@ -24,7 +24,7 @@ mongodb.db_name = weaver # --- Weaver Configuration --- weaver.configuration = ${WEAVER_CONFIG} -weaver.url = https://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME} +weaver.url = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME} # --- Weaver requests extension flags --- # use request-options for per-request specific disabling instead of globally disabled ssl_verify flag @@ -49,7 +49,7 @@ weaver.wps = true weaver.wps_path = ${WEAVER_WPS_PATH} weaver.wps_output = true weaver.wps_output_dir = ${WEAVER_WPS_OUTPUTS_DIR} -weaver.wps_output_url = https://${BIRDHOUSE_FQDN_PUBLIC}${WEAVER_WPS_OUTPUTS_PATH} +weaver.wps_output_url = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${WEAVER_WPS_OUTPUTS_PATH} weaver.wps_output_path = # default output sub-dir if user not specified (just for good measure, otherwise hook should default to it also) # see 'components/weaver/config/magpie/weaver_hooks.py:add_x_wps_output_context' that defines it when user is logged in @@ -60,7 +60,7 @@ weaver.wps_workdir = ${WEAVER_WPS_WORKDIR} weaver.wps_metadata_identification_title=Weaver weaver.wps_metadata_identification_abstract=Weaver is an Execution Management Service (EMS) that allows the execution of workflows chaining various applications and Web Processing Services (WPS) inputs and outputs. Remote execution is deferred by the EMS to an Application Deployment and Execution Service (ADES), as defined by Common Workflow Language (CWL) configurations. weaver.wps_metadata_identification_keywords=Weaver,WPS,OGC,${WEAVER_CONFIG} -weaver.wps_metadata_identification_accessconstraints=https://${BIRDHOUSE_FQDN_PUBLIC}/magpie +weaver.wps_metadata_identification_accessconstraints=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/magpie weaver.wps_metadata_identification_fees=NONE weaver.wps_metadata_provider_name=crim-ca/weaver weaver.wps_metadata_provider_url=http://pavics-weaver.readthedocs.org/en/latest/ diff --git a/birdhouse/components/weaver/post-docker-compose-up b/birdhouse/components/weaver/post-docker-compose-up index 5cb09b3e9..d4dbcfbc2 100755 --- a/birdhouse/components/weaver/post-docker-compose-up +++ b/birdhouse/components/weaver/post-docker-compose-up @@ -23,7 +23,7 @@ # # WEAVER_WPS_PROVIDERS: # list of provider names (comma or space delimited), all are assumed to be available at -# "https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/" +# "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/" # WEAVER_WPS_PROVIDERS_MAX_TIME: # limit script execution up to a maximum of this number of seconds # WEAVER_WPS_PROVIDERS_RETRY_COUNT: @@ -82,8 +82,8 @@ WARN="${PREFIX}${YELLOW}WARNING${NORMAL}: " echo "${PREFIX}Running: $0" -MAGPIE_URL="https://${BIRDHOUSE_FQDN_PUBLIC}/magpie" -WEAVER_URL="https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${WEAVER_MANAGER_NAME}" +MAGPIE_URL="${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/magpie" +WEAVER_URL="${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${WEAVER_MANAGER_NAME}" WEAVER_WPS_PROVIDERS_MAX_TIME=${WEAVER_WPS_PROVIDERS_MAX_TIME:-120} WEAVER_WPS_PROVIDERS_RETRY_AFTER=${WEAVER_WPS_PROVIDERS_RETRY_AFTER:-5} WEAVER_WPS_PROVIDERS_RETRY_COUNT=${WEAVER_WPS_PROVIDERS_RETRY_COUNT:-5} @@ -246,7 +246,7 @@ for prov in ${WEAVER_WPS_PROVIDERS}; do if [ -z "${prov}" ]; then continue fi - prov_url="https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${prov}" + prov_url="${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${prov}" prov_cap="${prov_url}?service=WPS&request=GetCapabilities" # wait for WPS provider to respond diff --git a/birdhouse/components/weaver/service-config.json.template b/birdhouse/components/weaver/service-config.json.template index 0e3f79fcb..a9acfd807 100644 --- a/birdhouse/components/weaver/service-config.json.template +++ b/birdhouse/components/weaver/service-config.json.template @@ -15,7 +15,7 @@ { "rel": "service", "type": "application/json", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/" }, { "rel": "service-doc", @@ -25,12 +25,12 @@ { "rel": "service-desc", "type": "application/json", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/" }, { "rel": "conformance", "type": "application/json", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/conformance/" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/conformance/" }, { "rel": "service-meta", diff --git a/birdhouse/deprecated-components/catalog/catalog.cfg.template b/birdhouse/deprecated-components/catalog/catalog.cfg.template index 07d6a71f0..a7ab2c7e5 100644 --- a/birdhouse/deprecated-components/catalog/catalog.cfg.template +++ b/birdhouse/deprecated-components/catalog/catalog.cfg.template @@ -3,7 +3,7 @@ solr_host=http://${BIRDHOUSE_FQDN}:8983/solr/${THREDDS_SERVICE_DATA_URL_PATH}/ # Multiple thredds hosts can be given, comma separated # note: this URL is also used as prefix when comparing authorizations from magpie -thredds_host=https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds +thredds_host=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds # Multiple esgf nodes can be given, comma separated esgf_nodes=https://esgf-node.llnl.gov/esg-search @@ -11,7 +11,7 @@ esgf_nodes=https://esgf-node.llnl.gov/esg-search # Provide a magpie host to filter results based on access permissions. # Must also provide credentials with read access so that the crawler can parse the thredds host(s) # Leave as a comment for a public catalog. -magpie_host=https://${BIRDHOUSE_FQDN_PUBLIC}/magpie +magpie_host=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/magpie magpie_user=${CATALOG_USERNAME} magpie_pw=${CATALOG_PASSWORD} # SSL verification (true or false) @@ -22,10 +22,10 @@ thredds_host_magpie_svc_name=${CATALOG_THREDDS_SERVICE} # WMS service url with replaced by each instance of the thredds_host, # without the port number and replaced by the base url in thredds. # Leave as comment to use the default WMS service -wms_alternate_server=https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/ncWMS2/wms?SERVICE=WMS&REQUEST=GetCapabilities&VERSION=1.3.0&DATASET=outputs/ +wms_alternate_server=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/ncWMS2/wms?SERVICE=WMS&REQUEST=GetCapabilities&VERSION=1.3.0&DATASET=outputs/ [pywps] -outputurl=https://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs/catalog +outputurl=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs/catalog parallelprocesses=30 [logging] diff --git a/birdhouse/deprecated-components/flyingpigeon/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/deprecated-components/flyingpigeon/config/canarie-api/canarie_api_monitoring.py.template index ec4110010..1dd847a0b 100644 --- a/birdhouse/deprecated-components/flyingpigeon/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/deprecated-components/flyingpigeon/config/canarie-api/canarie_api_monitoring.py.template @@ -24,7 +24,7 @@ SERVICES['flyingpigeon'] = { 'releasenotes': 'https://github.com/bird-house/flyingpigeon/blob/master/CHANGES.rst', 'support': 'https://github.com/bird-house/flyingpigeon/issues', 'source': 'https://github.com/bird-house/flyingpigeon', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}/flyingpigeon/wps?service=WPS&version=1.0.0&request=GetCapabilities', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/flyingpigeon/wps?service=WPS&version=1.0.0&request=GetCapabilities', 'licence': 'https://github.com/bird-house/flyingpigeon/blob/master/LICENSE.txt', 'provenance': 'https://github.com/bird-house/flyingpigeon' }, diff --git a/birdhouse/deprecated-components/flyingpigeon/service-config.json.template b/birdhouse/deprecated-components/flyingpigeon/service-config.json.template index 2924e081b..2ff81e734 100644 --- a/birdhouse/deprecated-components/flyingpigeon/service-config.json.template +++ b/birdhouse/deprecated-components/flyingpigeon/service-config.json.template @@ -14,7 +14,7 @@ { "rel": "service", "type": "text/xml", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/flyingpigeon?service=WPS&request=GetCapabilities" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/flyingpigeon?service=WPS&request=GetCapabilities" }, { "rel": "service-doc", @@ -24,7 +24,7 @@ { "rel": "service-desc", "type": "text/xml", - "href": "https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/flyingpigeon?service=WPS&request=GetCapabilities" + "href": "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/flyingpigeon?service=WPS&request=GetCapabilities" }, { "rel": "service-meta", diff --git a/birdhouse/deprecated-components/flyingpigeon/wps.cfg.template b/birdhouse/deprecated-components/flyingpigeon/wps.cfg.template index ff15cd061..218f1caa5 100644 --- a/birdhouse/deprecated-components/flyingpigeon/wps.cfg.template +++ b/birdhouse/deprecated-components/flyingpigeon/wps.cfg.template @@ -1,5 +1,5 @@ [server] -outputurl = https://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs/flyingpigeon +outputurl = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs/flyingpigeon outputpath = /data/wpsoutputs/flyingpigeon maxsingleinputsize = 2097152000.0 diff --git a/birdhouse/deprecated-components/frontend/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/deprecated-components/frontend/config/canarie-api/canarie_api_monitoring.py.template index 7a0a2175c..4b83a1ede 100644 --- a/birdhouse/deprecated-components/frontend/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/deprecated-components/frontend/config/canarie-api/canarie_api_monitoring.py.template @@ -1,5 +1,5 @@ PLATFORMS['server']['monitoring']['Frontend'] = { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}' } } diff --git a/birdhouse/deprecated-components/frontend/frontend.env.template b/birdhouse/deprecated-components/frontend/frontend.env.template index e588a127b..1507df2d6 100644 --- a/birdhouse/deprecated-components/frontend/frontend.env.template +++ b/birdhouse/deprecated-components/frontend/frontend.env.template @@ -3,6 +3,6 @@ PAVICS_FRONTEND_PORT=443 PAVICS_FRONTEND_PROTO=https BIRDHOUSE_HOST=${BIRDHOUSE_FQDN} NODE_TLS_REJECT_UNAUTHORIZED=0 -NCWMS_HOST=https://${BIRDHOUSE_FQDN}${TWITCHER_PROTECTED_PATH}/ncWMS2/wms -CATALOG_HOST=https://${BIRDHOUSE_FQDN}${TWITCHER_PROTECTED_PATH}/catalog/pywps -MALLEEFOWL_HOST=https://${BIRDHOUSE_FQDN}${TWITCHER_PROTECTED_PATH}/malleefowl/wps +NCWMS_HOST=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}${TWITCHER_PROTECTED_PATH}/ncWMS2/wms +CATALOG_HOST=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}${TWITCHER_PROTECTED_PATH}/catalog/pywps +MALLEEFOWL_HOST=${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}${TWITCHER_PROTECTED_PATH}/malleefowl/wps diff --git a/birdhouse/deprecated-components/malleefowl/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/deprecated-components/malleefowl/config/canarie-api/canarie_api_monitoring.py.template index 7ce8ef425..5c93687de 100644 --- a/birdhouse/deprecated-components/malleefowl/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/deprecated-components/malleefowl/config/canarie-api/canarie_api_monitoring.py.template @@ -19,7 +19,7 @@ SERVICES['Malleefowl'] = { 'releasenotes': 'https://github.com/Ouranosinc/malleefowl/blob/master/CHANGES.rst', 'support': 'https://github.com/Ouranosinc/malleefowl/issues', 'source': 'https://github.com/Ouranosinc/malleefowl', - 'tryme': 'https://${BIRDHOUSE_FQDN_PUBLIC}/malleefowl/', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/malleefowl/', 'licence': '', 'provenance': '' }, diff --git a/birdhouse/deprecated-components/phoenix/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/deprecated-components/phoenix/config/canarie-api/canarie_api_monitoring.py.template index 8bd804ceb..51cfbe8ed 100644 --- a/birdhouse/deprecated-components/phoenix/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/deprecated-components/phoenix/config/canarie-api/canarie_api_monitoring.py.template @@ -19,7 +19,7 @@ SERVICES['Phoenix'] = { 'releasenotes': 'https://github.com/ouranosinc/pyramid-phoenix/CHANGES.rst', 'support': 'https://github.com/ouranosinc/pyramid-phoenix/issues', 'source': 'https://github.com/ouranosinc/pyramid-phoenix', - 'tryme': 'https://${BIRDHOUSE_FQDN}:8443/', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}:8443/', 'licence': 'https://github.com/ouranosinc/pyramid-phoenix/blob/master/LICENSE.txt', 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' }, @@ -27,7 +27,7 @@ SERVICES['Phoenix'] = { "Phoenix": { 'request': { # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222) - 'url': 'https://${BIRDHOUSE_FQDN}:8443/' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}:8443/' } } } diff --git a/birdhouse/deprecated-components/project-api/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/deprecated-components/project-api/config/canarie-api/canarie_api_monitoring.py.template index 9b3b4e7e4..411c2158c 100644 --- a/birdhouse/deprecated-components/project-api/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/deprecated-components/project-api/config/canarie-api/canarie_api_monitoring.py.template @@ -1,6 +1,6 @@ PLATFORMS['server']['monitoring']['Project'] = { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}/project-api/explorer/' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/project-api/explorer/' } } PLATFORMS['server']['stats']['route'] = '/project-api/.*' diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example index c247d506b..8fd1dcd99 100644 --- a/birdhouse/env.local.example +++ b/birdhouse/env.local.example @@ -538,8 +538,8 @@ export THREDDS_ADDITIONAL_CATALOG='' # To setup Github as login, goto under section [OAuth Apps] # and create a new Magpie application with configurations: # -# Homepage URL: https://${BIRDHOUSE_FQDN} -# Authorization callback URL: https://${BIRDHOUSE_FQDN}/magpie/providers/github/signin +# Homepage URL: ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN} +# Authorization callback URL: ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}/magpie/providers/github/signin # # Then, specify obtained Github client ID/Secret for this Magpie OAuth App with following variables. # @@ -590,7 +590,7 @@ export THREDDS_ADDITIONAL_CATALOG='' # Raven to use the local Geoserver instead of the default production. # See raven/default.env for more info. -#export RAVEN_GEO_URL="https://${BIRDHOUSE_FQDN}/geoserver/" +#export RAVEN_GEO_URL="${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}/geoserver/" # Mount point on host machine to store mongodb server data # (note: if using 'BIRDHOUSE_DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env') diff --git a/birdhouse/optional-components/README.rst b/birdhouse/optional-components/README.rst index 2210bbe1b..b4aae01cb 100644 --- a/birdhouse/optional-components/README.rst +++ b/birdhouse/optional-components/README.rst @@ -487,3 +487,11 @@ For developers, to create a new parser that can be used to track log files: .. _log-parser: https://github.com/DACCS-Climate/log-parser/ .. _prometheus_python_metrics: https://prometheus.github.io/client_python/instrumenting/ + + +Local Dev Test +-------------- + +.. code:: shell + + echo '127.0.0.1 host.docker.internal' | sudo tee -a /etc/hosts diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/a_demo_override_precedence.py b/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/a_demo_override_precedence.py index b3c238cf6..31b503d0a 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/a_demo_override_precedence.py +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/a_demo_override_precedence.py @@ -5,7 +5,7 @@ # SERVICES['node']['monitoring'].update({ # 'Thredds-public': { # 'request': { -# 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}/toto', +# 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/toto', # }, # }, # }) diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/z_demo_only_py_file_are_loaded.wrongsuffix b/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/z_demo_only_py_file_are_loaded.wrongsuffix index a85d12907..5e2d072dd 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/z_demo_only_py_file_are_loaded.wrongsuffix +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/z_demo_only_py_file_are_loaded.wrongsuffix @@ -2,7 +2,7 @@ SERVICES['node']['monitoring'].update({ 'Thredds-public': { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}/wrong-suffix', + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/wrong-suffix', }, }, }) diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/catalog/canarie_api_full_monitoring.py.template b/birdhouse/optional-components/canarie-api-full-monitoring/config/catalog/canarie_api_full_monitoring.py.template index dc8296d8b..2241fbb32 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/catalog/canarie_api_full_monitoring.py.template +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/catalog/canarie_api_full_monitoring.py.template @@ -3,7 +3,7 @@ import copy service_public = "Catalog-public" service_public_cfg = { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/catalog?service=WPS&version=1.0.0&request=GetCapabilities' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/catalog?service=WPS&version=1.0.0&request=GetCapabilities' }, } diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/cowbird/canarie_api_full_monitoring.py.template b/birdhouse/optional-components/canarie-api-full-monitoring/config/cowbird/canarie_api_full_monitoring.py.template index 1e8596a70..58843b316 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/cowbird/canarie_api_full_monitoring.py.template +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/cowbird/canarie_api_full_monitoring.py.template @@ -4,7 +4,7 @@ if "Cowbird" in SERVICES: # See notes in 'components/cowbird/docker-compose-extra.yml' # about the conditional loading of this config in 'proxy' service. cowbird_cfg = copy.deepcopy(SERVICES["Cowbird"]["monitoring"]["Cowbird"]) - cowbird_cfg["request"]["url"] = "https://${BIRDHOUSE_FQDN_PUBLIC}/cowbird/" + cowbird_cfg["request"]["url"] = "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/cowbird/" SERVICES["Cowbird"]["monitoring"]["Cowbird-public"] = cowbird_cfg # vi: tabstop=8 expandtab shiftwidth=4 softtabstop=4 syntax=python diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/finch/canarie_api_full_monitoring.py.template b/birdhouse/optional-components/canarie-api-full-monitoring/config/finch/canarie_api_full_monitoring.py.template index 89b2066a8..b3e84fa7f 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/finch/canarie_api_full_monitoring.py.template +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/finch/canarie_api_full_monitoring.py.template @@ -3,7 +3,7 @@ import copy service_public = "Finch-public" service_public_cfg = { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&version=1.0.0&request=GetCapabilities' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&version=1.0.0&request=GetCapabilities' } } diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/hummingbird/canarie_api_full_monitoring.py.template b/birdhouse/optional-components/canarie-api-full-monitoring/config/hummingbird/canarie_api_full_monitoring.py.template index 9062bd33a..6258d88c6 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/hummingbird/canarie_api_full_monitoring.py.template +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/hummingbird/canarie_api_full_monitoring.py.template @@ -3,7 +3,7 @@ import copy service_public = "Hummingbird-public" service_public_cfg = { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird?service=WPS&version=1.0.0&request=GetCapabilities' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird?service=WPS&version=1.0.0&request=GetCapabilities' } } diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/malleefowl/canarie_api_full_monitoring.py.template b/birdhouse/optional-components/canarie-api-full-monitoring/config/malleefowl/canarie_api_full_monitoring.py.template index 8db3c6fbc..91aeedb35 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/malleefowl/canarie_api_full_monitoring.py.template +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/malleefowl/canarie_api_full_monitoring.py.template @@ -3,7 +3,7 @@ import copy service_public = "Malleefowl-public" service_public_cfg = { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/malleefowl?service=WPS&version=1.0.0&request=GetCapabilities' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/malleefowl?service=WPS&version=1.0.0&request=GetCapabilities' } } diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/ncwms2/canarie_api_full_monitoring.py.template b/birdhouse/optional-components/canarie-api-full-monitoring/config/ncwms2/canarie_api_full_monitoring.py.template index 3fb67f44a..9c81ec1f7 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/ncwms2/canarie_api_full_monitoring.py.template +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/ncwms2/canarie_api_full_monitoring.py.template @@ -3,7 +3,7 @@ import copy service_public = "ncWMS2-public" service_public_cfg = { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/ncWMS2/wms?SERVICE=WMS&REQUEST=GetCapabilities&VERSION=1.3.0' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/ncWMS2/wms?SERVICE=WMS&REQUEST=GetCapabilities&VERSION=1.3.0' } } diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/raven/canarie_api_full_monitoring.py.template b/birdhouse/optional-components/canarie-api-full-monitoring/config/raven/canarie_api_full_monitoring.py.template index 3420fa0dc..09fcce609 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/raven/canarie_api_full_monitoring.py.template +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/raven/canarie_api_full_monitoring.py.template @@ -3,7 +3,7 @@ import copy service_public = "Raven-public" service_public_cfg = { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven?service=WPS&version=1.0.0&request=GetCapabilities' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven?service=WPS&version=1.0.0&request=GetCapabilities' } } diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/thredds/canarie_api_full_monitoring.py.template b/birdhouse/optional-components/canarie-api-full-monitoring/config/thredds/canarie_api_full_monitoring.py.template index 03f44ab94..a5ae7bda1 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/thredds/canarie_api_full_monitoring.py.template +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/thredds/canarie_api_full_monitoring.py.template @@ -3,7 +3,7 @@ import copy service_public = "Thredds-public" service_public_cfg = { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}/thredds/catalog.html', + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/thredds/catalog.html', }, } diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/weaver/canarie_api_full_monitoring.py.template b/birdhouse/optional-components/canarie-api-full-monitoring/config/weaver/canarie_api_full_monitoring.py.template index 12ea37be9..717093961 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/weaver/canarie_api_full_monitoring.py.template +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/weaver/canarie_api_full_monitoring.py.template @@ -4,7 +4,7 @@ if "Weaver" in SERVICES: # See notes in 'components/weaver/docker-compose-extra.yml' # about the conditional loading of this config in 'proxy' service. weaver_cfg = copy.deepcopy(SERVICES["Weaver"]["monitoring"]["Weaver"]) - weaver_cfg["request"]["url"] = "https://${BIRDHOUSE_FQDN_PUBLIC}/weaver/" + weaver_cfg["request"]["url"] = "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/weaver/" SERVICES["Weaver"]["monitoring"]["Weaver-public"] = weaver_cfg # vi: tabstop=8 expandtab shiftwidth=4 softtabstop=4 syntax=python diff --git a/birdhouse/optional-components/emu/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/optional-components/emu/config/canarie-api/canarie_api_monitoring.py.template index 2b822d1b6..8f3b9b26f 100644 --- a/birdhouse/optional-components/emu/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/optional-components/emu/config/canarie-api/canarie_api_monitoring.py.template @@ -1,7 +1,7 @@ SERVICES['node']['monitoring'].update({ '${EMU_NAME}-public': { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${EMU_NAME}?service=WPS&version=1.0.0&request=GetCapabilities' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${EMU_NAME}?service=WPS&version=1.0.0&request=GetCapabilities' }, }, '${EMU_NAME}': { diff --git a/birdhouse/optional-components/emu/wps.cfg.template b/birdhouse/optional-components/emu/wps.cfg.template index 53001b15e..b9b9738c3 100644 --- a/birdhouse/optional-components/emu/wps.cfg.template +++ b/birdhouse/optional-components/emu/wps.cfg.template @@ -1,5 +1,5 @@ [server] -outputurl = https://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs +outputurl = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs outputpath = /data/wpsoutputs [logging] diff --git a/birdhouse/optional-components/generic_bird/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/optional-components/generic_bird/config/canarie-api/canarie_api_monitoring.py.template index c71b88ef5..6e12b859c 100644 --- a/birdhouse/optional-components/generic_bird/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/optional-components/generic_bird/config/canarie-api/canarie_api_monitoring.py.template @@ -1,7 +1,7 @@ SERVICES['node']['monitoring'].update({ '${GENERIC_BIRD_NAME}-public': { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${GENERIC_BIRD_NAME}?service=WPS&version=1.0.0&request=GetCapabilities' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${GENERIC_BIRD_NAME}?service=WPS&version=1.0.0&request=GetCapabilities' }, }, '${GENERIC_BIRD_NAME}': { diff --git a/birdhouse/optional-components/generic_bird/wps.cfg.template b/birdhouse/optional-components/generic_bird/wps.cfg.template index 9a029e823..076881c47 100644 --- a/birdhouse/optional-components/generic_bird/wps.cfg.template +++ b/birdhouse/optional-components/generic_bird/wps.cfg.template @@ -1,5 +1,5 @@ [server] -outputurl = https://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs +outputurl = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/wpsoutputs outputpath = /data/wpsoutputs # default 3mb, fix "Broken pipe" between the proxy and the wps service diff --git a/birdhouse/optional-components/local-dev-test/pre-docker-compose-up.include b/birdhouse/optional-components/local-dev-test/pre-docker-compose-up.include index 3d11b1f83..d0d7d5b90 100644 --- a/birdhouse/optional-components/local-dev-test/pre-docker-compose-up.include +++ b/birdhouse/optional-components/local-dev-test/pre-docker-compose-up.include @@ -2,9 +2,10 @@ # Note: filename is not docker-compose-extra.yml so that it won't get added prematurely to COMPOSE_CONF_LIST THIS_COMPOSE_FILE="${COMPOSE_DIR}/optional-components/local-dev-test/docker-compose-extra-ignore.yml" -echo "services:" > "${THIS_COMPOSE_FILE}" +echo 'version: "3.4"' > "${THIS_COMPOSE_FILE}" +echo "services:" >> "${THIS_COMPOSE_FILE}" -for service in $(PROXY_SECURE_PORT=443 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${COMPOSE_CONF_LIST} config --services); do +for service in $(PROXY_HTTP_PORT=80 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${COMPOSE_CONF_LIST} config --services); do printf ' %s:\n extra_hosts:\n - "host.docker.internal:host-gateway"\n' $service >> "${THIS_COMPOSE_FILE}" done @@ -12,6 +13,10 @@ if [ "${BIRDHOUSE_FQDN}" != "host.docker.internal" ]; then log WARN 'BIRDHOUSE_FQDN should be set to 'host.docker.internal' when the local-dev-test optional component is enabled' fi +if [ "${BIRDHOUSE_HTTP_ONLY}" != "True" ]; then + log WARN 'BIRDHOUSE_HTTP_ONLY should be set to 'True' when the local-dev-test optional component is enabled' +fi + COMPOSE_CONF_LIST="${COMPOSE_CONF_LIST} -f ${THIS_COMPOSE_FILE}" log INFO "adding ${THIS_COMPOSE_FILE} to COMPOSE_CONF_LIST" diff --git a/birdhouse/optional-components/secure-data-proxy/config/proxy/conf.extra-service.d/secure-data-auth.conf.template b/birdhouse/optional-components/secure-data-proxy/config/proxy/conf.extra-service.d/secure-data-auth.conf.template index 8034910c3..333f68d05 100644 --- a/birdhouse/optional-components/secure-data-proxy/config/proxy/conf.extra-service.d/secure-data-auth.conf.template +++ b/birdhouse/optional-components/secure-data-proxy/config/proxy/conf.extra-service.d/secure-data-auth.conf.template @@ -4,7 +4,7 @@ # note: using 'TWITCHER_VERIFY_PATH' path to avoid performing the request via 'proxy' endpoint # This ensures that the data access is validated for the user, but does not trigger its access/download twice. # Also, avoids getting an error as 'secure-data-proxy' private URL in Magpie doesn't resolve to a valid path. - proxy_pass https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/secure-data-proxy$request_uri; + proxy_pass ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/secure-data-proxy$request_uri; proxy_pass_request_body off; proxy_set_header Host $host; proxy_set_header Content-Length ""; diff --git a/birdhouse/optional-components/test-cowbird-jupyter-access/docker-compose-extra.yml b/birdhouse/optional-components/test-cowbird-jupyter-access/docker-compose-extra.yml index e32c111cb..50a444162 100644 --- a/birdhouse/optional-components/test-cowbird-jupyter-access/docker-compose-extra.yml +++ b/birdhouse/optional-components/test-cowbird-jupyter-access/docker-compose-extra.yml @@ -9,7 +9,7 @@ services: environment: MAGPIE_ADMIN_USERNAME: ${MAGPIE_ADMIN_USERNAME} MAGPIE_ADMIN_PASSWORD: ${MAGPIE_ADMIN_PASSWORD} - BIRDHOUSE_HOST_URL: https://${BIRDHOUSE_FQDN_PUBLIC} + BIRDHOUSE_HOST_URL: ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC} WPS_OUTPUTS_DIR: ${BIRDHOUSE_WPS_OUTPUTS_DIR} WORKSPACE_DIR: ${BIRDHOUSE_DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES} TEST_COWBIRD_JUPYTERHUB_USERNAME: ${TEST_COWBIRD_JUPYTERHUB_USERNAME} diff --git a/birdhouse/optional-components/test-weaver/config/weaver/request_options.yml.template b/birdhouse/optional-components/test-weaver/config/weaver/request_options.yml.template index f4434b3b7..1794dd160 100644 --- a/birdhouse/optional-components/test-weaver/config/weaver/request_options.yml.template +++ b/birdhouse/optional-components/test-weaver/config/weaver/request_options.yml.template @@ -11,5 +11,5 @@ requests: # disable SSL verification for test instance using self-signed certificate # avoid doing this on real instance to keep it secure against man-in-the-middle attacks - - url: https://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/weaver/providers/hummingbird/processes/ncdump/jobs + - url: ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/weaver/providers/hummingbird/processes/ncdump/jobs verify: false diff --git a/birdhouse/optional-components/testthredds/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/optional-components/testthredds/config/canarie-api/canarie_api_monitoring.py.template index 9c0051f35..9bbfc5879 100644 --- a/birdhouse/optional-components/testthredds/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/optional-components/testthredds/config/canarie-api/canarie_api_monitoring.py.template @@ -1,7 +1,7 @@ SERVICES['node']['monitoring'].update({ '${TESTTHREDDS_NAME}-public': { 'request': { - 'url': 'https://${BIRDHOUSE_FQDN_PUBLIC}/${TESTTHREDDS_CONTEXT_ROOT}/catalog.html' + 'url': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/${TESTTHREDDS_CONTEXT_ROOT}/catalog.html' }, }, '${TESTTHREDDS_NAME}': { diff --git a/birdhouse/scripts/check-instance-ready b/birdhouse/scripts/check-instance-ready index 122bd6924..e4f872353 100755 --- a/birdhouse/scripts/check-instance-ready +++ b/birdhouse/scripts/check-instance-ready @@ -19,7 +19,7 @@ if [ -f "${COMPOSE_DIR}/read-configs.include.sh" ]; then fi set -x -curl --include --silent "https://${BIRDHOUSE_FQDN}/canarie/node/service/stats" | head +curl --include --silent "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}/canarie/node/service/stats" | head set +x echo " @@ -28,7 +28,7 @@ The curl above should return the HTTP response code 200 to confirm instance is r set -x HTTP_RESPONSE_CODE="$( \ - curl --write-out '%{http_code}' --output /dev/null --silent "https://${BIRDHOUSE_FQDN}/canarie/node/service/stats" \ + curl --write-out '%{http_code}' --output /dev/null --silent "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}/canarie/node/service/stats" \ )" if [ "${HTTP_RESPONSE_CODE}" -ne 200 ]; then set +x @@ -41,5 +41,5 @@ Will retry only once more and exit immediately. " set -x sleep 65 - curl --include --silent "https://${BIRDHOUSE_FQDN}/canarie/node/service/stats" | head + curl --include --silent "${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}/canarie/node/service/stats" | head fi From 5a6f3b104a442d4bf9fd39873435fc2f688ab4bb Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Wed, 27 Nov 2024 14:18:38 -0500 Subject: [PATCH 05/18] test fixes --- birdhouse/scripts/logging.include.sh | 2 +- tests/test_deployment.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/birdhouse/scripts/logging.include.sh b/birdhouse/scripts/logging.include.sh index f57ed2a94..b64e0eabb 100644 --- a/birdhouse/scripts/logging.include.sh +++ b/birdhouse/scripts/logging.include.sh @@ -36,7 +36,7 @@ fi log_dest() { - read log_line + read log_line || true if [ -n "${log_line}" ]; then if [ -n "${BIRDHOUSE_LOG_FILE}" ]; then echo "$log_line" >> "${BIRDHOUSE_LOG_FILE}" diff --git a/tests/test_deployment.py b/tests/test_deployment.py index 9566ec7fa..10c999aca 100644 --- a/tests/test_deployment.py +++ b/tests/test_deployment.py @@ -13,6 +13,7 @@ "BIRDHOUSE_FQDN_PUBLIC": os.environ.get("BIRDHOUSE_FQDN_PUBLIC", "example.com"), "WEAVER_MANAGER_NAME": os.environ.get("WEAVER_MANAGER_NAME", "weaver"), "TWITCHER_PROTECTED_PATH": os.environ.get("TWITCHER_PROTECTED_PATH", "/twitcher/ows/proxy"), + "BIRDHOUSE_PROXY_SCHEME": os.environ.get("BIRDHOUSE_PROXY_SCHEME", "http") } From 9604d5ca80c00c66f16dd1e901cd23c4a17ba78d Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Wed, 27 Nov 2024 14:18:57 -0500 Subject: [PATCH 06/18] documentation fixes --- birdhouse/optional-components/README.rst | 19 +++++++++++++++++ .../pre-docker-compose-up.include | 21 ++++++++++++------- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/birdhouse/optional-components/README.rst b/birdhouse/optional-components/README.rst index b4aae01cb..3f9fc53ff 100644 --- a/birdhouse/optional-components/README.rst +++ b/birdhouse/optional-components/README.rst @@ -492,6 +492,25 @@ For developers, to create a new parser that can be used to track log files: Local Dev Test -------------- +This allows users to deploy the entire stack locally for development or testing purposes. + +If this component is enabled the following configuration settings must also be set in the local environment file: + + * ``BIRDHOUSE_FQDN=host.docker.internal`` + * ``BIRDHOUSE_HTTP_ONLY=True`` + +You should also add ``host.docker.internal`` to your ``/etc/hosts`` file pointing to the loopback address so that URLs +generated by Birdhouse that refer to ``host.docker.internal`` will resolve properly in a browser: + .. code:: shell echo '127.0.0.1 host.docker.internal' | sudo tee -a /etc/hosts + +After deploying the stack, you can now interact with the Birdhouse software at ``http://host.docker.internal`` from the +machine that is the docker host. + +Note that you do *not* need an SSL certificate set up to deploy the stack in this way. + +.. warning:: + + **DO NOT** enable this component in production. This is intended for local development and test purposes only! diff --git a/birdhouse/optional-components/local-dev-test/pre-docker-compose-up.include b/birdhouse/optional-components/local-dev-test/pre-docker-compose-up.include index d0d7d5b90..07830e752 100644 --- a/birdhouse/optional-components/local-dev-test/pre-docker-compose-up.include +++ b/birdhouse/optional-components/local-dev-test/pre-docker-compose-up.include @@ -1,5 +1,18 @@ #!/usr/bin/env sh +error= +if [ "${BIRDHOUSE_FQDN}" != "host.docker.internal" ]; then + log ERROR 'BIRDHOUSE_FQDN should be set to 'host.docker.internal' when the local-dev-test optional component is enabled' + error=True +fi + +if [ "${BIRDHOUSE_HTTP_ONLY}" != "True" ]; then + log ERROR 'BIRDHOUSE_HTTP_ONLY should be set to 'True' when the local-dev-test optional component is enabled' + error=True +fi + +[ "$error" = 'True' ] && exit 1 + # Note: filename is not docker-compose-extra.yml so that it won't get added prematurely to COMPOSE_CONF_LIST THIS_COMPOSE_FILE="${COMPOSE_DIR}/optional-components/local-dev-test/docker-compose-extra-ignore.yml" echo 'version: "3.4"' > "${THIS_COMPOSE_FILE}" @@ -9,14 +22,6 @@ for service in $(PROXY_HTTP_PORT=80 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${ printf ' %s:\n extra_hosts:\n - "host.docker.internal:host-gateway"\n' $service >> "${THIS_COMPOSE_FILE}" done -if [ "${BIRDHOUSE_FQDN}" != "host.docker.internal" ]; then - log WARN 'BIRDHOUSE_FQDN should be set to 'host.docker.internal' when the local-dev-test optional component is enabled' -fi - -if [ "${BIRDHOUSE_HTTP_ONLY}" != "True" ]; then - log WARN 'BIRDHOUSE_HTTP_ONLY should be set to 'True' when the local-dev-test optional component is enabled' -fi - COMPOSE_CONF_LIST="${COMPOSE_CONF_LIST} -f ${THIS_COMPOSE_FILE}" log INFO "adding ${THIS_COMPOSE_FILE} to COMPOSE_CONF_LIST" From 5afc7ad3d45937863e3bd2b66d034a7d354c81da Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Tue, 3 Dec 2024 11:46:44 -0500 Subject: [PATCH 07/18] stac fixes --- .../config/proxy/conf.extra-service.d/stac.conf.template | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/birdhouse/components/stac/config/proxy/conf.extra-service.d/stac.conf.template b/birdhouse/components/stac/config/proxy/conf.extra-service.d/stac.conf.template index 8db59fcc5..7af02c3dc 100644 --- a/birdhouse/components/stac/config/proxy/conf.extra-service.d/stac.conf.template +++ b/birdhouse/components/stac/config/proxy/conf.extra-service.d/stac.conf.template @@ -1,5 +1,5 @@ - location /stac { + location /stac/ { # We need the first `/stac` for service resolution. # We need the second `/stac` for API redirect in STAC (see `root-path` and `ROUTER_PREFIX`). # See https://github.com/stac-utils/stac-fastapi/issues/427 @@ -8,12 +8,19 @@ proxy_set_header Host $host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Host $host:$server_port; + #proxy_set_header X-Forwarded-Proto $scheme; proxy_buffering off; include /etc/nginx/conf.d/cors.include; } # Automatically redirect to /stac/stac and exclude redirect when already using /stac location ~ ^${TWITCHER_PROTECTED_PATH}/stac(?!/stac) { + #proxy_set_header Host $host; + #proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + #proxy_set_header X-Forwarded-Host $host:$server_port; + #proxy_set_header X-Forwarded-Proto $scheme; + #proxy_buffering off; + #include /etc/nginx/conf.d/cors.include; return 302 ${TWITCHER_PROTECTED_PATH}/stac/stac/; } From 2286baab992942308787a4b8791fbef89264b256 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Tue, 3 Dec 2024 13:43:03 -0500 Subject: [PATCH 08/18] fix proxy docker compose path --- birdhouse/components/proxy/pre-docker-compose-up.include | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/birdhouse/components/proxy/pre-docker-compose-up.include b/birdhouse/components/proxy/pre-docker-compose-up.include index b9bac8221..b46695038 100644 --- a/birdhouse/components/proxy/pre-docker-compose-up.include +++ b/birdhouse/components/proxy/pre-docker-compose-up.include @@ -3,8 +3,7 @@ # Note: filename is not docker-compose-extra.yml so that it won't get added prematurely to COMPOSE_CONF_LIST if [ x"$BIRDHOUSE_HTTP_ONLY" != x"True" ]; then - THIS_COMPOSE_FILE="${COMPOSE_DIR}/optional-components/local-dev-test/docker-compose-extra-ssl-cert.yml" - echo "services:" > "${THIS_COMPOSE_FILE}" + THIS_COMPOSE_FILE="${COMPOSE_DIR}/components/proxy/docker-compose-ssl-cert.yml" COMPOSE_CONF_LIST="${COMPOSE_CONF_LIST} -f ${THIS_COMPOSE_FILE}" From 6ee2df51d05d44960d15d6a1821938ee57a8c678 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Tue, 3 Dec 2024 15:56:09 -0500 Subject: [PATCH 09/18] enable extra hosts for docker spawner --- birdhouse/components/README.rst | 2 +- birdhouse/components/jupyterhub/default.env | 1 + birdhouse/components/jupyterhub/jupyterhub_config.py.template | 1 + birdhouse/optional-components/local-dev-test/default.env | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 birdhouse/optional-components/local-dev-test/default.env diff --git a/birdhouse/components/README.rst b/birdhouse/components/README.rst index c3733ce96..0adb96657 100644 --- a/birdhouse/components/README.rst +++ b/birdhouse/components/README.rst @@ -540,7 +540,7 @@ exposed by the current stack instance. Once this component is enabled, STAC API ``https:///stac-browser`` endpoint. In order to make the STAC browser the default entrypoint, define the following in the ``env.local`` file:: - export BIRDHOUSE_PROXY_ROOT_LOCATION="return 302 https://\$host/stac-browser;" + export BIRDHOUSE_PROXY_ROOT_LOCATION='return 302 ${BIRDHOUSE_PROXY_SCHEME}://\$host/stac-browser;' Here is a sample search query using a CLI:: diff --git a/birdhouse/components/jupyterhub/default.env b/birdhouse/components/jupyterhub/default.env index 91f40f08e..a8e6b0602 100644 --- a/birdhouse/components/jupyterhub/default.env +++ b/birdhouse/components/jupyterhub/default.env @@ -104,6 +104,7 @@ OPTIONAL_VARS=" \$JUPYTER_IDLE_SERVER_CULL_TIMEOUT \$JUPYTER_IDLE_KERNEL_CULL_TIMEOUT \$JUPYTER_IDLE_KERNEL_CULL_INTERVAL + \$JUPYTERHUB_DOCKER_EXTRA_HOSTS " # add any component that this component requires to run diff --git a/birdhouse/components/jupyterhub/jupyterhub_config.py.template b/birdhouse/components/jupyterhub/jupyterhub_config.py.template index 1b2434f82..170e2bfb4 100644 --- a/birdhouse/components/jupyterhub/jupyterhub_config.py.template +++ b/birdhouse/components/jupyterhub/jupyterhub_config.py.template @@ -203,6 +203,7 @@ c.Spawner.args = [ c.DockerSpawner.extra_host_config = { # start init pid 1 process to reap defunct processes 'init': True, + 'extra_hosts': dict(host_mapping.split(":") for host_mapping in "${JUPYTERHUB_DOCKER_EXTRA_HOSTS}".split()) } c.Authenticator.admin_users = ${JUPYTERHUB_ADMIN_USERS} # noqa diff --git a/birdhouse/optional-components/local-dev-test/default.env b/birdhouse/optional-components/local-dev-test/default.env new file mode 100644 index 000000000..34f6374ff --- /dev/null +++ b/birdhouse/optional-components/local-dev-test/default.env @@ -0,0 +1 @@ +export JUPYTERHUB_DOCKER_EXTRA_HOSTS='host.docker.internal:host-gateway' From afbcb40211d447ff4111581f8ca1e5df02c71b0e Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Wed, 4 Dec 2024 09:58:59 -0500 Subject: [PATCH 10/18] make it work for stac --- .../config/proxy/conf.extra-service.d/stac.conf.template | 7 ------- .../proxy/conf.extra-service.d/twitcher.conf.template | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/birdhouse/components/stac/config/proxy/conf.extra-service.d/stac.conf.template b/birdhouse/components/stac/config/proxy/conf.extra-service.d/stac.conf.template index 7af02c3dc..be72665bf 100644 --- a/birdhouse/components/stac/config/proxy/conf.extra-service.d/stac.conf.template +++ b/birdhouse/components/stac/config/proxy/conf.extra-service.d/stac.conf.template @@ -8,19 +8,12 @@ proxy_set_header Host $host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Host $host:$server_port; - #proxy_set_header X-Forwarded-Proto $scheme; proxy_buffering off; include /etc/nginx/conf.d/cors.include; } # Automatically redirect to /stac/stac and exclude redirect when already using /stac location ~ ^${TWITCHER_PROTECTED_PATH}/stac(?!/stac) { - #proxy_set_header Host $host; - #proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - #proxy_set_header X-Forwarded-Host $host:$server_port; - #proxy_set_header X-Forwarded-Proto $scheme; - #proxy_buffering off; - #include /etc/nginx/conf.d/cors.include; return 302 ${TWITCHER_PROTECTED_PATH}/stac/stac/; } diff --git a/birdhouse/components/twitcher/config/proxy/conf.extra-service.d/twitcher.conf.template b/birdhouse/components/twitcher/config/proxy/conf.extra-service.d/twitcher.conf.template index 2a7e897eb..44b558eaf 100644 --- a/birdhouse/components/twitcher/config/proxy/conf.extra-service.d/twitcher.conf.template +++ b/birdhouse/components/twitcher/config/proxy/conf.extra-service.d/twitcher.conf.template @@ -4,5 +4,5 @@ proxy_set_header X-Forwarded-Proto $real_scheme; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Host $host:$server_port; - proxy_set_header Forwarded "proto=https;host=${BIRDHOUSE_FQDN_PUBLIC}"; # Helps the STAC component to craft URLs containing the full BIRDHOUSE_FQDN_PUBLIC + proxy_set_header Forwarded "proto=${BIRDHOUSE_PROXY_SCHEME};host=${BIRDHOUSE_FQDN_PUBLIC}"; # Helps the STAC component to craft URLs containing the full BIRDHOUSE_FQDN_PUBLIC } From 7d069491e7a4070800ade1a6ede0031e7af8a21c Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Wed, 4 Dec 2024 11:04:20 -0500 Subject: [PATCH 11/18] update changes --- CHANGES.md | 44 +++++++++++++++++++++++- birdhouse/optional-components/README.rst | 4 +-- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 74f7fd58b..db775d94d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,7 +15,49 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ -[//]: # (list changes here, using '-' for each new entry, remove this when items are added) +## Changes + +- Enable local deployment to improve development and testing + + The Birdhouse stack can now be deployed locally and accessed on a browser on the host machine without the need + for an SSL certificate. This is useful for local development and for running tests against the full stack while developing and in CI environments. + + To enable this, add the new `optional-components/local-dev-test` component to `BIRDHOUSE_EXTRA_CONF_DIRS` and + set the following environment variables in the local environment file: + + * `export BIRDHOUSE_FQDN=host.docker.internal` + * `export BIRDHOUSE_HTTP_ONLY=True` + + You should also add ``host.docker.internal`` to your ``/etc/hosts`` file pointing to the loopback address so + that URLs generated by Birdhouse that refer to ``host.docker.internal`` will resolve properly in a browser: + + ``` + echo '127.0.0.1 host.docker.internal' | sudo tee -a /etc/hosts + ``` + + After deploying the stack, you can now interact with the Birdhouse software at ``http://host.docker.internal`` + from the machine that is the docker host. + + In order to implement the changes above, the following non-breaking changes have been made to the deployment code: + + - added a configuration variable `BIRDHOUSE_HTTP_ONLY` which is not set by default. If set to `True` the `proxy` component will only serve content over `http` (not `https`). + - added the following configuration variables. These should not be set directly unless you really know what you're doing: + - `BIRDHOUSE_PROXY_SCHEME`: default is `http` if `BIRDHOUSE_HTTP_ONLY` is `True`, otherwise `https` + - `PROXY_INCLUDE_HTTPS`: default is unset if `BIRDHOUSE_HTTP_ONLY` is `True`, otherwise `include /etc/nginx/conf.d/https.include;` + - changed the default values for the following configuration variables: + - `BIRDHOUSE_ALLOW_UNSECURE_HTTP`: default is now `True` if `BIRDHOUSE_HTTP_ONLY` is `True` + - logs are written to stderr by default. Previously they were written to stdout. + - this allows us to call scripts and programmatically use their outputs. Previously log entries would need to be + manually filtered out before program outputs could be used. + - added the `--log-stdout` and `--log-file` flags to the `bin/birdhouse` interface to allow redirecting logs to + stdout or to a specific file instead. + - log redirection can also now be set using environment variables: + - `BIRDHOUSE_LOG_FD` can be used to redirect logs to a file descriptor (ex: `BIRDHOUSE_LOG_FD=3`) + - `BIRDHOUSE_LOG_FILE` can be used to redirect logs to file (ex: `BIRDHOUSE_LOG_FILE=/some/file/on/disk.log`) + - Note that the variables here should not be set in the local environment file since that file is sourced **after** + some logs are written. Instead, set these by exporting them in the parent process that calls `bin/birdhouse`. + - for backwards compatibility, if scripts are not called through the `bin/birdhouse` interface, logs will still be + written to stdout. [2.6.2](https://github.com/bird-house/birdhouse-deploy/tree/2.6.2) (2024-12-03) ------------------------------------------------------------------------------------------------------------------ diff --git a/birdhouse/optional-components/README.rst b/birdhouse/optional-components/README.rst index 3f9fc53ff..ae1e208f7 100644 --- a/birdhouse/optional-components/README.rst +++ b/birdhouse/optional-components/README.rst @@ -496,8 +496,8 @@ This allows users to deploy the entire stack locally for development or testing If this component is enabled the following configuration settings must also be set in the local environment file: - * ``BIRDHOUSE_FQDN=host.docker.internal`` - * ``BIRDHOUSE_HTTP_ONLY=True`` + * ``export BIRDHOUSE_FQDN=host.docker.internal`` + * ``export BIRDHOUSE_HTTP_ONLY=True`` You should also add ``host.docker.internal`` to your ``/etc/hosts`` file pointing to the loopback address so that URLs generated by Birdhouse that refer to ``host.docker.internal`` will resolve properly in a browser: From 1156dd74eb58b3e7be7afcc44ea6e9d96abaed17 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Thu, 5 Dec 2024 10:10:59 -0500 Subject: [PATCH 12/18] review suggestions --- birdhouse/components/jupyterhub/default.env | 1 - birdhouse/components/jupyterhub/docker-compose-extra.yml | 1 + birdhouse/components/jupyterhub/jupyterhub_config.py.template | 3 ++- birdhouse/components/magpie/magpie.ini.template | 2 +- birdhouse/components/monitoring/default.env | 1 + birdhouse/components/monitoring/docker-compose-extra.yml | 2 +- .../stac/config/proxy/conf.extra-service.d/stac.conf.template | 2 +- .../config/canarie-api/canarie_api_monitoring.py.template | 2 +- birdhouse/optional-components/local-dev-test/default.env | 1 + 9 files changed, 9 insertions(+), 6 deletions(-) diff --git a/birdhouse/components/jupyterhub/default.env b/birdhouse/components/jupyterhub/default.env index a8e6b0602..91f40f08e 100644 --- a/birdhouse/components/jupyterhub/default.env +++ b/birdhouse/components/jupyterhub/default.env @@ -104,7 +104,6 @@ OPTIONAL_VARS=" \$JUPYTER_IDLE_SERVER_CULL_TIMEOUT \$JUPYTER_IDLE_KERNEL_CULL_TIMEOUT \$JUPYTER_IDLE_KERNEL_CULL_INTERVAL - \$JUPYTERHUB_DOCKER_EXTRA_HOSTS " # add any component that this component requires to run diff --git a/birdhouse/components/jupyterhub/docker-compose-extra.yml b/birdhouse/components/jupyterhub/docker-compose-extra.yml index 5c2cc0a1d..2ee32650a 100644 --- a/birdhouse/components/jupyterhub/docker-compose-extra.yml +++ b/birdhouse/components/jupyterhub/docker-compose-extra.yml @@ -28,6 +28,7 @@ services: USER_WORKSPACE_UID: ${USER_WORKSPACE_UID} USER_WORKSPACE_GID: ${USER_WORKSPACE_GID} JUPYTERHUB_CRYPT_KEY: ${JUPYTERHUB_CRYPT_KEY} + JUPYTERHUB_DOCKER_EXTRA_HOSTS: ${JUPYTERHUB_DOCKER_EXTRA_HOSTS:-} volumes: - ./components/jupyterhub/jupyterhub_config.py:/srv/jupyterhub/jupyterhub_config.py:ro - ./components/jupyterhub/custom_templates:/custom_templates:ro diff --git a/birdhouse/components/jupyterhub/jupyterhub_config.py.template b/birdhouse/components/jupyterhub/jupyterhub_config.py.template index 170e2bfb4..ad93e5058 100644 --- a/birdhouse/components/jupyterhub/jupyterhub_config.py.template +++ b/birdhouse/components/jupyterhub/jupyterhub_config.py.template @@ -200,10 +200,11 @@ c.Spawner.args = [ "--FileContentsManager.always_delete_dir=True", ] +## Note that JUPYTERHUB_DOCKER_EXTRA_HOSTS may be set by default in the local-dev-test component c.DockerSpawner.extra_host_config = { # start init pid 1 process to reap defunct processes 'init': True, - 'extra_hosts': dict(host_mapping.split(":") for host_mapping in "${JUPYTERHUB_DOCKER_EXTRA_HOSTS}".split()) + 'extra_hosts': dict(host_mapping.split(":") for host_mapping in os.getenv("JUPYTERHUB_DOCKER_EXTRA_HOSTS", "").split()) } c.Authenticator.admin_users = ${JUPYTERHUB_ADMIN_USERS} # noqa diff --git a/birdhouse/components/magpie/magpie.ini.template b/birdhouse/components/magpie/magpie.ini.template index 0af08d080..58e6b84da 100644 --- a/birdhouse/components/magpie/magpie.ini.template +++ b/birdhouse/components/magpie/magpie.ini.template @@ -28,7 +28,7 @@ pyramid.includes = pyramid_tm ziggurat_foundations.ext.pyramid.sign_in ziggurat_ # other overridable variables available in magpie/constants.py # magpie.port = 2001 -magpie.url = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN}/magpie +magpie.url = ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/magpie magpie.max_restart = 5 magpie.push_phoenix = true # This secret should be the same in Twitcher ! diff --git a/birdhouse/components/monitoring/default.env b/birdhouse/components/monitoring/default.env index 76188073d..5c9f3158b 100644 --- a/birdhouse/components/monitoring/default.env +++ b/birdhouse/components/monitoring/default.env @@ -19,6 +19,7 @@ export CADVISOR_IMAGE='${CADVISOR_DOCKER}:${CADVISOR_VERSION}' export NODE_EXPORTER_VERSION="v1.0.0" export NODE_EXPORTER_DOCKER="quay.io/prometheus/node-exporter" export NODE_EXPORTER_IMAGE='${NODE_EXPORTER_DOCKER}:${NODE_EXPORTER_VERSION}' +export NODE_EXPORTER_HOST_BIND_PROPOGATION=rslave export ALERTMANAGER_VERSION="v0.21.0" export ALERTMANAGER_DOCKER=prom/alertmanager diff --git a/birdhouse/components/monitoring/docker-compose-extra.yml b/birdhouse/components/monitoring/docker-compose-extra.yml index 4a99f7555..261826c83 100644 --- a/birdhouse/components/monitoring/docker-compose-extra.yml +++ b/birdhouse/components/monitoring/docker-compose-extra.yml @@ -21,7 +21,7 @@ services: image: ${NODE_EXPORTER_IMAGE} container_name: node-exporter volumes: - - /:/host:ro + - /:/host:ro,${NODE_EXPORTER_HOST_BIND_PROPOGATION} network_mode: "host" pid: "host" command: --path.rootfs=/host diff --git a/birdhouse/components/stac/config/proxy/conf.extra-service.d/stac.conf.template b/birdhouse/components/stac/config/proxy/conf.extra-service.d/stac.conf.template index be72665bf..8db59fcc5 100644 --- a/birdhouse/components/stac/config/proxy/conf.extra-service.d/stac.conf.template +++ b/birdhouse/components/stac/config/proxy/conf.extra-service.d/stac.conf.template @@ -1,5 +1,5 @@ - location /stac/ { + location /stac { # We need the first `/stac` for service resolution. # We need the second `/stac` for API redirect in STAC (see `root-path` and `ROUTER_PREFIX`). # See https://github.com/stac-utils/stac-fastapi/issues/427 diff --git a/birdhouse/components/weaver/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/weaver/config/canarie-api/canarie_api_monitoring.py.template index 94ee359e3..6ba1c83a2 100644 --- a/birdhouse/components/weaver/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/weaver/config/canarie-api/canarie_api_monitoring.py.template @@ -19,7 +19,7 @@ SERVICES['Weaver'] = { 'releasenotes': 'https://github.com/crim-ca/weaver/blob/master/CHANGES.rst', 'support': 'https://github.com/crim-ca/weaver/issues', 'source': 'https://github.com/crim-ca/weaver', - 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/weaver/', + 'tryme': '${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/', 'licence': 'https://github.com/crim-ca/weaver/blob/${WEAVER_VERSION}/LICENSE.txt', 'provenance': 'https://github.com/crim-ca/weaver' }, diff --git a/birdhouse/optional-components/local-dev-test/default.env b/birdhouse/optional-components/local-dev-test/default.env index 34f6374ff..4b9f67359 100644 --- a/birdhouse/optional-components/local-dev-test/default.env +++ b/birdhouse/optional-components/local-dev-test/default.env @@ -1 +1,2 @@ export JUPYTERHUB_DOCKER_EXTRA_HOSTS='host.docker.internal:host-gateway' +export NODE_EXPORTER_HOST_BIND_PROPOGATION= From 0466059256bcf3be49ec68fc0b08d665475d7b1d Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Thu, 5 Dec 2024 16:03:01 -0500 Subject: [PATCH 13/18] review suggestions --- CHANGES.md | 6 +++--- tests/test_deployment.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index db775d94d..1e5124e2b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -42,10 +42,10 @@ - added a configuration variable `BIRDHOUSE_HTTP_ONLY` which is not set by default. If set to `True` the `proxy` component will only serve content over `http` (not `https`). - added the following configuration variables. These should not be set directly unless you really know what you're doing: - - `BIRDHOUSE_PROXY_SCHEME`: default is `http` if `BIRDHOUSE_HTTP_ONLY` is `True`, otherwise `https` - - `PROXY_INCLUDE_HTTPS`: default is unset if `BIRDHOUSE_HTTP_ONLY` is `True`, otherwise `include /etc/nginx/conf.d/https.include;` + - `BIRDHOUSE_PROXY_SCHEME`: default remains `https`. If `BIRDHOUSE_HTTP_ONLY` is `True` then the default becomes `http` + - `PROXY_INCLUDE_HTTPS`: default remains `include /etc/nginx/conf.d/https.include;`. If `BIRDHOUSE_HTTP_ONLY` is `True`, the default is that the variable is unset. - changed the default values for the following configuration variables: - - `BIRDHOUSE_ALLOW_UNSECURE_HTTP`: default is now `True` if `BIRDHOUSE_HTTP_ONLY` is `True` + - `BIRDHOUSE_ALLOW_UNSECURE_HTTP`: default remains `""`. If `BIRDHOUSE_HTTP_ONLY` is `True` then the default becomes `True`. - logs are written to stderr by default. Previously they were written to stdout. - this allows us to call scripts and programmatically use their outputs. Previously log entries would need to be manually filtered out before program outputs could be used. diff --git a/tests/test_deployment.py b/tests/test_deployment.py index 10c999aca..c926b6bcb 100644 --- a/tests/test_deployment.py +++ b/tests/test_deployment.py @@ -13,7 +13,7 @@ "BIRDHOUSE_FQDN_PUBLIC": os.environ.get("BIRDHOUSE_FQDN_PUBLIC", "example.com"), "WEAVER_MANAGER_NAME": os.environ.get("WEAVER_MANAGER_NAME", "weaver"), "TWITCHER_PROTECTED_PATH": os.environ.get("TWITCHER_PROTECTED_PATH", "/twitcher/ows/proxy"), - "BIRDHOUSE_PROXY_SCHEME": os.environ.get("BIRDHOUSE_PROXY_SCHEME", "http") + "BIRDHOUSE_PROXY_SCHEME": os.environ.get("BIRDHOUSE_PROXY_SCHEME", "https") } From e655404d79bfbae4bf03d8865a76acacd9cb13ff Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Mon, 9 Dec 2024 11:54:34 -0500 Subject: [PATCH 14/18] add log destination options for each log level --- bin/birdhouse | 94 +++++++++++---- birdhouse/scripts/logging.include.sh | 165 +++++++++++++++++++-------- 2 files changed, 190 insertions(+), 69 deletions(-) diff --git a/bin/birdhouse b/bin/birdhouse index 798a84944..379b46290 100755 --- a/bin/birdhouse +++ b/bin/birdhouse @@ -2,12 +2,23 @@ THIS_FILE="$(readlink -f "$0" || realpath "$0")" THIS_DIR="$(dirname "${THIS_FILE}")" +THIS_BASENAME="$(basename "${THIS_FILE}")" COMPOSE_DIR="$(dirname "${THIS_DIR}")/birdhouse" export BIRDHOUSE_COMPOSE="${BIRDHOUSE_COMPOSE:-"${COMPOSE_DIR}/birdhouse-compose.sh"}" export __BIRDHOUSE_SUPPORTED_INTERFACE=True -USAGE="USAGE: $0 [-h|--help] [-b|--backwards-compatible] [-e|--env-file local-env-file] {[--log-stdout] | [--log-file log-file-path]} {info|compose|configs}" +USAGE="USAGE: $THIS_BASENAME [-h|--help] + [-b|--backwards-compatible] + [-e|--env-file local-env-file] + [-q|--quiet] [-q|--quiet {DEBUG|INFO|WARN|ERROR|CRITICAL}] + [-s|--log-stdout] [-s|--log-stdout {DEBUG|INFO|WARN|ERROR|CRITICAL}] + [-l|--log-file log-file-path] + [-l|--log-file {DEBUG|INFO|WARN|ERROR|CRITICAL} log-file-path] + [-L|--log-level {DEBUG|INFO|WARN|ERROR|CRITICAL}] + {info|compose|configs}" +USAGE=$(echo $USAGE | tr "\n" " ") + HELP="$USAGE Manage the Birdhouse software stack. @@ -18,14 +29,19 @@ Commands: configs Load or execute commands in the Birdhouse configuration environment Options: - -h, --help Print this message and exit - -b, --backwards-compatible Run in backwards compatible mode - -e, --env-file string Override the local environment file, default is ${COMPOSE_DIR}/env.local - --log-stdout Write logs to stdout, default is to write to stderr - --log-file string Write logs to this file path, default is to write to stderr + -h, --help Print this message and exit + -b, --backwards-compatible Run in backwards compatible mode + -e, --env-file string Override the local environment file, default is ${COMPOSE_DIR}/env.local + -s, --log-stdout Write logs to stdout, default is to write to stderr + -s, --log-stdout {DEBUG|INFO|WARN|ERROR|CRITICAL} Write logs to stdout for the given log level only, default is determined by --log-stdout (this option can be repeated) + -l, --log-file path Write logs to this file path, default is to write to stderr + -l, --log-file {DEBUG|INFO|WARN|ERROR|CRITICAL} path Write logs to this file path for the given log level only default is determined by --log-file (this option can be repeated) + -q, --quiet Do not write logs to stdout or stderr. Logs will still be written to a file if --log-file is set + -q, --quiet {DEBUG|INFO|WARN|ERROR|CRITICAL} Do not write logs to stdout or stderr for the given log level only, default is determined by --quiet (this option can be repeated) + " -CONFIGS_USAGE="USAGE: $0 configs [-h|--help] [-d|--default] {[-p|--print-config-command] | [-c|--command command]}" +CONFIGS_USAGE="USAGE: $THIS_BASENAME configs [-h|--help] [-d|--default] {[-p|--print-config-command] | [-c|--command command]}" CONFIGS_HELP="$CONFIGS_USAGE Load or execute commands in the Birdhouse configuration environment. @@ -34,15 +50,16 @@ Options: -d, --default Only load/print a command for the default configuration settings, not those specified by the local environment file -p, --print-config-command Print a command that can be used to load configuration settings as environment variables -c, --command string Execute the given command after loading configuration settings - -q, --quiet Suppress stdout when loading configuration settings for the '--command' option. +Deprecated Options: + -q, --quiet Suppress stdout when loading configuration settings for the '--command' option. [DEPRECATED: use the --quiet option directly under birdhouse instead] Example Usage: - $ ${0} configs -c 'echo \${BIRDHOUSE_FQDN}' + $ ${THIS_BASENAME} configs -c 'echo \${BIRDHOUSE_FQDN}' example.com # This is the value of BIRDHOUSE_FQDN as determined by the current configuration settings - $ ${0} configs -p + $ ${THIS_BASENAME} configs -p . /path/to/configs/file/to/source && read_configs - $ eval \$(${0} configs) + $ eval \$(${THIS_BASENAME} configs) $ echo \${BIRDHOUSE_FQDN} example.com # This is the value of BIRDHOUSE_FQDN as determined by the current configuration settings " @@ -150,7 +167,7 @@ parse_configs_args() { print_config_command elif [ "${CONFIGS_CMD+set}" = 'set' ]; then if [ "${CONFIGS_QUIET}" = "True" ]; then - eval "$(print_config_command)" > /dev/null + eval "$(print_config_command)" 2> /dev/null else eval "$(print_config_command)" fi @@ -166,6 +183,15 @@ parse_configs_args() { esac } +# Echos "True" if the first argument is a valid log level +check_log_dest_override() { + case "$1" in + DEBUG|INFO|WARN|ERROR|CRITICAL) + echo True + ;; + esac +} + # Parse arguments and options parse_args() { case "$1" in @@ -196,23 +222,51 @@ parse_args() { shift parse_args "$@" ;; - --log-stdout) + -q|--quiet) shift - [ "${LOG_FILE}" = 'True' ] && parse_args 'invalid arg that triggers usage message' - export BIRDHOUSE_LOG_FD=1 # The argument here takes precedence over the env variable - LOG_STDOUT=True + if [ "$(check_log_dest_override "$1")" ]; then + export BIRDHOUSE_LOG_DEST_OVERRIDE="${BIRDHOUSE_LOG_DEST_OVERRIDE}:$1:quiet:" + shift + else + export BIRDHOUSE_LOG_QUIET=True # The argument here takes precedence over the env variable + fi parse_args "$@" ;; - -l=|--log-file=*) + -s|--log-stdout) + shift + if [ "$(check_log_dest_override "$1")" ]; then + export BIRDHOUSE_LOG_DEST_OVERRIDE="${BIRDHOUSE_LOG_DEST_OVERRIDE}:$1:fd:1" + shift + else + export BIRDHOUSE_LOG_FD=1 # The argument here takes precedence over the env variable + fi + parse_args "$@" + ;; + -l=*|--log-file=*) arg_value="${1#*=}" shift parse_args --log-file "${arg_value}" "$@" ;; -l|--log-file) shift - [ "${LOG_STDOUT}" = 'True' ] && parse_args 'invalid arg that triggers usage message' - export BIRDHOUSE_LOG_FILE=$(realpath "$1") # The argument here takes precedence over the env variable - LOG_FILE=True + # Note: cannot log to a file named DEBUG, INFO, WARN, ERROR, or CRITICAL + if [ "$(check_log_dest_override "$1")" ]; then + export BIRDHOUSE_LOG_DEST_OVERRIDE="${BIRDHOUSE_LOG_DEST_OVERRIDE}:$1:file:$(realpath -- "$2")" + shift + else + export BIRDHOUSE_LOG_FILE=$(realpath -- "$1") # The argument here takes precedence over the env variable + fi + shift + parse_args "$@" + ;; + -L=*|--log-level=*) + arg_value="${1#*=}" + shift + parse_args --log-level "${arg_value}" "$@" + ;; + -L|--log-level) + shift + export BIRDHOUSE_LOG_LEVEL="$1" # The argument here takes precedence over the env variable shift parse_args "$@" ;; diff --git a/birdhouse/scripts/logging.include.sh b/birdhouse/scripts/logging.include.sh index b64e0eabb..6129117bf 100644 --- a/birdhouse/scripts/logging.include.sh +++ b/birdhouse/scripts/logging.include.sh @@ -14,12 +14,12 @@ if [ "${BIRDHOUSE_COLOR}" -eq "1" ]; then fi BIRDHOUSE_LOG_LEVEL=${BIRDHOUSE_LOG_LEVEL:-INFO} -if [ "${__BIRDHOUSE_SUPPORTED_INTERFACE}" = 'True' ]; then - __DEFAULT_BIRDHOUSE_LOG_FD=2 -else - # logs were previously written to stdout +BIRDHOUSE_LOG_FD=${BIRDHOUSE_LOG_FD:-2} +if [ "${__BIRDHOUSE_SUPPORTED_INTERFACE}" != 'True' ]; then + # logs were previously written to stdout for DEBUG and INFO + # logs were previously intended to be written to stderr for WARN, ERROR, and CRITICAL # (this supports backwards compatible scripts that don't use the interface) - __DEFAULT_BIRDHOUSE_LOG_FD=1 + BIRDHOUSE_LOG_DEST_OVERRIDE=${BIRDHOUSE_LOG_DEST_OVERRIDE:-"DEBUG:fd:1:INFO:fd:1:WARN:fd:2:ERROR:fd:2:CRITICAL:fd:2"} fi export LOG_DEBUG="${GRAY}DEBUG${NORMAL}: " export LOG_INFO="${BLUE}INFO${NORMAL}: " @@ -27,62 +27,129 @@ export LOG_WARN="${YELLOW}WARNING${NORMAL}: " export LOG_ERROR="${RED}ERROR${NORMAL}: " export LOG_CRITICAL="${REG_BG_BOLD}CRITICAL${NORMAL}: " # to report misuse of functions -if [ -n "${BIRDHOUSE_LOG_FD}" ]; then - if [ -z "${BIRDHOUSE_LOG_FD##*[!0-9]*}" ]; then - echo "${LOG_CRITICAL}Invalid log file descriptor setting (not an integer): [BIRDHOUSE_LOG_FD=${BIRDHOUSE_LOG_FD}]." - exit 2 - fi -fi - +# Determines where to send log messages: +# - to the file descriptor set by BIRDHOUSE_LOG_FD, or if there is a "fd" option in BIRDHOUSE_LOG_DEST_OVERRIDE for the given log level +# - to the file set by BIRDHOUSE_LOG_FILE, or if there is a "file" option in BIRDHOUSE_LOG_DEST_OVERRIDE for the given log level +# - suppresses writing to a file descriptor if BIRDHOUSE_LOG_QUIET is "True" or if there is a "quiet" option in BIRDHOUSE_LOG_DEST_OVERRIDE for the given log level +# +# The BIRDHOUSE_LOG_DEST_OVERRIDE contains a ':' delimited string that determines how to override the log destination for specific log levels. +# BIRDHOUSE_LOG_DEST_OVERRIDE sections contain ':