feat: Add Native Docker config

.dockerignore

@@ -1,2 +1,3 @@
 .dev/
 htmlcov/
+Dockerfile

Dockerfile

@@ -1,68 +1,144 @@
 FROM ubuntu:focal as app
 
 # System requirements.
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && \
-	apt-get upgrade -qy
-RUN apt-get install --yes \
-	git \
-	language-pack-en \
-	python3-venv \
-	python3.8-dev \
-	python3.8-venv \
-	build-essential \
-	libffi-dev \
-	libmysqlclient-dev \
-	libxml2-dev \
-	libxslt1-dev \
-	libjpeg-dev \
-	libssl-dev
-
-RUN rm -rf /var/lib/apt/lists/*
-
-ENV VIRTUAL_ENV=/venv
-RUN python3.8 -m venv $VIRTUAL_ENV
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-
-RUN pip install pip==20.2.3 setuptools==50.3.0 nodeenv
+RUN apt update && \
+  apt-get install -qy \ 
+  curl \
+  vim \
+  git-core \
+  language-pack-en \
+  build-essential \
+  python3.8-dev \
+  python3-virtualenv \
+  python3.8-distutils \
+  libmysqlclient-dev \
+  libssl-dev && \
+  rm -rf /var/lib/apt/lists/*
 
 # Use UTF-8.
 RUN locale-gen en_US.UTF-8
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
+ARG COMMON_APP_DIR="/edx/app"
+ARG DISCOVERY_SERVICE_NAME="discovery"
+ENV DISCOVERY_HOME "${COMMON_APP_DIR}/${DISCOVERY_SERVICE_NAME}"
+ARG DISCOVERY_APP_DIR="${COMMON_APP_DIR}/${DISCOVERY_SERVICE_NAME}"
+ARG SUPERVISOR_APP_DIR="${COMMON_APP_DIR}/supervisor"
+ARG DISCOVERY_VENV_DIR="${COMMON_APP_DIR}/${DISCOVERY_SERVICE_NAME}/venvs/${DISCOVERY_SERVICE_NAME}"
+ARG SUPERVISOR_VENVS_DIR="${SUPERVISOR_APP_DIR}/venvs"
+ARG SUPERVISOR_VENV_DIR="${SUPERVISOR_VENVS_DIR}/supervisor"
+ARG DISCOVERY_CODE_DIR="${DISCOVERY_APP_DIR}/${DISCOVERY_SERVICE_NAME}"
+ARG DISCOVERY_NODEENV_DIR="${COMMON_APP_DIR}/${DISCOVERY_SERVICE_NAME}/nodeenvs/${DISCOVERY_SERVICE_NAME}"
+ARG SUPERVISOR_AVAILABLE_DIR="${COMMON_APP_DIR}/supervisor/conf.available.d"
+ARG SUPERVISOR_VENV_BIN="${SUPERVISOR_VENV_DIR}/bin"
+ARG SUPEVISOR_CTL="${SUPERVISOR_VENV_BIN}/supervisorctl"
+ARG SUPERVISOR_VERSION="4.2.1"
+ARG SUPERVISOR_CFG_DIR="${SUPERVISOR_APP_DIR}/conf.d"
+ARG DISCOVERY_NODE_VERSION="16.14.0"
+ARG DISCOVERY_NPM_VERSION="8.5.x"
+
+# These variables were defined in Ansible configuration but I couldn't find them being used anywhere.
+# I have commented these out for now but I would like to take opinion from someone having more knowledge about them
+# and whether it is safe to comment them out. I did basic smoke testing and everything seems to be working fine.
+
+# ENV DISCOVERY_ECOMMERCE_API_URL 'https://localhost:8002/api/v2/'
+# ENV DISCOVERY_COURSES_API_URL '${DISCOVERY_LMS_ROOT_URL}/api/courses/v1/'
+# ENV DISCOVERY_ORGANIZATIONS_API_URL '${DISCOVERY_LMS_ROOT_URL}/api/organizations/v0/'
+# ENV DISCOVERY_MARKETING_API_URL 'https://example.org/api/catalog/v2/'
+# ENV DISCOVERY_MARKETING_URL_ROOT 'https://example.org/'
+
+
+ENV HOME /root
+ENV PATH "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"
+ENV PATH "${DISCOVERY_VENV_DIR}/bin:${DISCOVERY_NODEENV_DIR}/bin:$PATH"
+ENV COMMON_CFG_DIR "/edx/etc"
+ENV DISCOVERY_CFG_DIR "${COMMON_CFG_DIR}/discovery"
+ENV DISCOVERY_CFG "/edx/etc/discovery.yml"
+
+ENV DISCOVERY_NODEENV_DIR "${DISCOVERY_HOME}/nodeenvs/${DISCOVERY_SERVICE_NAME}"
+ENV DISCOVERY_NODEENV_BIN "${DISCOVERY_NODEENV_DIR}/bin"
+ENV DISCOVERY_NODE_MODULES_DIR "${DISCOVERY_CODE_DIR}}/node_modules"
+ENV DISCOVERY_NODE_BIN "${DISCOVERY_NODE_MODULES_DIR}/.bin"
+
+RUN addgroup discovery
+RUN adduser --disabled-login --disabled-password discovery --ingroup discovery
+
 # Make necessary directories and environment variables.
 RUN mkdir -p /edx/var/discovery/staticfiles
 RUN mkdir -p /edx/var/discovery/media
-ENV DJANGO_SETTINGS_MODULE course_discovery.settings.production
+# Log dir
+RUN mkdir /edx/var/log/
+
+RUN virtualenv -p python3.8 --always-copy ${DISCOVERY_VENV_DIR}
+RUN virtualenv -p python3.8 --always-copy ${SUPERVISOR_VENV_DIR}
+
+# No need to activate discovery venv as it is already in path
+RUN pip install nodeenv
+
+#install supervisor and deps in its virtualenv
+RUN . ${SUPERVISOR_VENV_BIN}/activate && \
+  pip install supervisor==${SUPERVISOR_VERSION} backoff==1.4.3 boto==2.48.0 && \
+  deactivate
+
+RUN nodeenv ${DISCOVERY_NODEENV_DIR} --node=${DISCOVERY_NODE_VERSION} --prebuilt
+RUN npm install -g npm@${DISCOVERY_NPM_VERSION}
+
+COPY requirements/production.txt ${DISCOVERY_CODE_DIR}/requirements/production.txt
+
+RUN pip install -r ${DISCOVERY_CODE_DIR}/requirements/production.txt
 
 # Working directory will be root of repo.
-WORKDIR /edx/app/discovery
+WORKDIR ${DISCOVERY_CODE_DIR}
 
 # Copy just JS requirements and install them.
 COPY package.json package.json
 COPY package-lock.json package-lock.json
-RUN nodeenv /edx/app/nodeenv --node=16.14.2 --npm=8.5.x --prebuilt
-ENV PATH /edx/app/nodeenv/bin:${PATH}
 RUN npm install --production
 COPY bower.json bower.json
 RUN ./node_modules/.bin/bower install --allow-root --production
 
-# Copy just Python requirements & install them.
-COPY requirements/ requirements/
-RUN pip install -r requirements/production.txt
-
 # Copy over rest of code.
 # We do this AFTER requirements so that the requirements cache isn't busted
 # every time any bit of code is changed.
 COPY . .
+COPY /configuration_files/discovery_gunicorn.py ${DISCOVERY_HOME}/discovery_gunicorn.py
+# deleted this file completely and defined the env variables in dockerfile's respective target images.
+# COPY configuration_files/discovery_env ${DISCOVERY_HOME}/discovery_env
+COPY /configuration_files/discovery-workers.sh ${DISCOVERY_HOME}/discovery-workers.sh
+COPY /configuration_files/discovery.yml ${DISCOVERY_CFG}
+COPY /scripts/discovery.sh ${DISCOVERY_HOME}/discovery.sh
+# create supervisor job
+COPY /configuration_files/supervisor.service /etc/systemd/system/supervisor.service
+COPY /configuration_files/supervisor.conf ${SUPERVISOR_CFG_DIR}/supervisor.conf
+COPY /configuration_files/supervisorctl ${SUPERVISOR_VENV_BIN}/supervisorctl
+# Manage.py symlink
+COPY /manage.py /edx/bin/manage.discovery
 
 # Expose canonical Discovery port
-EXPOSE 8381
+EXPOSE 18381
+
+FROM app as prod
+
+ENV DJANGO_SETTINGS_MODULE "course_discovery.settings.production"
+
+RUN make static
+
+ENTRYPOINT ["/edx/app/discovery/discovery.sh"]
+
+FROM app as dev
+
+ENV DJANGO_SETTINGS_MODULE "course_discovery.settings.devstack"
+
+RUN pip install -r ${DISCOVERY_CODE_DIR}/requirements/local.txt
+
+COPY /scripts/devstack.sh ${DISCOVERY_HOME}/devstack.sh
+
+RUN chown discovery:discovery /edx/app/discovery/devstack.sh && chmod a+x /edx/app/discovery/devstack.sh
 
-CMD gunicorn --bind=0.0.0.0:8381 --workers 2 --max-requests=1000 -c course_discovery/docker_gunicorn_configuration.py course_discovery.wsgi:application
+# Devstack related step for backwards compatibility
+RUN touch /edx/app/${DISCOVERY_SERVICE_NAME}/${DISCOVERY_SERVICE_NAME}_env
 
-FROM app as newrelic
-RUN pip install newrelic
-CMD newrelic-admin run-program gunicorn --bind=0.0.0.0:8381 --workers 2 --max-requests=1000 -c course_discovery/docker_gunicorn_configuration.py course_discovery.wsgi:application
+ENTRYPOINT ["/edx/app/discovery/devstack.sh"]
+CMD ["start"]

configuration_files/discovery-workers.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+# Ansible managed
+
+source /edx/app/discovery/discovery_env
+
+# We exec so that celery is the child of supervisor and can be managed properly
+exec /edx/app/discovery/venvs/discovery/bin/celery $@

configuration_files/discovery.yml

@@ -0,0 +1,95 @@
+---
+# Ansible managed
+
+
+API_ROOT: null
+AWS_SES_REGION_ENDPOINT: email.us-east-1.amazonaws.com
+AWS_SES_REGION_NAME: us-east-1
+BACKEND_SERVICE_EDX_OAUTH2_KEY: discovery-backend-service-key
+BACKEND_SERVICE_EDX_OAUTH2_PROVIDER_URL: http://localhost:18000/oauth2
+BACKEND_SERVICE_EDX_OAUTH2_SECRET: discovery-backend-service-secret
+CACHES:
+    default:
+        BACKEND: django.core.cache.backends.memcached.MemcachedCache
+        KEY_PREFIX: discovery
+        LOCATION:
+        - edx.devstack.memcached:11211
+CELERY_BROKER_URL: redis://:@127.0.0.1:6379/
+CORS_ORIGIN_WHITELIST: []
+CSRF_COOKIE_SECURE: false
+DATABASES:
+    default:
+        ATOMIC_REQUESTS: 'false'
+        CONN_MAX_AGE: 60
+        ENGINE: django.db.backends.mysql
+        HOST: edx.devstack.mysql
+        NAME: discovery
+        OPTIONS:
+            connect_timeout: 10
+            init_command: SET sql_mode='STRICT_TRANS_TABLES'
+        PASSWORD: password
+        PORT: 3306
+        USER: discov001
+    read_replica:
+        ATOMIC_REQUESTS: 'false'
+        CONN_MAX_AGE: 60
+        ENGINE: django.db.backends.mysql
+        HOST: edx.devstack.mysql
+        NAME: discovery
+        OPTIONS:
+            connect_timeout: 10
+            init_command: SET sql_mode='STRICT_TRANS_TABLES'
+        PASSWORD: password
+        PORT: 3306
+        USER: discov001
+DEFAULT_PARTNER_ID: 1
+EDX_DRF_EXTENSIONS:
+    OAUTH2_USER_INFO_URL: http://127.0.0.1:8000/user_info
+ELASTICSEARCH_CLUSTER_URL: http://127.0.0.1:9200/
+ELASTICSEARCH_INDEX_NAME: catalog
+EMAIL_BACKEND: django_ses.SESBackend
+EMAIL_HOST: localhost
+EMAIL_HOST_PASSWORD: ''
+EMAIL_HOST_USER: ''
+EMAIL_PORT: 25
+EMAIL_USE_TLS: false
+ENABLE_PUBLISHER: false
+EXTRA_APPS:
+- course_discovery.apps.edx_catalog_extensions
+JWT_AUTH:
+    JWT_AUTH_COOKIE_HEADER_PAYLOAD: edx-jwt-cookie-header-payload
+    JWT_AUTH_COOKIE_SIGNATURE: edx-jwt-cookie-signature
+    JWT_AUTH_REFRESH_COOKIE: edx-jwt-refresh-cookie
+    JWT_ISSUERS:
+    -   AUDIENCE: lms-key
+        ISSUER: http://edx.devstack.lms:18000/oauth2
+        SECRET_KEY: lms-secret
+    JWT_PUBLIC_SIGNING_JWK_SET: ''
+LANGUAGE_CODE: en
+MEDIA_STORAGE_BACKEND:
+    DEFAULT_FILE_STORAGE: django.core.files.storage.FileSystemStorage
+    MEDIA_ROOT: /edx/var/discovery/media
+    MEDIA_URL: /media/
+OPENEXCHANGERATES_API_KEY: ''
+PARLER_DEFAULT_LANGUAGE_CODE: en
+PARLER_LANGUAGES:
+    1:
+    -   code: en
+    default:
+        fallbacks:
+        - en
+        hide_untranslated: 'False'
+PLATFORM_NAME: Your Platform Name Here
+PUBLISHER_FROM_EMAIL: null
+SECRET_KEY: Your secret key here
+SESSION_EXPIRE_AT_BROWSER_CLOSE: false
+SOCIAL_AUTH_EDX_OAUTH2_ISSUER: http://127.0.0.1:8000
+SOCIAL_AUTH_EDX_OAUTH2_KEY: discovery-sso-key
+SOCIAL_AUTH_EDX_OAUTH2_LOGOUT_URL: http://localhost:18000/logout
+SOCIAL_AUTH_EDX_OAUTH2_SECRET: discovery-sso-secret
+SOCIAL_AUTH_EDX_OAUTH2_URL_ROOT: http://127.0.0.1:8000
+SOCIAL_AUTH_REDIRECT_IS_HTTPS: false
+STATICFILES_STORAGE: django.contrib.staticfiles.storage.StaticFilesStorage
+STATIC_ROOT: /edx/var/discovery/staticfiles
+TIME_ZONE: UTC
+USERNAME_REPLACEMENT_WORKER: OVERRIDE THIS WITH A VALID USERNAME

configuration_files/discovery_gunicorn.py

@@ -0,0 +1,18 @@
+"""
+gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html
+Ansible managed
+"""
+
+timeout = 300
+bind = "127.0.0.1:8381"
+pythonpath = "/edx/app/discovery/discovery"
+workers = 2
+worker_class = "gevent"
+
+limit_request_field_size = 16384
+
+
+
+
+def pre_request(worker, req):
+    worker.log.info("%s %s" % (req.method, req.path))

configuration_files/supervisor.conf

@@ -0,0 +1,17 @@
+#
+# Ansible managed
+#
+
+[program:nginx]
+command=nginx -g 'daemon off;'
+killasgroup=true
+stopasgroup=true
+
+[program:discovery]
+command=/edx/app/discovery/discovery.sh
+user=www-data
+directory=/edx/app/discovery/discovery
+stdout_logfile=/edx/var/log/supervisor/%(program_name)s-stdout.log
+stderr_logfile=/edx/var/log/supervisor/%(program_name)s-stderr.log
+killasgroup=true
+stopasgroup=true

configuration_files/supervisor.service

@@ -0,0 +1,29 @@
+[Unit]
+Description=supervisord - Supervisor process control system
+Documentation=http://supervisord.org
+After=network.target
+
+
+[Service]
+
+# User will be applied only to ExecStart, not other commands (i.e. ExecStartPre)
+# This is needed because pre_supervisor needs to write to supervisor/conf.d, which
+# supervisor_service_user does not have permission to do.
+PermissionsStartOnly=true
+User=www-data
+
+Type=forking
+TimeoutSec=432000
+
+ExecStart=/edx/app/supervisor/venvs/supervisor/bin/supervisord --configuration /edx/app/supervisor/supervisord.conf
+ExecReload=/edx/app/supervisor/venvs/supervisor/bin/supervisorctl reload
+ExecStop=/edx/app/supervisor/venvs/supervisor/bin/supervisorctl shutdown
+
+# Trust supervisor to kill all its children
+# Otherwise systemd will see that ExecStop ^ comes back synchronously and say "Oh, I can kill everyone in this cgroup"
+# https://www.freedesktop.org/software/systemd/man/systemd.service.html#ExecStop=
+# https://www.freedesktop.org/software/systemd/man/systemd.kill.html
+KillMode=none
+
+[Install]
+WantedBy=multi-user.target

configuration_files/supervisorctl

@@ -0,0 +1,10 @@
+#!/edx/app/supervisor/venvs/supervisor/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+
+from supervisor.supervisorctl import main
+
+if __name__ == '__main__':
+    sys.exit(main())
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])