diff --git a/Dockerfile b/Dockerfile index eb361552a76..99a69344024 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,6 @@ FROM ubuntu:focal as app RUN apt update && \ apt-get install -qy \ curl \ - vim \ - git-core \ language-pack-en \ build-essential \ python3.8-dev \ @@ -23,72 +21,28 @@ ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 ARG COMMON_APP_DIR="/edx/app" +ARG COMMON_CFG_DIR "/edx/etc" ARG DISCOVERY_SERVICE_NAME="discovery" -ENV DISCOVERY_HOME "${COMMON_APP_DIR}/${DISCOVERY_SERVICE_NAME}" +ARG DISCOVERY_HOME "${COMMON_APP_DIR}/${DISCOVERY_SERVICE_NAME}" ARG DISCOVERY_APP_DIR="${COMMON_APP_DIR}/${DISCOVERY_SERVICE_NAME}" -ARG SUPERVISOR_APP_DIR="${COMMON_APP_DIR}/supervisor" ARG DISCOVERY_VENV_DIR="${COMMON_APP_DIR}/${DISCOVERY_SERVICE_NAME}/venvs/${DISCOVERY_SERVICE_NAME}" -ARG SUPERVISOR_VENVS_DIR="${SUPERVISOR_APP_DIR}/venvs" -ARG SUPERVISOR_VENV_DIR="${SUPERVISOR_VENVS_DIR}/supervisor" ARG DISCOVERY_CODE_DIR="${DISCOVERY_APP_DIR}/${DISCOVERY_SERVICE_NAME}" -ARG DISCOVERY_NODEENV_DIR="${COMMON_APP_DIR}/${DISCOVERY_SERVICE_NAME}/nodeenvs/${DISCOVERY_SERVICE_NAME}" -ARG SUPERVISOR_AVAILABLE_DIR="${COMMON_APP_DIR}/supervisor/conf.available.d" -ARG SUPERVISOR_VENV_BIN="${SUPERVISOR_VENV_DIR}/bin" -ARG SUPEVISOR_CTL="${SUPERVISOR_VENV_BIN}/supervisorctl" -ARG SUPERVISOR_VERSION="4.2.1" -ARG SUPERVISOR_CFG_DIR="${SUPERVISOR_APP_DIR}/conf.d" -ARG DISCOVERY_NODE_VERSION="16.14.0" -ARG DISCOVERY_NPM_VERSION="8.5.x" - -# These variables were defined in Ansible configuration but I couldn't find them being used anywhere. -# I have commented these out for now but I would like to take opinion from someone having more knowledge about them -# and whether it is safe to comment them out. I did basic smoke testing and everything seems to be working fine. - -# ENV DISCOVERY_ECOMMERCE_API_URL 'https://localhost:8002/api/v2/' -# ENV DISCOVERY_COURSES_API_URL '${DISCOVERY_LMS_ROOT_URL}/api/courses/v1/' -# ENV DISCOVERY_ORGANIZATIONS_API_URL '${DISCOVERY_LMS_ROOT_URL}/api/organizations/v0/' -# ENV DISCOVERY_MARKETING_API_URL 'https://example.org/api/catalog/v2/' -# ENV DISCOVERY_MARKETING_URL_ROOT 'https://example.org/' - - -ENV HOME /root -ENV PATH "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin" -ENV PATH "${DISCOVERY_VENV_DIR}/bin:${DISCOVERY_NODEENV_DIR}/bin:$PATH" -ENV COMMON_CFG_DIR "/edx/etc" -ENV DISCOVERY_CFG_DIR "${COMMON_CFG_DIR}/discovery" -ENV DISCOVERY_CFG "/edx/etc/discovery.yml" - -ENV DISCOVERY_NODEENV_DIR "${DISCOVERY_HOME}/nodeenvs/${DISCOVERY_SERVICE_NAME}" -ENV DISCOVERY_NODEENV_BIN "${DISCOVERY_NODEENV_DIR}/bin" -ENV DISCOVERY_NODE_MODULES_DIR "${DISCOVERY_CODE_DIR}}/node_modules" -ENV DISCOVERY_NODE_BIN "${DISCOVERY_NODE_MODULES_DIR}/.bin" +ARG DISCOVERY_NODEENV_DIR="${DISCOVERY_HOME}/nodeenvs/${DISCOVERY_SERVICE_NAME}" -RUN addgroup discovery -RUN adduser --disabled-login --disabled-password discovery --ingroup discovery -# Make necessary directories and environment variables. -RUN mkdir -p /edx/var/discovery/staticfiles -RUN mkdir -p /edx/var/discovery/media -# Log dir -RUN mkdir /edx/var/log/ +ENV PATH "${DISCOVERY_VENV_DIR}/bin:${DISCOVERY_NODEENV_DIR}/bin:$PATH" +ENV DISCOVERY_CFG "/edx/etc/discovery.yml" +ENV DISCOVERY_CODE_DIR "${DISCOVERY_CODE_DIR}" +ENV DISCOVERY_HOME "${DISCOVERY_HOME}" RUN virtualenv -p python3.8 --always-copy ${DISCOVERY_VENV_DIR} -RUN virtualenv -p python3.8 --always-copy ${SUPERVISOR_VENV_DIR} # No need to activate discovery venv as it is already in path RUN pip install nodeenv -#install supervisor and deps in its virtualenv -RUN . ${SUPERVISOR_VENV_BIN}/activate && \ - pip install supervisor==${SUPERVISOR_VERSION} backoff==1.4.3 boto==2.48.0 && \ - deactivate +RUN nodeenv ${DISCOVERY_NODEENV_DIR} --node=16.14.0 --prebuilt +RUN npm install -g npm@8.5.x -RUN nodeenv ${DISCOVERY_NODEENV_DIR} --node=${DISCOVERY_NODE_VERSION} --prebuilt -RUN npm install -g npm@${DISCOVERY_NPM_VERSION} - -COPY requirements/production.txt ${DISCOVERY_CODE_DIR}/requirements/production.txt - -RUN pip install -r ${DISCOVERY_CODE_DIR}/requirements/production.txt # Working directory will be root of repo. WORKDIR ${DISCOVERY_CODE_DIR} @@ -96,24 +50,14 @@ WORKDIR ${DISCOVERY_CODE_DIR} # Copy just JS requirements and install them. COPY package.json package.json COPY package-lock.json package-lock.json -RUN npm install --production COPY bower.json bower.json +RUN npm install --production RUN ./node_modules/.bin/bower install --allow-root --production # Copy over rest of code. # We do this AFTER requirements so that the requirements cache isn't busted # every time any bit of code is changed. COPY . . -COPY /configuration_files/discovery_gunicorn.py ${DISCOVERY_HOME}/discovery_gunicorn.py -COPY /configuration_files/discovery-workers.sh ${DISCOVERY_HOME}/discovery-workers.sh -COPY /configuration_files/discovery.yml ${DISCOVERY_CFG} -COPY /scripts/discovery.sh ${DISCOVERY_HOME}/discovery.sh -# create supervisor job -COPY /configuration_files/supervisor.service /etc/systemd/system/supervisor.service -COPY /configuration_files/supervisor.conf ${SUPERVISOR_CFG_DIR}/supervisor.conf -COPY /configuration_files/supervisorctl ${SUPERVISOR_VENV_BIN}/supervisorctl -# Manage.py symlink -COPY /manage.py /edx/bin/manage.discovery # Expose canonical Discovery port EXPOSE 18381 @@ -122,7 +66,9 @@ FROM app as prod ENV DJANGO_SETTINGS_MODULE "course_discovery.settings.production" -RUN make static +COPY requirements/production.txt ${DISCOVERY_CODE_DIR}/requirements/production.txt + +RUN pip install -r ${DISCOVERY_CODE_DIR}/requirements/production.txt ENTRYPOINT ["/edx/app/discovery/discovery.sh"] @@ -132,12 +78,5 @@ ENV DJANGO_SETTINGS_MODULE "course_discovery.settings.devstack" RUN pip install -r ${DISCOVERY_CODE_DIR}/requirements/local.txt -COPY /scripts/devstack.sh ${DISCOVERY_HOME}/devstack.sh - -RUN chown discovery:discovery /edx/app/discovery/devstack.sh && chmod a+x /edx/app/discovery/devstack.sh - # Devstack related step for backwards compatibility -RUN touch /edx/app/${DISCOVERY_SERVICE_NAME}/${DISCOVERY_SERVICE_NAME}_env - -ENTRYPOINT ["/edx/app/discovery/devstack.sh"] -CMD ["start"] +RUN touch ${DISCOVERY_HOME}/${DISCOVERY_SERVICE_NAME}_env diff --git a/configuration_files/discovery-workers.sh b/configuration_files/discovery-workers.sh deleted file mode 100644 index 7f197b728f9..00000000000 --- a/configuration_files/discovery-workers.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - - -source /edx/app/discovery/discovery_env - -# We exec so that celery is the child of supervisor and can be managed properly -exec /edx/app/discovery/venvs/discovery/bin/celery $@ diff --git a/configuration_files/discovery_gunicorn.py b/configuration_files/discovery_gunicorn.py deleted file mode 100644 index 82ea4832d56..00000000000 --- a/configuration_files/discovery_gunicorn.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html -""" - -timeout = 300 -bind = "127.0.0.1:8381" -pythonpath = "/edx/app/discovery/discovery" -workers = 2 -worker_class = "gevent" - -limit_request_field_size = 16384 - -def pre_request(worker, req): - worker.log.info("%s %s" % (req.method, req.path)) diff --git a/configuration_files/supervisor.conf b/configuration_files/supervisor.conf deleted file mode 100644 index 59a864b22ee..00000000000 --- a/configuration_files/supervisor.conf +++ /dev/null @@ -1,13 +0,0 @@ -[program:nginx] -command=nginx -g 'daemon off;' -killasgroup=true -stopasgroup=true - -[program:discovery] -command=/edx/app/discovery/discovery.sh -user=www-data -directory=/edx/app/discovery/discovery -stdout_logfile=/edx/var/log/supervisor/%(program_name)s-stdout.log -stderr_logfile=/edx/var/log/supervisor/%(program_name)s-stderr.log -killasgroup=true -stopasgroup=true diff --git a/configuration_files/supervisor.service b/configuration_files/supervisor.service deleted file mode 100644 index c73ddb5212b..00000000000 --- a/configuration_files/supervisor.service +++ /dev/null @@ -1,29 +0,0 @@ -[Unit] -Description=supervisord - Supervisor process control system -Documentation=http://supervisord.org -After=network.target - - -[Service] - -# User will be applied only to ExecStart, not other commands (i.e. ExecStartPre) -# This is needed because pre_supervisor needs to write to supervisor/conf.d, which -# supervisor_service_user does not have permission to do. -PermissionsStartOnly=true -User=www-data - -Type=forking -TimeoutSec=432000 - -ExecStart=/edx/app/supervisor/venvs/supervisor/bin/supervisord --configuration /edx/app/supervisor/supervisord.conf -ExecReload=/edx/app/supervisor/venvs/supervisor/bin/supervisorctl reload -ExecStop=/edx/app/supervisor/venvs/supervisor/bin/supervisorctl shutdown - -# Trust supervisor to kill all its children -# Otherwise systemd will see that ExecStop ^ comes back synchronously and say "Oh, I can kill everyone in this cgroup" -# https://www.freedesktop.org/software/systemd/man/systemd.service.html#ExecStop= -# https://www.freedesktop.org/software/systemd/man/systemd.kill.html -KillMode=none - -[Install] -WantedBy=multi-user.target diff --git a/configuration_files/supervisorctl b/configuration_files/supervisorctl deleted file mode 100644 index 47225cdf7a3..00000000000 --- a/configuration_files/supervisorctl +++ /dev/null @@ -1,10 +0,0 @@ -#!/edx/app/supervisor/venvs/supervisor/bin/python -# -*- coding: utf-8 -*- -import re -import sys - -from supervisor.supervisorctl import main - -if __name__ == '__main__': - sys.exit(main()) - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) diff --git a/scripts/devstack.sh b/scripts/devstack.sh deleted file mode 100644 index 1e57b3745ae..00000000000 --- a/scripts/devstack.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - - -source /edx/app/discovery/discovery_env -COMMAND=$1 - -case $COMMAND in - start) - /edx/app/supervisor/venvs/supervisor/bin/supervisord -n --configuration /edx/app/supervisor/supervisord.conf - ;; - open) - . /edx/app/discovery/nodeenvs/discovery/bin/activate - . /edx/app/discovery/venvs/discovery/bin/activate - cd /edx/app/discovery/discovery - - /bin/bash - ;; - exec) - shift - - . /edx/app/discovery/nodeenvs/discovery/bin/activate - . /edx/app/discovery/venvs/discovery/bin/activate - cd /edx/app/discovery/discovery - - "$@" - ;; - *) - "$@" - ;; -esac diff --git a/scripts/discovery.sh b/scripts/discovery.sh deleted file mode 100644 index dd21db36303..00000000000 --- a/scripts/discovery.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - - -export EDX_REST_API_CLIENT_NAME="default_env-default_deployment-discovery" - -exec /edx/app/discovery/venvs/discovery/bin/gunicorn -c /edx/app/discovery/discovery_gunicorn.py --reload course_discovery.wsgi:application