From 09c7fde231a0a6fdd8008d3a510277b052ecfcca Mon Sep 17 00:00:00 2001 From: Hunia Fatima Date: Tue, 19 Nov 2024 15:52:19 +0500 Subject: [PATCH] chore: add enterprise catalog in devstack --- .github/workflows/provisioning-tests.yml | 2 +- docker-compose-host.yml | 11 ++- docker-compose.yml | 76 +++++++++++++++++++ docs/service_list.rst | 3 + options.mk | 2 +- provision-enterprise-catalog.sh | 20 +++++ provision-mysql80.sql | 4 + provision.sh | 3 +- provision.sql | 2 + ...cs_dashboard.py => analytics_dashboard.py} | 0 repo.sh | 2 + requirements/dev.txt | 8 +- requirements/doc.txt | 6 +- requirements/pip-tools.txt | 4 +- requirements/pip.txt | 4 +- requirements/test.txt | 4 +- 16 files changed, 134 insertions(+), 17 deletions(-) create mode 100755 provision-enterprise-catalog.sh rename py_configuration_files/{anayltics_dashboard.py => analytics_dashboard.py} (100%) diff --git a/.github/workflows/provisioning-tests.yml b/.github/workflows/provisioning-tests.yml index 5af1594975..d8fc8ce00f 100644 --- a/.github/workflows/provisioning-tests.yml +++ b/.github/workflows/provisioning-tests.yml @@ -30,7 +30,7 @@ jobs: os: - ubuntu-20.04 # Ubuntu 20.04 "Focal Fossa" python-version: [ '3.11' ] - services: [ discovery+lms+forum ,registrar+lms, ecommerce+lms, edx_notes_api+lms, credentials+lms, xqueue, analyticsapi+insights+lms] + services: [ discovery+lms+forum ,registrar+lms, ecommerce+lms, edx_notes_api+lms, credentials+lms, xqueue, analyticsapi+insights+lms, enterprise-catalog+lms] fail-fast: false # some services can be flaky; let others run to completion even if one fails steps: diff --git a/docker-compose-host.yml b/docker-compose-host.yml index 85f7a2a272..ee11a203cd 100644 --- a/docker-compose-host.yml +++ b/docker-compose-host.yml @@ -71,7 +71,16 @@ services: - ${DEVSTACK_WORKSPACE}/edx-analytics-data-api:/edx/app/analytics_api/analytics_api - ${DEVSTACK_WORKSPACE}/src:/edx/src - ${PWD}/py_configuration_files/analytics_data_api.py:/edx/app/analytics_api/analytics_api/analyticsdataserver/settings/devstack.py - + enterprise-catalog: + volumes: + - ${DEVSTACK_WORKSPACE}/enterprise-catalog:/edx/app/enterprise_catalog/enterprise_catalog + enterprise-catalog-worker: + volumes: + - ${DEVSTACK_WORKSPACE}/enterprise-catalog:/edx/app/enterprise_catalog/enterprise_catalog + - ${PWD}/py_configuration_files/enterprise_catalog.py:/edx/app/enterprise_catalog/enterprise_catalog/settings/devstack.py + enterprise-catalog-curations-worker: + volumes: + - ${DEVSTACK_WORKSPACE}/enterprise-catalog:/edx/app/enterprise_catalog/enterprise_catalog # Note that frontends mount `src` to /edx/app/src instead of /edx/src. # See ADR #5 for rationale. frontend-app-account: diff --git a/docker-compose.yml b/docker-compose.yml index a165a6bf22..85dbf7f863 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -693,6 +693,82 @@ services: aliases: - edx.devstack.xqueue_consumer + enterprise-catalog: + image: edxops/enterprise-catalog-dev + container_name: edx.devstack.enterprise-catalog + hostname: enterprise-catalog.devstack.edx + command: bash -c 'while true; do python /edx/app/enterprise_catalog/enterprise_catalog/manage.py runserver 0.0.0.0:18160; sleep 2; done' + ports: + - "18160:18160" + depends_on: + - memcached + - mysql80 + - enterprise-catalog-worker + networks: + default: + aliases: + - edx.devstack.enterprise-catalog + # Allows attachment to this container using 'docker attach '. + stdin_open: true + tty: true + environment: + CELERY_ALWAYS_EAGER: 'false' + CELERY_BROKER_TRANSPORT: redis + CELERY_BROKER_HOSTNAME: edx.devstack.redis:6379 + CELERY_BROKER_VHOST: 0 + CELERY_BROKER_PASSWORD: password + DJANGO_SETTINGS_MODULE: enterprise_catalog.settings.devstack + ENABLE_DJANGO_TOOLBAR: 1 + DB_HOST: edx.devstack.mysql80 + DB_NAME: enterprise_catalog + DB_PORT: 3306 + DB_USER: catalog001 + DB_PASSWORD: 'password' + + enterprise-catalog-worker: + image: edxops/enterprise-catalog-dev + command: bash -c 'cd /edx/app/enterprise_catalog/enterprise_catalog && celery -A enterprise_catalog worker -Q enterprise_catalog.default -l DEBUG' + container_name: edx.devstack.enterprise.catalog.worker + depends_on: + - mysql80 + environment: + CELERY_ALWAYS_EAGER: 'false' + CELERY_BROKER_TRANSPORT: redis + CELERY_BROKER_HOSTNAME: edx.devstack.redis:6379 + CELERY_BROKER_VHOST: 0 + CELERY_BROKER_PASSWORD: password + DJANGO_SETTINGS_MODULE: enterprise_catalog.settings.devstack + COLUMNS: 80 + hostname: worker.catalog.enterprise + ports: + - "18161:18161" + restart: always + # Allows attachment to this container using 'docker attach '. + stdin_open: true + tty: true + + enterprise-catalog-curations-worker: + image: edxops/enterprise-catalog-dev + command: bash -c 'cd /edx/app/enterprise_catalog/enterprise_catalog && celery -A enterprise_catalog worker -Q enterprise_catalog.curations -l DEBUG' + container_name: enterprise.catalog.curations + depends_on: + - mysql80 + environment: + CELERY_ALWAYS_EAGER: 'false' + CELERY_BROKER_TRANSPORT: redis + CELERY_BROKER_HOSTNAME: edx.devstack.redis:6379 + CELERY_BROKER_VHOST: 0 + CELERY_BROKER_PASSWORD: password + DJANGO_SETTINGS_MODULE: enterprise_catalog.settings.devstack + COLUMNS: 80 + hostname: curations.catalog.enterprise + ports: + - "18162:18162" + restart: always + # Allows attachment to this container using 'docker attach '. + stdin_open: true + tty: true + # ========================================================================== # edX Microfrontends # diff --git a/docs/service_list.rst b/docs/service_list.rst index 092b8028c6..fd750f117a 100644 --- a/docs/service_list.rst +++ b/docs/service_list.rst @@ -63,6 +63,8 @@ Instead of a service name or list, you can also run commands like ``make dev.pro +------------------------------------+-------------------------------------+----------------+--------------+ | `frontend-app-ora-grading`_ | http://localhost:1993 | MFE (React.js) | Extra | +------------------------------------+-------------------------------------+----------------+--------------+ +| `enterprise-catalog`_ | http://localhost:18160/ | Python/Django | Extra | ++------------------------------------+-------------------------------------+----------------+--------------+ Some common service combinations include: @@ -95,3 +97,4 @@ Some common service combinations include: .. _frontend-app-ora-grading: https://github.com/edx/frontend-app-ora-grading .. _insights: https://github.com/edx/edx-analytics-dashboard .. _analyticsapi: https://github.com/edx/edx-analytics-data-api +.. _enterprise-catalog: https://github.com/openedx/enterprise-catalog diff --git a/options.mk b/options.mk index 5a4d9b4e2c..ad7449435b 100644 --- a/options.mk +++ b/options.mk @@ -67,7 +67,7 @@ credentials+discovery+ecommerce+edx_notes_api+forum+frontend-app-authn+frontend- # Separated by plus signs. # Separated by plus signs. Listed in alphabetical order for clarity. EDX_SERVICES ?= \ -analyticsapi+credentials+cms+cms-worker+cms_watcher+discovery+ecommerce+edx_notes_api+forum+frontend-app-account+frontend-app-learner-dashboard+frontend-app-learner-record+frontend-app-profile+frontend-app-authn+frontend-app-course-authoring+frontend-app-gradebook+frontend-app-ora-grading+frontend-app-learning+frontend-app-library-authoring+frontend-app-payment+frontend-app-program-console+frontend-app-publisher+insights+lms+lms-worker+lms_watcher+registrar+registrar-worker+xqueue+xqueue_consumer +analyticsapi+credentials+cms+cms-worker+cms_watcher+discovery+ecommerce+edx_notes_api+forum+frontend-app-account+frontend-app-learner-dashboard+frontend-app-learner-record+frontend-app-profile+frontend-app-authn+frontend-app-course-authoring+frontend-app-gradebook+frontend-app-ora-grading+frontend-app-learning+frontend-app-library-authoring+frontend-app-payment+frontend-app-program-console+frontend-app-publisher+insights+lms+lms-worker+lms_watcher+registrar+registrar-worker+xqueue+xqueue_consumer+enterprise-catalog # Services with database migrations. # Should be a subset of $(EDX_SERVICES). diff --git a/provision-enterprise-catalog.sh b/provision-enterprise-catalog.sh new file mode 100755 index 0000000000..76d7eac221 --- /dev/null +++ b/provision-enterprise-catalog.sh @@ -0,0 +1,20 @@ +name="enterprise-catalog" +port="18160" + +docker compose up -d $name + +# Run migrations +echo -e "${GREEN}Running migrations for ${name}...${NC}" +docker compose exec -T ${name} bash -c "cd /edx/app/${name}/${name}/ && make migrate" + +echo -e "${GREEN}Installing requirements for ${name}...${NC}" +docker compose exec -T ${name} bash -e -c 'cd /edx/app/enterprise-catalog/ && make requirements' -- "$name" + +# Create superuser +echo -e "${GREEN}Creating super-user for ${name}...${NC}" +docker compose exec -T ${name} bash -c "echo 'from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None' | python /edx/app/${name}/${name}/manage.py shell" + +./provision-ida-user.sh ${name} ${name} ${port} + +# Restart enterprise.catalog app and worker containers +docker compose restart enterprise-catalog enterprise-catalog-worker diff --git a/provision-mysql80.sql b/provision-mysql80.sql index 19b32e0ebd..be903f1d7f 100644 --- a/provision-mysql80.sql +++ b/provision-mysql80.sql @@ -6,6 +6,10 @@ CREATE DATABASE IF NOT EXISTS credentials; CREATE USER IF NOT EXISTS 'credentials001'@'%' IDENTIFIED BY 'password'; GRANT ALL ON credentials.* TO 'credentials001'@'%'; +CREATE DATABASE IF NOT EXISTS enterprise_catalog; +CREATE USER IF NOT EXISTS 'catalog001'@'%' IDENTIFIED BY 'password'; +GRANT ALL ON enterprise_catalog.* TO 'catalog001'@'%'; + CREATE DATABASE IF NOT EXISTS discovery; CREATE USER IF NOT EXISTS 'discov001'@'%' IDENTIFIED BY 'password'; GRANT ALL ON discovery.* TO 'discov001'@'%'; diff --git a/provision.sh b/provision.sh index cde27b6fcd..551349ed68 100755 --- a/provision.sh +++ b/provision.sh @@ -3,7 +3,7 @@ # This script will provision the services specified in the argument list, # or all services if no arguments are provided. # -# Non-existant services will be ignored. +# Non-existent services will be ignored. # Specifying services more than once will cause them to be provisioned more # than once. # @@ -49,6 +49,7 @@ xqueue \ coursegraph \ insights \ analyticsapi \ +enterprise-catalog \ " # What should we provision? diff --git a/provision.sql b/provision.sql index 0b672c9986..94c3fd48c0 100644 --- a/provision.sql +++ b/provision.sql @@ -33,5 +33,7 @@ GRANT ALL ON `reports`.* TO 'analytics001'@'%' IDENTIFIED BY 'password'; CREATE DATABASE IF NOT EXISTS `reports_v1`; GRANT ALL ON `reports_v1`.* TO 'analytics001'@'%' IDENTIFIED BY 'password'; +CREATE DATABASE IF NOT EXISTS `enterprise_catalog`; +GRANT ALL ON `enterprise_catalog`.* TO 'catalog001'@'%' IDENTIFIED BY 'password'; FLUSH PRIVILEGES; diff --git a/py_configuration_files/anayltics_dashboard.py b/py_configuration_files/analytics_dashboard.py similarity index 100% rename from py_configuration_files/anayltics_dashboard.py rename to py_configuration_files/analytics_dashboard.py diff --git a/repo.sh b/repo.sh index cb93108f63..84792fb35c 100755 --- a/repo.sh +++ b/repo.sh @@ -36,6 +36,7 @@ repos=( "https://github.com/openedx/frontend-app-publisher.git" "https://github.com/edx/edx-analytics-dashboard.git" "https://github.com/edx/edx-analytics-data-api.git" + "https://github.com/openedx/enterprise-catalog.git" ) non_release_repos=( @@ -56,6 +57,7 @@ ssh_repos=( "git@github.com:openedx/cs_comments_service.git" "git@github.com:edx/ecommerce.git" "git@github.com:openedx/edx-notes-api.git" + "git@github.com:openedx/enterprise-catalog.git" "git@github.com:openedx/edx-platform.git" "git@github.com:openedx/xqueue.git" "git@github.com:edx/edx-analytics-dashboard.git" diff --git a/requirements/dev.txt b/requirements/dev.txt index 13d572ecd3..b75265827d 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -28,7 +28,7 @@ iniconfig==2.0.0 # via # -r requirements/test.txt # pytest -packaging==24.1 +packaging==24.2 # via # -r requirements/pip-tools.txt # -r requirements/test.txt @@ -60,7 +60,7 @@ pyproject-hooks==1.2.0 # -r requirements/pip-tools.txt # build # pip-tools -pytest==8.3.3 +pytest==8.3.4 # via -r requirements/test.txt pyyaml==6.0.2 # via @@ -68,9 +68,9 @@ pyyaml==6.0.2 # -r requirements/test.txt tox==4.23.2 # via -r requirements/dev.in -virtualenv==20.27.1 +virtualenv==20.28.0 # via tox -wheel==0.44.0 +wheel==0.45.1 # via # -r requirements/pip-tools.txt # pip-tools diff --git a/requirements/doc.txt b/requirements/doc.txt index 80429a1d09..7bdc64212a 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -35,9 +35,9 @@ jinja2==3.1.4 # via sphinx markupsafe==3.0.2 # via jinja2 -nh3==0.2.18 +nh3==0.2.19 # via readme-renderer -packaging==24.1 +packaging==24.2 # via sphinx pbr==6.1.0 # via stevedore @@ -81,7 +81,7 @@ sphinxcontrib-qthelp==2.0.0 # via sphinx sphinxcontrib-serializinghtml==2.0.0 # via sphinx -stevedore==5.3.0 +stevedore==5.4.0 # via doc8 typing-extensions==4.12.2 # via pydata-sphinx-theme diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index db6c463229..8308278227 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -8,7 +8,7 @@ build==1.2.2.post1 # via pip-tools click==8.1.7 # via pip-tools -packaging==24.1 +packaging==24.2 # via build pip-tools==7.4.1 # via -r requirements/pip-tools.in @@ -16,7 +16,7 @@ pyproject-hooks==1.2.0 # via # build # pip-tools -wheel==0.44.0 +wheel==0.45.1 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/pip.txt b/requirements/pip.txt index a5d49fadf9..25d1d695f4 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -4,7 +4,7 @@ # # make upgrade # -wheel==0.44.0 +wheel==0.45.1 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: @@ -12,5 +12,5 @@ pip==24.2 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/pip.in -setuptools==75.3.0 +setuptools==75.6.0 # via -r requirements/pip.in diff --git a/requirements/test.txt b/requirements/test.txt index cb6ed0cd6b..cc1937d704 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -6,7 +6,7 @@ # iniconfig==2.0.0 # via pytest -packaging==24.1 +packaging==24.2 # via pytest pexpect==4.9.0 # via -r requirements/test.in @@ -14,7 +14,7 @@ pluggy==1.5.0 # via pytest ptyprocess==0.7.0 # via pexpect -pytest==8.3.3 +pytest==8.3.4 # via -r requirements/test.in pyyaml==6.0.2 # via -r requirements/base.txt