diff --git a/.annotation_safe_list.yml b/.annotation_safe_list.yml index c57eeeb2503d..e91fe39cd613 100644 --- a/.annotation_safe_list.yml +++ b/.annotation_safe_list.yml @@ -142,8 +142,6 @@ workflow.AssessmentWorkflowStep: # Via edx-celeryutils celery_utils.ChordData: ".. no_pii:": "No PII" -celery_utils.FailedTask: - ".. no_pii:": "No PII" # Via completion XBlock completion.BlockCompletion: diff --git a/.eslintignore b/.eslintignore index 9044a0cc711f..b7754944dca5 100644 --- a/.eslintignore +++ b/.eslintignore @@ -20,9 +20,21 @@ test_root/staticfiles common/static/xmodule -# Symlinks into xmodule/js +# Various intra-repo symlinks that we've added over the years to duct-tape the JS build together. +# Ignore them so that we're not double-counting these violations. +cms/static/edx-ui-toolkit cms/static/xmodule_js +lms/static/common +lms/static/course_bookmarks +lms/static/course_experience +lms/static/course_search +lms/static/discussion +lms/static/edx-ui-toolkit +lms/static/learner_profile +lms/static/support +lms/static/teams lms/static/xmodule_js +xmodule/js/common_static # Mako templates that generate .js files diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml deleted file mode 100644 index 6831e3563d81..000000000000 --- a/.github/workflows/docker-publish.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Push Docker Images - -on: - push: - branches: - - master - -jobs: - # Push image to GitHub Packages. - # See also https://docs.docker.com/docker-hub/builds/ - push: - runs-on: ubuntu-latest - if: github.event_name == 'push' - - strategy: - matrix: - variant: - - "lms_dev" - - "cms_dev" - - "cms" - - "lms" - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Build and push lms/cms base docker images - env: - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - run: make docker_tag_build_push_${{matrix.variant}} diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index 4496a4b61c41..463352e1c552 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -64,13 +64,13 @@ jobs: make base-requirements - uses: c-hive/gha-npm-cache@v1 + + - name: Install npm + run: npm ci + - name: Run JS Tests - env: - TEST_SUITE: js-unit - SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh run: | - npm install -g jest - xvfb-run --auto-servernum ./scripts/all-tests.sh + npm run test - name: Save Job Artifacts uses: actions/upload-artifact@v4 diff --git a/.github/workflows/publish-ci-docker-image.yml b/.github/workflows/publish-ci-docker-image.yml deleted file mode 100644 index 6a0f3768b7e6..000000000000 --- a/.github/workflows/publish-ci-docker-image.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Push CI Runner Docker Image - -on: - workflow_dispatch: - schedule: - - cron: "0 1 * * 3" - -jobs: - push: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - # This has to happen after checkout in order for gh to work. - - name: "Cancel scheduled job on forks" - if: github.repository != 'openedx/edx-platform' && github.event_name == 'schedule' - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - run: | - gh run cancel "${{ github.run_id }}" - gh run watch "${{ github.run_id }}" - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.TOOLS_EDX_ECR_USER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.TOOLS_EDX_ECR_USER_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Log in to ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Build, tag, and push image to Amazon ECR - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: actions-runner - IMAGE_TAG: latest - run: | - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f scripts/ci-runner.Dockerfile . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG diff --git a/.github/workflows/pylint-checks.yml b/.github/workflows/pylint-checks.yml index 9a654e09e711..1afa032b6ad1 100644 --- a/.github/workflows/pylint-checks.yml +++ b/.github/workflows/pylint-checks.yml @@ -22,7 +22,7 @@ jobs: - module-name: openedx-2 path: "openedx/core/djangoapps/geoinfo/ openedx/core/djangoapps/header_control/ openedx/core/djangoapps/heartbeat/ openedx/core/djangoapps/lang_pref/ openedx/core/djangoapps/models/ openedx/core/djangoapps/monkey_patch/ openedx/core/djangoapps/oauth_dispatch/ openedx/core/djangoapps/olx_rest_api/ openedx/core/djangoapps/password_policy/ openedx/core/djangoapps/plugin_api/ openedx/core/djangoapps/plugins/ openedx/core/djangoapps/profile_images/ openedx/core/djangoapps/programs/ openedx/core/djangoapps/safe_sessions/ openedx/core/djangoapps/schedules/ openedx/core/djangoapps/service_status/ openedx/core/djangoapps/session_inactivity_timeout/ openedx/core/djangoapps/signals/ openedx/core/djangoapps/site_configuration/ openedx/core/djangoapps/system_wide_roles/ openedx/core/djangoapps/theming/ openedx/core/djangoapps/user_api/ openedx/core/djangoapps/user_authn/ openedx/core/djangoapps/util/ openedx/core/djangoapps/verified_track_content/ openedx/core/djangoapps/video_config/ openedx/core/djangoapps/video_pipeline/ openedx/core/djangoapps/waffle_utils/ openedx/core/djangoapps/xblock/ openedx/core/djangoapps/xmodule_django/ openedx/core/tests/ openedx/features/ openedx/testing/ openedx/tests/ openedx/core/djangoapps/notifications/ openedx/core/djangoapps/staticfiles/ openedx/core/djangoapps/content_tagging/" - module-name: common - path: "common pavelib" + path: "common" - module-name: cms path: "cms" - module-name: xmodule diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index 310f9f83bf3d..2452f54da14b 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -60,16 +60,29 @@ jobs: PIP_SRC: ${{ runner.temp }} run: | make test-requirements - + + - name: Install npm + env: + PIP_SRC: ${{ runner.temp }} + run: npm ci + + - name: Install python packages + env: + PIP_SRC: ${{ runner.temp }} + run: | + pip install -e . + - name: Run Quality Tests env: - TEST_SUITE: quality - SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh PIP_SRC: ${{ runner.temp }} TARGET_BRANCH: ${{ github.base_ref }} run: | - ./scripts/all-tests.sh - + make pycodestyle + npm run lint + make xsslint + make pii_check + make check_keywords + - name: Save Job Artifacts if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/unit-test-shards.json b/.github/workflows/unit-test-shards.json index 1af8af814254..4709930493ce 100644 --- a/.github/workflows/unit-test-shards.json +++ b/.github/workflows/unit-test-shards.json @@ -255,15 +255,13 @@ "common-with-lms": { "settings": "lms.envs.test", "paths": [ - "common/djangoapps/", - "pavelib/" + "common/djangoapps/" ] }, "common-with-cms": { "settings": "cms.envs.test", "paths": [ - "common/djangoapps/", - "pavelib/" + "common/djangoapps/" ] }, "xmodule-with-lms": { diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index e691e16e47f1..c635972d5f4c 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -71,29 +71,7 @@ jobs: - name: install system requirements run: | - sudo apt-get update && sudo apt-get install libmysqlclient-dev libxmlsec1-dev lynx openssl - - # This is needed until the ENABLE_BLAKE2B_HASHING can be removed and we - # can stop using MD4 by default. - - name: enable md4 hashing in libssl - run: | - cat <> $GITHUB_ENV - echo "root_lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test lms/ openedx/ common/djangoapps/ xmodule/ pavelib/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV + echo "root_lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test lms/ openedx/ common/djangoapps/ xmodule/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV - name: get GHA unit test paths shell: bash diff --git a/.pii_annotations.yml b/.pii_annotations.yml index 328520738f10..9000115a253e 100644 --- a/.pii_annotations.yml +++ b/.pii_annotations.yml @@ -1,7 +1,7 @@ source_path: ./ report_path: pii_report safelist_path: .annotation_safe_list.yml -coverage_target: 94.5 +coverage_target: 85.3 # See OEP-30 for more information on these values and what they mean: # https://open-edx-proposals.readthedocs.io/en/latest/oep-0030-arch-pii-markup-and-auditing.html#docstring-annotations annotations: diff --git a/.stylelintignore b/.stylelintignore deleted file mode 100644 index cd53bacf3cf9..000000000000 --- a/.stylelintignore +++ /dev/null @@ -1,5 +0,0 @@ -xmodule/css -common/static/sass/bourbon -common/static/xmodule/modules/css -common/test/test-theme -lms/static/sass/vendor diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index a67c46738bcc..000000000000 --- a/Dockerfile +++ /dev/null @@ -1,200 +0,0 @@ -FROM ubuntu:focal as minimal-system - -# Warning: This file is experimental. -# -# Short-term goals: -# * Be a suitable replacement for the `edxops/edxapp` image in devstack (in progress). -# * Take advantage of Docker caching layers: aim to put commands in order of -# increasing cache-busting frequency. -# * Related to ^, use no Ansible or Paver. -# Long-term goal: -# * Be a suitable base for production LMS and CMS images (THIS IS NOT YET THE CASE!). - -ARG DEBIAN_FRONTEND=noninteractive -ARG SERVICE_VARIANT -ARG SERVICE_PORT - -# Env vars: paver -# We intentionally don't use paver in this Dockerfile, but Devstack may invoke paver commands -# during provisioning. Enabling NO_PREREQ_INSTALL tells paver not to re-install Python -# requirements for every paver command, potentially saving a lot of developer time. -ARG NO_PREREQ_INSTALL='1' - -# Env vars: locale -ENV LANG='en_US.UTF-8' -ENV LANGUAGE='en_US:en' -ENV LC_ALL='en_US.UTF-8' - -# Env vars: configuration -ENV CONFIG_ROOT='/edx/etc' -ENV LMS_CFG="$CONFIG_ROOT/lms.yml" -ENV CMS_CFG="$CONFIG_ROOT/cms.yml" - -# Env vars: path -ENV VIRTUAL_ENV="/edx/app/edxapp/venvs/edxapp" -ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" -ENV PATH="/edx/app/edxapp/edx-platform/node_modules/.bin:${PATH}" -ENV PATH="/edx/app/edxapp/edx-platform/bin:${PATH}" -ENV PATH="/edx/app/edxapp/nodeenv/bin:${PATH}" - -WORKDIR /edx/app/edxapp/edx-platform - -# Create user before assigning any directory ownership to it. -RUN useradd -m --shell /bin/false app - -# Use debconf to set locales to be generated when the locales apt package is installed later. -RUN echo "locales locales/default_environment_locale select en_US.UTF-8" | debconf-set-selections -RUN echo "locales locales/locales_to_be_generated multiselect en_US.UTF-8 UTF-8" | debconf-set-selections - -# Setting up ppa deadsnakes to get python 3.11 -RUN apt-get update && \ - apt-get install -y software-properties-common && \ - apt-add-repository -y ppa:deadsnakes/ppa - -# Install requirements that are absolutely necessary -RUN apt-get update && \ - apt-get -y dist-upgrade && \ - apt-get -y install --no-install-recommends \ - python3-pip \ - python3.11 \ - # python3-dev: required for building mysqlclient python package - python3.11-dev \ - python3.11-venv \ - libpython3.11 \ - libpython3.11-stdlib \ - libmysqlclient21 \ - # libmysqlclient-dev: required for building mysqlclient python package - libmysqlclient-dev \ - pkg-config \ - libssl1.1 \ - libxmlsec1-openssl \ - # lynx: Required by https://github.com/openedx/edx-platform/blob/b489a4ecb122/openedx/core/lib/html_to_text.py#L16 - lynx \ - ntp \ - git \ - build-essential \ - gettext \ - gfortran \ - graphviz \ - locales \ - swig \ - && \ - apt-get clean all && \ - rm -rf /var/lib/apt/* - -RUN mkdir -p /edx/var/edxapp -RUN mkdir -p /edx/etc -RUN chown app:app /edx/var/edxapp - -# The builder-production stage is a temporary stage that installs required packages and builds the python virtualenv, -# installs nodejs and node_modules. -# The built artifacts from this stage are then copied to the base stage. -FROM minimal-system as builder-production - -RUN apt-get update && \ - apt-get -y install --no-install-recommends \ - curl \ - libssl-dev \ - libffi-dev \ - libfreetype6-dev \ - libgeos-dev \ - libgraphviz-dev \ - libjpeg8-dev \ - liblapack-dev \ - libpng-dev \ - libsqlite3-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev - -# Setup python virtual environment -# It is already 'activated' because $VIRTUAL_ENV/bin was put on $PATH -RUN python3.11 -m venv "${VIRTUAL_ENV}" - -# Install python requirements -# Requires copying over requirements files, but not entire repository -COPY requirements requirements -RUN pip install -r requirements/pip.txt -RUN pip install -r requirements/edx/base.txt - -# Install node and npm -RUN nodeenv /edx/app/edxapp/nodeenv --node=20.15.1 --prebuilt -RUN npm install -g npm@10.7.x - -# This script is used by an npm post-install hook. -# We copy it into the image now so that it will be available when we run `npm install` in the next step. -# The script itself will copy certain modules into some uber-legacy parts of edx-platform which still use RequireJS. -COPY scripts/copy-node-modules.sh scripts/copy-node-modules.sh - -# Install node modules -COPY package.json package.json -COPY package-lock.json package-lock.json -RUN npm set progress=false && npm ci - -# The builder-development stage is a temporary stage that installs python modules required for development purposes -# The built artifacts from this stage are then copied to the development stage. -FROM builder-production as builder-development - -RUN pip install -r requirements/edx/development.txt - -# base stage -FROM minimal-system as base - -# Copy python virtual environment, nodejs and node_modules -COPY --from=builder-production /edx/app/edxapp/venvs/edxapp /edx/app/edxapp/venvs/edxapp -COPY --from=builder-production /edx/app/edxapp/nodeenv /edx/app/edxapp/nodeenv -COPY --from=builder-production /edx/app/edxapp/edx-platform/node_modules /edx/app/edxapp/edx-platform/node_modules - -# Copy over remaining parts of repository (including all code) -COPY . . - -# Install Python requirements again in order to capture local projects -RUN pip install -e . - -# Setting edx-platform directory as safe for git commands -RUN git config --global --add safe.directory /edx/app/edxapp/edx-platform - -# Production target -FROM base as production - -USER app - -ENV EDX_PLATFORM_SETTINGS='docker-production' -ENV SERVICE_VARIANT="${SERVICE_VARIANT}" -ENV SERVICE_PORT="${SERVICE_PORT}" -ENV DJANGO_SETTINGS_MODULE="${SERVICE_VARIANT}.envs.$EDX_PLATFORM_SETTINGS" -EXPOSE ${SERVICE_PORT} - -CMD gunicorn \ - -c /edx/app/edxapp/edx-platform/${SERVICE_VARIANT}/docker_${SERVICE_VARIANT}_gunicorn.py \ - --name ${SERVICE_VARIANT} \ - --bind=0.0.0.0:${SERVICE_PORT} \ - --max-requests=1000 \ - --access-logfile \ - - ${SERVICE_VARIANT}.wsgi:application - -# Development target -FROM base as development - -RUN apt-get update && \ - apt-get -y install --no-install-recommends \ - # wget is used in Makefile for common_constraints.txt - wget \ - && \ - apt-get clean all && \ - rm -rf /var/lib/apt/* - -COPY --from=builder-development /edx/app/edxapp/venvs/edxapp /edx/app/edxapp/venvs/edxapp - -RUN ln -s "$(pwd)/lms/envs/devstack-experimental.yml" "$LMS_CFG" -RUN ln -s "$(pwd)/cms/envs/devstack-experimental.yml" "$CMS_CFG" -# Temporary compatibility hack while devstack is supporting both the old `edxops/edxapp` image and this image. -# * Add in a dummy ../edxapp_env file -# * devstack sets /edx/etc/studio.yml as CMS_CFG. -RUN ln -s "$(pwd)/cms/envs/devstack-experimental.yml" "/edx/etc/studio.yml" -RUN touch ../edxapp_env - -ENV EDX_PLATFORM_SETTINGS='devstack_docker' -ENV SERVICE_VARIANT="${SERVICE_VARIANT}" -EXPOSE ${SERVICE_PORT} -CMD ./manage.py ${SERVICE_VARIANT} runserver 0.0.0.0:${SERVICE_PORT} diff --git a/Makefile b/Makefile index 15bab5df67a9..08b6f14893cc 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,7 @@ # Do things in edx-platform .PHONY: base-requirements check-types clean \ compile-requirements detect_changed_source_translations dev-requirements \ - docker_auth docker_build docker_tag_build_push_lms docker_tag_build_push_lms_dev \ - docker_tag_build_push_cms docker_tag_build_push_cms_dev docs extract_translations \ + docs extract_translations \ guides help lint-imports local-requirements migrate migrate-lms migrate-cms \ pre-requirements pull pull_xblock_translations pull_translations push_translations \ requirements shell swagger \ @@ -67,9 +66,6 @@ pull_translations: clean_translations ## pull translations via atlas detect_changed_source_translations: ## check if translation files are up-to-date i18n_tool changed -pull: ## update the Docker image used by "make shell" - docker pull edxops/edxapp:latest - pre-requirements: ## install Python requirements for running pip-tools pip install -r requirements/pip.txt pip install -r requirements/pip-tools.txt @@ -94,17 +90,9 @@ test-requirements: pre-requirements requirements: dev-requirements ## install development environment requirements -shell: ## launch a bash shell in a Docker container with all edx-platform dependencies installed - docker run -it -e "NO_PYTHON_UNINSTALL=1" -e "PIP_INDEX_URL=https://pypi.python.org/simple" -e TERM \ - -v `pwd`:/edx/app/edxapp/edx-platform:cached \ - -v edxapp_lms_assets:/edx/var/edxapp/staticfiles/ \ - -v edxapp_node_modules:/edx/app/edxapp/edx-platform/node_modules \ - edxops/edxapp:latest /edx/app/edxapp/devstack.sh open - # Order is very important in this list: files must appear after everything they include! REQ_FILES = \ requirements/edx/coverage \ - requirements/edx/paver \ requirements/edx-sandbox/base \ requirements/edx/base \ requirements/edx/doc \ @@ -164,27 +152,6 @@ upgrade-package: ## update just one package to the latest usable release check-types: ## run static type-checking tests mypy -docker_auth: - echo "$$DOCKERHUB_PASSWORD" | docker login -u "$$DOCKERHUB_USERNAME" --password-stdin - -docker_build: docker_auth - DOCKER_BUILDKIT=1 docker build . --build-arg SERVICE_VARIANT=lms --build-arg SERVICE_PORT=8000 --target development -t openedx/lms-dev - DOCKER_BUILDKIT=1 docker build . --build-arg SERVICE_VARIANT=lms --build-arg SERVICE_PORT=8000 --target production -t openedx/lms - DOCKER_BUILDKIT=1 docker build . --build-arg SERVICE_VARIANT=cms --build-arg SERVICE_PORT=8010 --target development -t openedx/cms-dev - DOCKER_BUILDKIT=1 docker build . --build-arg SERVICE_VARIANT=cms --build-arg SERVICE_PORT=8010 --target production -t openedx/cms - -docker_tag_build_push_lms: docker_auth - docker buildx build -t openedx/lms:latest -t openedx/lms:${GITHUB_SHA} --platform linux/amd64,linux/arm64 --build-arg SERVICE_VARIANT=lms --build-arg SERVICE_PORT=8000 --target production --push . - -docker_tag_build_push_lms_dev: docker_auth - docker buildx build -t openedx/lms-dev:latest -t openedx/lms-dev:${GITHUB_SHA} --platform linux/amd64,linux/arm64 --build-arg SERVICE_VARIANT=lms --build-arg SERVICE_PORT=8000 --target development --push . - -docker_tag_build_push_cms: docker_auth - docker buildx build -t openedx/cms:latest -t openedx/cms:${GITHUB_SHA} --platform linux/amd64,linux/arm64 --build-arg SERVICE_VARIANT=cms --build-arg SERVICE_PORT=8010 --target production --push . - -docker_tag_build_push_cms_dev: docker_auth - docker buildx build -t openedx/cms-dev:latest -t openedx/cms-dev:${GITHUB_SHA} --platform linux/amd64,linux/arm64 --build-arg SERVICE_VARIANT=cms --build-arg SERVICE_PORT=8010 --target development --push . - lint-imports: lint-imports @@ -204,3 +171,37 @@ migrate: migrate-lms migrate-cms # Part of https://github.com/openedx/wg-developer-experience/issues/136 ubuntu-requirements: ## Install ubuntu 22.04 system packages needed for `pip install` to work on ubuntu. sudo apt install libmysqlclient-dev libxmlsec1-dev + +xsslint: ## check xss for quality issuest + python scripts/xsslint/xss_linter.py \ + --rule-totals \ + --config=scripts.xsslint_config \ + --thresholds=scripts/xsslint_thresholds.json + +pycodestyle: ## check python files for quality issues + pycodestyle . + +## Re-enable --lint flag when this issue https://github.com/openedx/edx-platform/issues/35775 is resolved +pii_check: ## check django models for pii annotations + DJANGO_SETTINGS_MODULE=cms.envs.test \ + code_annotations django_find_annotations \ + --config_file .pii_annotations.yml \ + --app_name cms \ + --coverage \ + --lint + + DJANGO_SETTINGS_MODULE=lms.envs.test \ + code_annotations django_find_annotations \ + --config_file .pii_annotations.yml \ + --app_name lms \ + --coverage \ + --lint + +check_keywords: ## check django models for reserve keywords + DJANGO_SETTINGS_MODULE=cms.envs.test \ + python manage.py cms check_reserved_keywords \ + --override_file db_keyword_overrides.yml + + DJANGO_SETTINGS_MODULE=lms.envs.test \ + python manage.py lms check_reserved_keywords \ + --override_file db_keyword_overrides.yml diff --git a/cms/envs/common.py b/cms/envs/common.py index 20e99974b3fb..591247388a9d 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -566,9 +566,6 @@ # .. toggle_tickets: https://github.com/openedx/edx-platform/pull/33911 'ENABLE_GRADING_METHOD_IN_PROBLEMS': False, - # See annotations in lms/envs/common.py for details. - 'ENABLE_BLAKE2B_HASHING': False, - # .. toggle_name: FEATURES['BADGES_ENABLED'] # .. toggle_implementation: DjangoSetting # .. toggle_default: False diff --git a/cms/static/sass/elements/_vendor.scss b/cms/static/sass/elements/_vendor.scss index 5418745922f9..b57fa04175c2 100644 --- a/cms/static/sass/elements/_vendor.scss +++ b/cms/static/sass/elements/_vendor.scss @@ -66,13 +66,6 @@ z-index: 100000 !important; } -//jQuery loupeAndLightbox Plugin -.zooming-image-place { - .larger { - left: 0 !important; - bottom: 100% !important; - } -} // ==================== // reset styles to remove ui-lightness jquery ui theme from the tabs component (used in the add component problem tab menu) diff --git a/common/djangoapps/util/memcache.py b/common/djangoapps/util/memcache.py index 2f7e6dc623a0..ce8c70219e2c 100644 --- a/common/djangoapps/util/memcache.py +++ b/common/djangoapps/util/memcache.py @@ -7,7 +7,6 @@ import hashlib from urllib.parse import quote_plus -from django.conf import settings from django.utils.encoding import smart_str @@ -15,10 +14,7 @@ def fasthash(string): """ Hashes `string` into a string representation of a 128-bit digest. """ - if settings.FEATURES.get("ENABLE_BLAKE2B_HASHING", False): - hash_obj = hashlib.new("blake2b", digest_size=16) - else: - hash_obj = hashlib.new("md4") + hash_obj = hashlib.new("blake2b", digest_size=16) hash_obj.update(string.encode('utf-8')) return hash_obj.hexdigest() diff --git a/common/djangoapps/util/tests/test_memcache.py b/common/djangoapps/util/tests/test_memcache.py index 13f67f386a00..0047e9fa66cf 100644 --- a/common/djangoapps/util/tests/test_memcache.py +++ b/common/djangoapps/util/tests/test_memcache.py @@ -3,15 +3,11 @@ """ -from django.conf import settings from django.core.cache import caches -from django.test import TestCase, override_settings +from django.test import TestCase from common.djangoapps.util.memcache import safe_key -BLAKE2B_ENABLED_FEATURES = settings.FEATURES.copy() -BLAKE2B_ENABLED_FEATURES["ENABLE_BLAKE2B_HASHING"] = True - class MemcacheTest(TestCase): """ @@ -55,20 +51,6 @@ def test_safe_key_long(self): # The key should now be valid assert self._is_valid_key(key), f'Failed for key length {length}' - @override_settings(FEATURES=BLAKE2B_ENABLED_FEATURES) - def test_safe_key_long_with_blake2b_enabled(self): - # Choose lengths close to memcached's cutoff (250) - for length in [248, 249, 250, 251, 252]: - - # Generate a key of that length - key = 'a' * length - - # Make the key safe - key = safe_key(key, '', '') - - # The key should now be valid - assert self._is_valid_key(key), f'Failed for key length {length}' - def test_long_key_prefix_version(self): # Long key @@ -83,34 +65,6 @@ def test_long_key_prefix_version(self): key = safe_key('key', 'prefix', 'a' * 300) assert self._is_valid_key(key) - @override_settings(FEATURES=BLAKE2B_ENABLED_FEATURES) - def test_long_key_prefix_version_with_blake2b_enabled(self): - - # Long key - key = safe_key('a' * 300, 'prefix', 'version') - assert self._is_valid_key(key) - - # Long prefix - key = safe_key('key', 'a' * 300, 'version') - assert self._is_valid_key(key) - - # Long version - key = safe_key('key', 'prefix', 'a' * 300) - assert self._is_valid_key(key) - - def test_safe_key_unicode(self): - - for unicode_char in self.UNICODE_CHAR_CODES: - - # Generate a key with that character - key = chr(unicode_char) - - # Make the key safe - key = safe_key(key, '', '') - - # The key should now be valid - assert self._is_valid_key(key), f'Failed for unicode character {unicode_char}' - def test_safe_key_prefix_unicode(self): for unicode_char in self.UNICODE_CHAR_CODES: diff --git a/common/static/data/geoip/GeoLite2-Country.mmdb b/common/static/data/geoip/GeoLite2-Country.mmdb index 4404d374206e..f32738e62173 100644 Binary files a/common/static/data/geoip/GeoLite2-Country.mmdb and b/common/static/data/geoip/GeoLite2-Country.mmdb differ diff --git a/docs/concepts/extension_points.rst b/docs/concepts/extension_points.rst index d4e802baec0e..3136aa8057c2 100644 --- a/docs/concepts/extension_points.rst +++ b/docs/concepts/extension_points.rst @@ -139,10 +139,10 @@ Here are the different integration points that python plugins can use: - This decorator allows overriding any function or method by pointing to an alternative implementation in settings. Read the |pluggable_override docstring|_ to learn more. * - Open edX Events - Adopt, Stable - - Events are part of the greater Hooks Extension Framework for open extension of edx-platform. Events are a stable way for plugin developers to react to learner or author events. They are defined by a `separate events library`_ that developers can include in their requirements to develop and test the code without creating a dependency on this large repo. For more information see the `hooks guide`_. + - Events are part of the greater Hooks Extension Framework for open extension of edx-platform. Events are a stable way for plugin developers to react to learner or author events. They are defined by a `separate events library`_ that developers can include in their requirements to develop and test the code without creating a dependency on this large repo. For more information see the `Hooks Extension Framework docs`_ or for more detailed documentation about Open edX Events, see the `Open edX Events documentation`_. * - Open edX Filters - Adopt, Stable - - Filters are also part of Hooks Extension Framework for open extension of edx-platform. Filters are a flexible way for plugin developers to modify learner or author application flows. They are defined by a `separate filters library`_ that developers can include in their requirements to develop and test the code without creating a dependency on this large repo. For more information see the `hooks guide`_. + - Filters are also part of Hooks Extension Framework for open extension of edx-platform. Filters are a flexible way for plugin developers to modify learner or author application flows. They are defined by a `separate filters library`_ that developers can include in their requirements to develop and test the code without creating a dependency on this large repo. For more information see the `Hooks Extension Framework docs`_ or for more detailed documentation about Open edX Filters, see the `Open edX Filters documentation`_. .. _Application: https://docs.djangoproject.com/en/3.0/ref/applications/ .. _Django app plugin documentation: https://github.com/openedx/edx-platform/blob/master/openedx/core/djangoapps/plugins/README.rst @@ -159,7 +159,9 @@ Here are the different integration points that python plugins can use: .. _pluggable_override docstring: https://github.com/openedx/edx-django-utils/blob/master/edx_django_utils/plugins/pluggable_override.py .. _separate events library: https://github.com/eduNEXT/openedx-events/ .. _separate filters library: https://github.com/eduNEXT/openedx-filters/ -.. _hooks guide: https://github.com/openedx/edx-platform/blob/master/docs/guides/hooks/index.rst +.. _Hooks Extension Framework docs: https://docs.openedx.org/en/latest/developers/concepts/hooks_extension_framework.html +.. _Open edX Events documentation: https://docs.openedx.org/projects/openedx-events/en/latest/ +.. _Open edX Filters documentation: https://docs.openedx.org/projects/openedx-filters/en/latest/ Platform Look & Feel ==================== diff --git a/docs/concepts/testing/testing.rst b/docs/concepts/testing/testing.rst index 9d448afd5bdc..d8a79b9d6619 100644 --- a/docs/concepts/testing/testing.rst +++ b/docs/concepts/testing/testing.rst @@ -1,4 +1,3 @@ -####### Testing ####### @@ -7,7 +6,7 @@ Testing :depth: 3 Overview -======== +******** We maintain two kinds of tests: unit tests and integration tests. @@ -26,10 +25,10 @@ tests. Most of our tests are unit tests or integration tests. Test Types ----------- +========== Unit Tests -~~~~~~~~~~ +---------- - Each test case should be concise: setup, execute, check, and teardown. If you find yourself writing tests with many steps, @@ -38,18 +37,18 @@ Unit Tests - As a rule of thumb, your unit tests should cover every code branch. -- Mock or patch external dependencies. We use the voidspace `Mock Library`_. +- Mock or patch external dependencies using `unittest.mock`_ functions. - We unit test Python code (using `unittest`_) and Javascript (using `Jasmine`_) -.. _Mock Library: http://www.voidspace.org.uk/python/mock/ +.. _unittest.mock: https://docs.python.org/3/library/unittest.mock.html .. _unittest: http://docs.python.org/2/library/unittest.html .. _Jasmine: http://jasmine.github.io/ Integration Tests -~~~~~~~~~~~~~~~~~ +----------------- - Test several units at the same time. Note that you can still mock or patch dependencies that are not under test! For example, you might test that @@ -67,7 +66,7 @@ Integration Tests .. _Django test client: https://docs.djangoproject.com/en/dev/topics/testing/overview/ Test Locations --------------- +============== - Python unit and integration tests: Located in subpackages called ``tests``. For example, the tests for the ``capa`` package are @@ -80,14 +79,29 @@ Test Locations the test for ``src/views/module.js`` should be written in ``spec/views/module_spec.js``. -Running Tests -============= +Factories +========= -**Unless otherwise mentioned, all the following commands should be run from inside the lms docker container.** +Many tests delegate set-up to a "factory" class. For example, there are +factories for creating courses, problems, and users. This encapsulates +set-up logic from tests. +Factories are often implemented using `FactoryBoy`_. + +In general, factories should be located close to the code they use. For +example, the factory for creating problem XML definitions is located in +``xmodule/capa/tests/response_xml_factory.py`` because the +``capa`` package handles problem XML. + +.. _FactoryBoy: https://readthedocs.org/projects/factoryboy/ Running Python Unit tests -------------------------- +************************* + +The following commands need to be run within a Python environment in +which requirements/edx/testing.txt has been installed. If you are using a +Docker-based Open edX distribution, then you probably will want to run these +commands within the LMS and/or CMS Docker containers. We use `pytest`_ to run Python tests. Pytest is a testing framework for python and should be your goto for local Python unit testing. @@ -97,16 +111,16 @@ Pytest (and all of the plugins we use with it) has a lot of options. Use `pytest Running Python Test Subsets -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=========================== When developing tests, it is often helpful to be able to really just run one single test without the overhead of PIP installs, UX builds, etc. Various ways to run tests using pytest:: - pytest path/test_m­odule.py # Run all tests in a module. - pytest path/test_m­odule.p­y:­:te­st_func # Run a specific test within a module. - pytest path/test_m­odule.p­y:­:Te­stC­las­s # Run all tests in a class - pytest path/test_m­odule.p­y:­:Te­stC­las­s::­tes­t_m­ethod # Run a specific method of a class. + pytest path/test_module.py # Run all tests in a module. + pytest path/test_module.py::test_func # Run a specific test within a module. + pytest path/test_module.py::TestClass # Run all tests in a class + pytest path/test_module.py::TestClass::test_method # Run a specific method of a class. pytest path/testing/ # Run all tests in a directory. For example, this command runs a single python unit test file:: @@ -114,7 +128,7 @@ For example, this command runs a single python unit test file:: pytest xmodule/tests/test_stringify.py Note - -edx-platorm has multiple services (lms, cms) in it. The environment for each service is different enough that we run some tests in both environments in Github Actions. +edx-platorm has multiple services (lms, cms) in it. The environment for each service is different enough that we run some tests in both environments in Github Actions. To test in each of these environments (especially for tests in "common" and "xmodule" directories), you will need to test in each seperately. To specify that the tests are run with the relevant service as root, Add --rootdir flag at end of your pytest call and specify the env to test in:: @@ -139,7 +153,7 @@ Various tools like ddt create tests with very complex names, rather than figurin pytest xmodule/tests/test_stringify.py --collectonly Testing with migrations -*********************** +======================= For the sake of speed, by default the python unit test database tables are created directly from apps' models. If you want to run the tests @@ -149,7 +163,7 @@ against a database created by applying the migrations instead, use the pytest test --create-db --migrations Debugging a test -~~~~~~~~~~~~~~~~ +================ There are various ways to debug tests in Python and more specifically with pytest: @@ -173,7 +187,7 @@ There are various ways to debug tests in Python and more specifically with pytes How to output coverage locally -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +============================== These are examples of how to run a single test and get coverage:: @@ -220,234 +234,84 @@ run one of these commands:: .. _YouTube stub server: https://github.com/openedx/edx-platform/blob/master/common/djangoapps/terrain/stubs/tests/test_youtube_stub.py -Debugging Unittest Flakiness -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -As we move over to running our unittests with Jenkins Pipelines and pytest-xdist, -there are new ways for tests to flake, which can sometimes be difficult to debug. -If you run into flakiness, check (and feel free to contribute to) this -`confluence document `__ for help. - -Running Javascript Unit Tests ------------------------------ - -Before running Javascript unit tests, you will need to be running Firefox or Chrome in a place visible to edx-platform. If running this in devstack, you can run ``make dev.up.firefox`` or ``make dev.up.chrome``. Firefox is the default browser for the tests, so if you decide to use Chrome, you will need to prefix the test command with ``SELENIUM_BROWSER=chrome SELENIUM_HOST=edx.devstack.chrome`` (if using devstack). - -We use Jasmine to run JavaScript unit tests. To run all the JavaScript -tests:: - - paver test_js - -To run a specific set of JavaScript tests and print the results to the -console, run these commands:: - - paver test_js_run -s lms - paver test_js_run -s cms - paver test_js_run -s cms-squire - paver test_js_run -s xmodule - paver test_js_run -s xmodule-webpack - paver test_js_run -s common - paver test_js_run -s common-requirejs - -To run JavaScript tests in a browser, run these commands:: - - paver test_js_dev -s lms - paver test_js_dev -s cms - paver test_js_dev -s cms-squire - paver test_js_dev -s xmodule - paver test_js_dev -s xmodule-webpack - paver test_js_dev -s common - paver test_js_dev -s common-requirejs - -Debugging Specific Javascript Tests -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The best way to debug individual tests is to run the test suite in the browser and -use your browser's Javascript debugger. The debug page will allow you to select -an individual test and only view the results of that test. - - -Debugging Tests in a Browser -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To debug these tests on devstack in a local browser: - -* first run the appropriate test_js_dev command from above -* open http://localhost:19876/debug.html in your host system's browser of choice -* this will run all the tests and show you the results including details of any failures -* you can click on an individually failing test and/or suite to re-run it by itself -* you can now use the browser's developer tools to debug as you would any other JavaScript code - -Note: the port is also output to the console that you ran the tests from if you find that easier. - -These paver commands call through to Karma. For more -info, see `karma-runner.github.io `__. - -Testing internationalization with dummy translations ----------------------------------------------------- - -Any text you add to the platform should be internationalized. To generate translations for your new strings, run the following command:: - - paver i18n_dummy - -This command generates dummy translations for each dummy language in the -platform and puts the dummy strings in the appropriate language files. -You can then preview the dummy languages on your local machine and also in your sandbox, if and when you create one. - -The dummy language files that are generated during this process can be -found in the following locations:: - - conf/locale/{LANG_CODE} - -There are a few JavaScript files that are generated from this process. You can find those in the following locations:: - - lms/static/js/i18n/{LANG_CODE} - cms/static/js/i18n/{LANG_CODE} - -Do not commit the ``.po``, ``.mo``, ``.js`` files that are generated -in the above locations during the dummy translation process! +Handling flaky unit tests +========================= -Test Coverage and Quality -------------------------- +See this `confluence document `_. -Viewing Test Coverage -~~~~~~~~~~~~~~~~~~~~~ -We currently collect test coverage information for Python -unit/integration tests. +Running JavaScript Unit Tests +***************************** -To view test coverage: +Before running Javascript unit tests, you will need to be running Firefox or Chrome in a place visible to edx-platform. +If you are using Tutor Dev to run edx-platform, then you can do so by installing and enabling the +``test-legacy-js`` plugin from `openedx-tutor-plugins`_, and then rebuilding +the ``openedx-dev`` image:: -1. Run the test suite with this command:: + tutor plugins install https://github.com/openedx/openedx-tutor-plugins/tree/main/plugins/tutor-contrib-test-legacy-js + tutor plugins enable test-legacy-js + tutor images build openedx-dev - paver test +.. _openedx-tutor-plugins: https://github.com/openedx/openedx-tutor-plugins/ -2. Generate reports with this command:: +We use Jasmine (via Karma) to run most JavaScript unit tests. We use Jest to +run a small handful of additional JS unit tests. You can use the ``npm run +test*`` commands to run them:: - paver coverage + npm run test-karma # Run all Jasmine+Karma tests. + npm run test-jest # Run all Jest tests. + npm run test # Run both of the above. -3. Reports are located in the ``reports`` folder. The command generates - HTML and XML (Cobertura format) reports. +The Karma tests are further broken down into three types depending on how the +JavaScript it is testing is built:: -Python Code Style Quality -~~~~~~~~~~~~~~~~~~~~~~~~~ + npm run test-karma-vanilla # Our very oldest JS, which doesn't even use RequireJS + npm run test-karma-require # Old JS that uses RequireJS + npm run test-karma-webpack # Slightly "newer" JS which is built with Webpack -To view Python code style quality (including PEP 8 and pylint violations) run this command:: +Unfortunately, at the time of writing, the build for the ``test-karma-webpack`` +tests is broken. The tests are excluded from ``npm run test-karma`` as to not +fail CI. We `may fix this one day`_. - paver run_quality +.. _may fix this one day: https://github.com/openedx/edx-platform/issues/35956 -More specific options are below. +To run all Karma+Jasmine tests for a particular top-level edx-platform folder, +you can run:: -- These commands run a particular quality report:: + npm run test-cms + npm run test-lms + npm run test-xmodule + npm run test-common - paver run_pep8 - paver run_pylint - -- This command runs a report, and sets it to fail if it exceeds a given number - of violations:: - - paver run_pep8 --limit=800 - -- The ``run_quality`` uses the underlying diff-quality tool (which is packaged - with `diff-cover`_). With that, the command can be set to fail if a certain - diff threshold is not met. For example, to cause the process to fail if - quality expectations are less than 100% when compared to master (or in other - words, if style quality is worse than what is already on master):: - - paver run_quality --percentage=100 - -- Note that 'fixme' violations are not counted with run\_quality. To - see all 'TODO' lines, use this command:: - - paver find_fixme --system=lms - - ``system`` is an optional argument here. It defaults to - ``cms,lms,common``. - -.. _diff-cover: https://github.com/Bachmann1234/diff-cover - - -JavaScript Code Style Quality -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To view JavaScript code style quality run this command:: - - paver run_eslint - -- This command also comes with a ``--limit`` switch, this is an example of that switch:: - - paver run_eslint --limit=50000 - - -Code Complexity Tools -===================== - -Tool(s) available for evaluating complexity of edx-platform code: - - -- `plato `__ for JavaScript code - complexity. Several options are available on the command line; see - documentation. Below, the following command will produce an HTML report in a - subdirectory called "jscomplexity":: - - plato -q -x common/static/js/vendor/ -t common -e .eslintrc.json -r -d jscomplexity common/static/js/ - -Other Testing Tips -================== - -Connecting to Browser ---------------------- - -If you want to see the browser being automated for JavaScript, -you can connect to the container running it via VNC. - -+------------------------+----------------------+ -| Browser | VNC connection | -+========================+======================+ -| Firefox (Default) | vnc://0.0.0.0:25900 | -+------------------------+----------------------+ -| Chrome (via Selenium) | vnc://0.0.0.0:15900 | -+------------------------+----------------------+ - -On macOS, enter the VNC connection string in Safari to connect via VNC. The VNC -passwords for both browsers are randomly generated and logged at container -startup, and can be found by running ``make vnc-passwords``. - -Most tests are run in Firefox by default. To use Chrome for tests that normally -use Firefox instead, prefix the test command with -``SELENIUM_BROWSER=chrome SELENIUM_HOST=edx.devstack.chrome`` - -Factories ---------- - -Many tests delegate set-up to a "factory" class. For example, there are -factories for creating courses, problems, and users. This encapsulates -set-up logic from tests. - -Factories are often implemented using `FactoryBoy`_. - -In general, factories should be located close to the code they use. For -example, the factory for creating problem XML definitions is located in -``xmodule/capa/tests/response_xml_factory.py`` because the -``capa`` package handles problem XML. - -.. _FactoryBoy: https://readthedocs.org/projects/factoryboy/ +Finally, if you want to pass any options to the underlying ``node`` invocation +for Karma+Jasmine tests, you can run one of these specific commands, and put +your arguments after the ``--`` separator:: -Running Tests on Paver Scripts ------------------------------- + npm run test-cms-vanilla -- --your --args --here + npm run test-cms-require -- --your --args --here + npm run test-cms-webpack -- --your --args --here + npm run test-lms-webpack -- --your --args --here + npm run test-xmodule-vanilla -- --your --args --here + npm run test-xmodule-webpack -- --your --args --here + npm run test-common-vanilla -- --your --args --here + npm run test-common-require -- --your --args --here -To run tests on the scripts that power the various Paver commands, use the following command:: - pytest pavelib +Code Quality +************ -Testing using queue servers ---------------------------- +We use several tools to analyze code quality. The full set of them is:: -When testing problems that use a queue server on AWS (e.g. -sandbox-xqueue.edx.org), you'll need to run your server on your public IP, like so:: + mypy $PATHS... + pycodestyle $PATHS... + pylint $PATHS... + lint-imports + scripts/verify-dunder-init.sh + make xsslint + make pii_check + make check_keywords + npm run lint - ./manage.py lms runserver 0.0.0.0:8000 +Where ``$PATHS...`` is a list of folders and files to analyze, or nothing if +you would like to analyze the entire codebase (which can take a while). -When you connect to the LMS, you need to use the public ip. Use -``ifconfig`` to figure out the number, and connect e.g. to -``http://18.3.4.5:8000/`` diff --git a/docs/conf.py b/docs/conf.py index b755f3986c93..ec416f1c19e6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -68,6 +68,7 @@ 'code_annotations.contrib.sphinx.extensions.featuretoggles', 'code_annotations.contrib.sphinx.extensions.settings', # 'autoapi.extension', # Temporarily disabled + 'sphinx_reredirects', ] # Temporarily disabling autoapi_dirs and the AutoAPI extension due to performance issues. @@ -304,6 +305,16 @@ # 'xmodule': 'references/docstrings/xmodule', } +# Mapping permanently moved pages to appropriate new location outside of edx-platform +# with by sphinx-reredirects extension redirects. +# More information: https://documatt.com/sphinx-reredirects/usage.html + +redirects = { + 'hooks/events': 'https://docs.openedx.org/projects/openedx-events/en/latest/', + 'hooks/filters': 'https://docs.openedx.org/projects/openedx-filters/en/latest/', + 'hooks/index': 'https://docs.openedx.org/en/latest/developers/concepts/hooks_extension_framework.html', +} + def update_settings_module(service='lms'): """ diff --git a/docs/concepts/frontend/static_assets.rst b/docs/decisions/0000-static-asset-plan.rst similarity index 90% rename from docs/concepts/frontend/static_assets.rst rename to docs/decisions/0000-static-asset-plan.rst index 89e7a64f4455..00fcc0726ad7 100644 --- a/docs/concepts/frontend/static_assets.rst +++ b/docs/decisions/0000-static-asset-plan.rst @@ -1,6 +1,22 @@ -####################################### -edx-platform Static Asset Pipeline Plan -####################################### +0. edx-platform Static Asset Pipeline Plan +########################################## + +Status +****** + +Accepted ~2017 +Partially adopted 2017-2024 + +This was an old plan for modernizing Open edX's frontend assets. We've +retroactively turned it into an ADR because it has some valuable insights. +Although most of these improvements weren't applied as written, these ideas +(particularly, separating Python concerns from frontend tooling concerns) were +applied to both legacy edx-platform assets as well as the Micro-Frontend +framework that was developed 2017-2019. + +Context, Decision, Consequences +******************************* + Static asset handling in edx-platform has evolved in a messy way over the years. This has led to a lot of complexity and inconsistencies. This is a proposal for @@ -9,20 +25,9 @@ this is not a detailed guide for how to write React or Bootstrap code. This is instead going to talk about conventions for how we arrange, extract, and compile static assets. -Big Open Questions (TODO) -************************* - -This document is a work in progress, as the design for some of this is still in -flux, particularly around extensibility. - -* Pluggable third party apps and Webpack packaging. -* Keep the Django i18n mechanism? -* Stance on HTTP/2 and bundling granularity. -* Optimizing theme assets. -* Tests Requirements -************ +============ Any proposed solution must support: @@ -35,7 +40,7 @@ Any proposed solution must support: * Other kinds of pluggability??? Assumptions -*********** +=========== Some assumptions/opinions that this proposal is based on: @@ -54,8 +59,8 @@ Some assumptions/opinions that this proposal is based on: * It should be possible to pre-build static assets and deploy them onto S3 or similar. -Where We Are Today -****************** +Where We Are Today (2017) +========================= We have a static asset pipeline that is mostly driven by Django's built-in staticfiles finders and the collectstatic process. We use the popular @@ -95,9 +100,9 @@ places (typically ``/edx/var/edxapp/staticfiles`` for LMS and ``/edx/var/edxapp/staticfiles/studio`` for Studio) and can be collected separately. However in practice they're always run together because we deploy them from the same commits and to the same servers. - + Django vs. Webpack Conventions -****************************** +============================== The Django convention for having an app with bundled assets is to namespace them locally with the app name so that they get their own directories when they are @@ -112,7 +117,7 @@ the root of edx-platform, which would specify all bundles in the project. TODO: The big, "pluggable Webpack components" question. Proposed Repo Structure -*********************** +======================= All assets that are in common spaces like ``common/static``, ``lms/static``, and ``cms/static`` would be moved to be under the Django apps that they are a @@ -122,7 +127,7 @@ part of and follow the Django naming convention (e.g. any client-side templates will be put in ``static/{appname}/templates``. Proposed Compiled Structure -*************************** +=========================== This is meant to be a sample of the different types of things we'd have, not a full list: @@ -150,14 +155,14 @@ full list: /theming/themes/open-edx /red-theme /edx.org - + # XBlocks still collect their assets into a common space (/xmodule goes away) # We consider this to be the XBlock Runtime's app, and it collects static # assets from installed XBlocks. /xblock Django vs. Webpack Roles -************************ +======================== Rule of thumb: Django/Python still serves static assets, Webpack processes and optimizes them. diff --git a/docs/decisions/0021-fixing-quality-and-js-checks.rst b/docs/decisions/0021-fixing-quality-and-js-checks.rst new file mode 100644 index 000000000000..7d80039a8cb0 --- /dev/null +++ b/docs/decisions/0021-fixing-quality-and-js-checks.rst @@ -0,0 +1,143 @@ +Fixing the Quality and JS checks +################################ + +Status +****** + +Accepted + +Implemented by https://github.com/openedx/edx-platform/pull/35159 + +Context +******* + +edx-platform PRs need to pass a series of CI checks before merging, including +but not limited to: a CLA check, various unit tests, and various code quality +tests. Of these checks, two checks were implemented using the "Paver" Python +package, a scripting library `which we have been trying to move off of`_. These +two checks and their steps were: + +* **Check: Quality others** + + * **pii_check**: Ensure that Django models have PII annotations as + described in `OEP-30`_, with a minimum threshold of **94.5%** of models + annotated. + * **stylelint**: Statically check sass stylesheets for common errors. + * **pep8**: Run pycodestyle against Python code. + * **eslint**: Statically check javascript code for common errors. + * **xsslint**: Check python & javascript for xss vulnerabilities. + * **check_keywords**: Compare Django model field names against a denylist of + reserved keywords. + +* **Check: JS** + + * **test-js**: Run javascript unit tests. + * **coverage-js**: Check that javascript test coverage has not dropped. + +As we worked to reimplement these checks without Paver, we unfortunately +noticed that four of those steps had bugs in their implementations, and thus +had not been enforcing what they promised to: + +* **pii_check**: Instead of just checking the result of the underlying + code_annotations command, this check wrote an annotations report to a file, + and then used regex to parse the report and determine whether the check + should pass. However, the check failed to validate that the generation of the + report itself was successful. So, when malformed annotations were introduced + to the edx-proctoring repository, which edx-platform installs, the check + began silently passing. + +* **stylelint**: At some point, the `stylelint` binary stopped being available + on the runner's `$PATH`. Rather than causing the Quality Others check to + fail, the Paver code quietly ignored the shell error, and considered the + empty stylelint report file to indicate that there were not linting + violations. + +* **test-js**: There are eight suites within test-js. Six of them work fine. + But three of them--specifically the suites that test code built by Webpack-- + have not been running for some unknown amount of time. The Webpack test build + has been failing without signalling that the test suite should fail, + both preventing the tests from runnning and preventing anyone from noticing + that the tests weren't running. + +* **coverage-js**: This check tried to use `diff-cover` in order to compare the + coverage report on the current branch with the coverage report on the master + branch. However, the coverage report does not exist on the master branch, and + it's not clear when it ever did. The coverage-js step failed to validate that + `diff-cover` ran successfully, and instead of raising an error, it allowed + the JS check to pass. + +Decision & Consequences +*********************** + +pii_check +========= + +We `fixed the malformed annotations`_ in edx-proctoring, allowing the pii_check +to once again check model coverage. We have ensured that any future failure of +the code_annotations command (due to, for example, future malformed +annotations) will cause the pii_check step and the overall Quality Others check +to fail. We have stopped trying to parse the result of the annotations report +in CI, as this was and is completely unneccessary. + +In order to keep "Quality others" passing on the edx-platform master branch, we +lowered the PII annotation coverage threshold to reflect the percentage of +then-annotated models: **71.6%**. After a timeboxed effort to add missing +annotations and expand the annotation allowlist as appropriate, we have managed +to raise the threshold to **85.3%**. It is not clear whether we will put in +further effort to raise the annotation threshold back to 95%. + +This was all already `announced on the forums`_. + +stylelint +========= + +We have removed the **stylelint** step entirely from the "Quality Others" +check. Sass code in the edx-platform repository will no longer be subject to +any static analysis. + +test-js +======= + +We have stopped running these Webpack-based suites in CI: + +* ``npm run test-lms-webpack`` +* ``npm run test-cms-webpack`` +* ``npm run test-xmodule-webpack`` + +We have created a new edx-platform backlog issue for +`fixing and re-enabling these suites`_. +It is not clear whether we will prioritize that issue, or instead prioritize +deprecation and removal of the code that those suites were supposed to be +testing. + +coverage-js +=========== + +We will remove the **coverage-js** step entirely from the "JS" check. +JavaScript code in the edx-platform repository will no longer be subject to any +unit test coverage checking. + +Rejected Alternatives +********************* + +* While it would be ideal to raise the pii_check threshold to 94.5% or even + 100%, we do not have the resources to promise this. + +* It would also be nice to institute a "racheting" mechanism for the PII + annotation coverage threshold. That is, every commit to master could save the + coverage percentage to a persisted artifact, allowing subsequent PRs to + ensure that the pii_check never returns lower than the current threshold. We + will put this in the Aximprovements backlog, but we cannot commit to + implementing it right now. + +* We will not fix or apply amnestly in order to re-enable stlylint or + coverage-js. That could take significant effort, which we believe would be + better spent completing the migration off of this legacy Sass and JS and onto + our modern React frontends. + + +.. _fixing and re-enabling these suites: https://github.com/openedx/edx-platform/issues/35956 +.. _which we have been trying to move off of: https://github.com/openedx/edx-platform/issues/34467 +.. _announced on the forums: https://discuss.openedx.org/t/checking-pii-annotations-with-a-lower-coverage-threshold/14254 +.. _OEP-30: https://docs.openedx.org/projects/openedx-proposals/en/latest/architectural-decisions/oep-0030-arch-pii-markup-and-auditing.html +.. _fix the malformed annotations: https://github.com/openedx/edx-proctoring/issues/1241 diff --git a/docs/hooks/events.rst b/docs/hooks/events.rst deleted file mode 100644 index bccb98e56a42..000000000000 --- a/docs/hooks/events.rst +++ /dev/null @@ -1,261 +0,0 @@ -Open edX Events -=============== - -How to use ----------- - -Using openedx-events in your code is very straight forward. We can consider the -two possible cases, sending or receiving an event. - - -Receiving events -^^^^^^^^^^^^^^^^ - -This is one of the most common use cases for plugins. The edx-platform will send -an event and you want to react to it in your plugin. - -For this you need to: - -1. Include openedx-events in your dependencies. -2. Connect your receiver functions to the signals being sent. - -Connecting signals can be done using regular django syntax: - -.. code-block:: python - - from openedx_events.learning.signals import SESSION_LOGIN_COMPLETED - - @receiver(SESSION_LOGIN_COMPLETED) - # your receiver function here - - -Or at the apps.py - -.. code-block:: python - - "signals_config": { - "lms.djangoapp": { - "relative_path": "your_module_name", - "receivers": [ - { - "receiver_func_name": "your_receiver_function", - "signal_path": "openedx_events.learning.signals.SESSION_LOGIN_COMPLETED", - }, - ], - } - }, - -In case you are listening to an event in the edx-platform repo, you can directly -use the django syntax since the apps.py method will not be available without the -plugin. - - -Sending events -^^^^^^^^^^^^^^ - -Sending events requires you to import both the event definition as well as the -attr data classes that encapsulate the event data. - -.. code-block:: python - - from openedx_events.learning.data import UserData, UserPersonalData - from openedx_events.learning.signals import STUDENT_REGISTRATION_COMPLETED - - STUDENT_REGISTRATION_COMPLETED.send_event( - user=UserData( - pii=UserPersonalData( - username=user.username, - email=user.email, - name=user.profile.name, - ), - id=user.id, - is_active=user.is_active, - ), - ) - -You can do this both from the edx-platform code as well as from an openedx -plugin. - - -Testing events -^^^^^^^^^^^^^^ - -Testing your code in CI, specially for plugins is now possible without having to -import the complete edx-platform as a dependency. - -To test your functions you need to include the openedx-events library in your -testing dependencies and make the signal connection in your test case. - -.. code-block:: python - - from openedx_events.learning.signals import STUDENT_REGISTRATION_COMPLETED - - def test_your_receiver(self): - STUDENT_REGISTRATION_COMPLETED.connect(your_function) - STUDENT_REGISTRATION_COMPLETED.send_event( - user=UserData( - pii=UserPersonalData( - username='test_username', - email='test_email@example.com', - name='test_name', - ), - id=1, - is_active=True, - ), - ) - - # run your assertions - - -Changes in the openedx-events library that are not compatible with your code -should break this kind of test in CI and let you know you need to upgrade your -code. - - -Live example -^^^^^^^^^^^^ - -For a complete and detailed example you can see the `openedx-events-2-zapier`_ -plugin. This is a fully functional plugin that connects to -``STUDENT_REGISTRATION_COMPLETED`` and ``COURSE_ENROLLMENT_CREATED`` and sends -the relevant information to zapier.com using a webhook. - -.. _openedx-events-2-zapier: https://github.com/eduNEXT/openedx-events-2-zapier - - -Index of Events ------------------ - -This list contains the events currently being sent by edx-platform. The provided -links target both the definition of the event in the openedx-events library as -well as the trigger location in this same repository. - - -Learning Events -^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 35 50 20 - - * - *Name* - - *Type* - - *Date added* - - * - `STUDENT_REGISTRATION_COMPLETED `_ - - org.openedx.learning.student.registration.completed.v1 - - `2022-06-14 `_ - - * - `SESSION_LOGIN_COMPLETED `_ - - org.openedx.learning.auth.session.login.completed.v1 - - `2022-06-14 `_ - - * - `COURSE_ENROLLMENT_CREATED `_ - - org.openedx.learning.course.enrollment.created.v1 - - `2022-06-14 `_ - - * - `COURSE_ENROLLMENT_CHANGED `_ - - org.openedx.learning.course.enrollment.changed.v1 - - `2022-06-14 `_ - - * - `COURSE_UNENROLLMENT_COMPLETED `_ - - org.openedx.learning.course.unenrollment.completed.v1 - - `2022-06-14 `_ - - * - `CERTIFICATE_CREATED `_ - - org.openedx.learning.certificate.created.v1 - - `2022-06-14 `_ - - * - `CERTIFICATE_CHANGED `_ - - org.openedx.learning.certificate.changed.v1 - - `2022-06-14 `_ - - * - `CERTIFICATE_REVOKED `_ - - org.openedx.learning.certificate.revoked.v1 - - `2022-06-14 `_ - - * - `COHORT_MEMBERSHIP_CHANGED `_ - - org.openedx.learning.cohort_membership.changed.v1 - - `2022-06-14 `_ - - * - `COURSE_DISCUSSIONS_CHANGED `_ - - org.openedx.learning.discussions.configuration.changed.v1 - - `2022-06-14 `_ - - -Content Authoring Events -^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 35 50 20 - - * - *Name* - - *Type* - - *Date added* - - * - `COURSE_CATALOG_INFO_CHANGED `_ - - org.openedx.content_authoring.course.catalog_info.changed.v1 - - `2022-08-24 `_ - - * - `XBLOCK_PUBLISHED `_ - - org.openedx.content_authoring.xblock.published.v1 - - `2022-12-06 `_ - - * - `XBLOCK_DELETED `_ - - org.openedx.content_authoring.xblock.deleted.v1 - - `2022-12-06 `_ - - * - `XBLOCK_DUPLICATED `_ - - org.openedx.content_authoring.xblock.duplicated.v1 - - `2022-12-06 `_ - - * - `XBLOCK_CREATED `_ - - org.openedx.content_authoring.xblock.created.v1 - - 2023-07-20 - - * - `XBLOCK_UPDATED `_ - - org.openedx.content_authoring.xblock.updated.v1 - - 2023-07-20 - - * - `COURSE_CREATED `_ - - org.openedx.content_authoring.course.created.v1 - - 2023-07-20 - - * - `CONTENT_LIBRARY_CREATED `_ - - org.openedx.content_authoring.content_library.created.v1 - - 2023-07-20 - - * - `CONTENT_LIBRARY_UPDATED `_ - - org.openedx.content_authoring.content_library.updated.v1 - - 2023-07-20 - - * - `CONTENT_LIBRARY_DELETED `_ - - org.openedx.content_authoring.content_library.deleted.v1 - - 2023-07-20 - - * - `LIBRARY_BLOCK_CREATED `_ - - org.openedx.content_authoring.library_block.created.v1 - - 2023-07-20 - - * - `LIBRARY_BLOCK_UPDATED `_ - - org.openedx.content_authoring.library_block.updated.v1 - - 2023-07-20 - - * - `LIBRARY_BLOCK_DELETED `_ - - org.openedx.content_authoring.library_block.deleted.v1 - - 2023-07-20 - - * - `LIBRARY_COLLECTION_CREATED `_ - - org.openedx.content_authoring.content_library.collection.created.v1 - - 2024-08-23 - - * - `LIBRARY_COLLECTION_UPDATED `_ - - org.openedx.content_authoring.content_library.collection.updated.v1 - - 2024-08-23 - - * - `LIBRARY_COLLECTION_DELETED `_ - - org.openedx.content_authoring.content_library.collection.deleted.v1 - - 2024-08-23 - - * - `CONTENT_OBJECT_ASSOCIATIONS_CHANGED `_ - - org.openedx.content_authoring.content.object.associations.changed.v1 - - 2024-09-06 diff --git a/docs/hooks/filters.rst b/docs/hooks/filters.rst deleted file mode 100644 index b2ce68fc147d..000000000000 --- a/docs/hooks/filters.rst +++ /dev/null @@ -1,191 +0,0 @@ -Open edX Filters -================ - -How to use ----------- - -Using openedx-filters in your code is very straight forward. We can consider the -two possible cases: - -Configuring a filter -^^^^^^^^^^^^^^^^^^^^ - -Implement pipeline steps -************************ - -Let's say you want to consult student's information with a third party service -before generating the students certificate. This is a common use case for filters, -where the functions part of the filter's pipeline will perform the consulting tasks and -decide the execution flow for the application. These functions are the pipeline steps, -and can be implemented in an installable Python library: - -.. code-block:: python - - # Step implementation taken from openedx-filters-samples plugin - from openedx_filters import PipelineStep - from openedx_filters.learning.filters import CertificateCreationRequested - - class StopCertificateCreation(PipelineStep): - - def run_filter(self, user, course_id, mode, status): - # Consult third party service and check if continue - # ... - # User not in third party service, denied certificate generation - raise CertificateCreationRequested.PreventCertificateCreation( - "You can't generate a certificate from this site." - ) - -There's two key components to the implementation: - -1. The filter step must be a subclass of ``PipelineStep``. - -2. The ``run_filter`` signature must match the filters definition, eg., -the previous step matches the method's definition in CertificateCreationRequested. - -Attach/hook pipeline to filter -****************************** - -After implementing the pipeline steps, we have to tell the certificate creation -filter to execute our pipeline. - -.. code-block:: python - - OPEN_EDX_FILTERS_CONFIG = { - "org.openedx.learning.certificate.creation.requested.v1": { - "fail_silently": False, - "pipeline": [ - "openedx_filters_samples.samples.pipeline.StopCertificateCreation" - ] - }, - } - -Triggering a filter -^^^^^^^^^^^^^^^^^^^ - -In order to execute a filter in your own plugin/library, you must install the -plugin where the steps are implemented and also, ``openedx-filters``. - -.. code-block:: python - - # Code taken from lms/djangoapps/certificates/generation_handler.py - from openedx_filters.learning.filters import CertificateCreationRequested - - try: - self.user, self.course_id, self.mode, self.status = CertificateCreationRequested.run_filter( - user=self.user, course_id=self.course_id, mode=self.mode, status=self.status, - ) - except CertificateCreationRequested.PreventCertificateCreation as exc: - raise CertificateGenerationNotAllowed(str(exc)) from exc - -Testing filters' steps -^^^^^^^^^^^^^^^^^^^^^^ - -It's pretty straightforward to test your pipeline steps, you'll need to include the -``openedx-filters`` library in your testing dependencies and configure them in your test case. - -.. code-block:: python - - from openedx_filters.learning.filters import CertificateCreationRequested - - @override_settings( - OPEN_EDX_FILTERS_CONFIG={ - "org.openedx.learning.certificate.creation.requested.v1": { - "fail_silently": False, - "pipeline": [ - "openedx_filters_samples.samples.pipeline.StopCertificateCreation" - ] - } - } - ) - def test_certificate_creation_requested_filter(self): - """ - Test filter triggered before the certificate creation process starts. - - Expected results: - - The pipeline step configured for the filter raises PreventCertificateCreation - when the conditions are met. - """ - with self.assertRaises(CertificateCreationRequested.PreventCertificateCreation): - CertificateCreationRequested.run_filter( - user=self.user, course_key=self.course_key, mode="audit", - ) - - # run your assertions - -Changes in the ``openedx-filters`` library that are not compatible with your code -should break this kind of test in CI and let you know you need to upgrade your code. -The main limitation while testing filters' steps it's their arguments, as they are edxapp -memory objects, but that can be solved in CI using Python mocks. - -Live example -^^^^^^^^^^^^ - -For filter steps samples you can visit the `openedx-filters-samples`_ plugin, where -you can find minimal steps exemplifying the different ways on how to use -``openedx-filters``. - -.. _openedx-filters-samples: https://github.com/eduNEXT/openedx-filters-samples - - -Index of Filters ------------------ - -This list contains the filters currently being executed by edx-platform. The provided -links target both the definition of the filter in the openedx-filters library as -well as the trigger location in this same repository. - - -.. list-table:: - :widths: 35 50 20 - - * - *Name* - - *Type* - - *Date added* - - * - `StudentRegistrationRequested `_ - - org.openedx.learning.student.registration.requested.v1 - - `2022-06-14 `_ - - * - `StudentLoginRequested `_ - - org.openedx.learning.student.login.requested.v1 - - `2022-06-14 `_ - - * - `CourseEnrollmentStarted `_ - - org.openedx.learning.course.enrollment.started.v1 - - `2022-06-14 `_ - - * - `CourseUnenrollmentStarted `_ - - org.openedx.learning.course.unenrollment.started.v1 - - `2022-06-14 `_ - - * - `CertificateCreationRequested `_ - - org.openedx.learning.certificate.creation.requested.v1 - - `2022-06-14 `_ - - * - `CertificateRenderStarted `_ - - org.openedx.learning.certificate.render.started.v1 - - `2022-06-14 `_ - - * - `CohortChangeRequested `_ - - org.openedx.learning.cohort.change.requested.v1 - - `2022-06-14 `_ - - * - `CohortAssignmentRequested `_ - - org.openedx.learning.cohort.assignment.requested.v1 - - `2022-06-14 `_ - - * - `CourseAboutRenderStarted `_ - - org.openedx.learning.course_about.render.started.v1 - - `2022-06-14 `_ - - * - `DashboardRenderStarted `_ - - org.openedx.learning.dashboard.render.started.v1 - - `2022-06-14 `_ - - * - `VerticalBlockChildRenderStarted `_ - - org.openedx.learning.veritical_block_child.render.started.v1 - - `2022-08-18 `_ - - * - `VerticalBlockRenderCompleted `_ - - org.openedx.learning.veritical_block.render.completed.v1 - - `2022-02-18 `_ diff --git a/docs/hooks/index.rst b/docs/hooks/index.rst deleted file mode 100644 index 99cb25133cd2..000000000000 --- a/docs/hooks/index.rst +++ /dev/null @@ -1,50 +0,0 @@ -Open edX Hooks Extension Framework -================================== - -To sustain the growth of the Open edX ecosystem, the business rules of the -platform must be open for extension following the open-closed principle. This -framework allows developers to do just that without needing to fork and modify -the main edx-platform repository. - - -Context -------- - -Hooks are predefined places in the edx-platform core where externally defined -functions can take place. In some cases, those functions can alter what the user -sees or experiences in the platform. Other cases are informative only. All cases -are meant to be extended using Open edX plugins and configuration. - -Hooks can be of two types, events and filters. Events are in essence signals, in -that they are sent in specific application places and whose listeners can extend -functionality. On the other hand Filters are passed data and can act on it -before this data is put back in the original application flow. In order to allow -extension developers to use the Events and Filters definitions on their plugins, -both kinds of hooks are defined in lightweight external libraries. - -* openedx-filters (`guide <./filters.rst>`_, `source code `_) -* openedx-events (`guide <./events.rst>`_, `source code `_) - -Hooks are designed with stability in mind. The main goal is that developers can -use them to change the functionality of the platform as needed and still be able -to migrate to newer open releases with very little to no development effort. In -the case of the events, this is detailed in the `versioning ADR`_ and the -`payload ADR`_. - -A longer description of the framework and it's history can be found in `OEP 50`_. - -.. _OEP 50: https://open-edx-proposals.readthedocs.io/en/latest/oep-0050-hooks-extension-framework.html -.. _versioning ADR: https://github.com/eduNEXT/openedx-events/blob/main/docs/decisions/0002-events-naming-and-versioning.rst -.. _payload ADR: https://github.com/eduNEXT/openedx-events/blob/main/docs/decisions/0003-events-payload.rst - -On the technical side events are implemented through django signals which makes -them run in the same python process as the lms or cms. Furthermore, events block -the running process. Listeners of an event are encouraged to monitor the -performance or use alternative arch patterns such as receiving the event and -defer to launching async tasks than do the slow processing. - -On the other hand, filters are implemented using a pipeline mechanism, that executes -a list of functions called ``steps`` configured through Django settings. Each -pipeline step receives a dictionary with data, process it and returns an output. During -this process, they can alter the application execution flow by halting the process -or modifying their input arguments. diff --git a/docs/index.rst b/docs/index.rst index 8d89969398bc..190ba12db906 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -20,6 +20,7 @@ locations. .. _Developer Documentation Index: https://openedx.atlassian.net/wiki/spaces/DOC/overview .. _Open edX Development space: https://openedx.atlassian.net/wiki/spaces/COMM/overview .. _Open edX ReadTheDocs: http://docs.edx.org/ +.. _Hooks Extensions Framework: https://docs.openedx.org/en/latest/developers/concepts/hooks_extension_framework.html .. toctree:: :maxdepth: 1 @@ -32,7 +33,6 @@ locations. how-tos/index references/index concepts/index - hooks/index extensions/tinymce_plugins .. grid:: 1 2 2 2 @@ -80,14 +80,16 @@ locations. :class-card: sd-shadow-md sd-p-2 :class-footer: sd-border-0 - * :doc:`hooks/index` + * `Hooks Extensions Framework`_ * :doc:`extensions/tinymce_plugins` +++ - .. button-ref:: hooks/index + .. button-link:: https://docs.openedx.org/en/latest/developers/concepts/hooks_extension_framework.html :color: primary :outline: :expand: + Hooks Extensions Framework + Change History ************** diff --git a/lms/djangoapps/bulk_email/models.py b/lms/djangoapps/bulk_email/models.py index 0e26ea559c20..d4f238fd9453 100644 --- a/lms/djangoapps/bulk_email/models.py +++ b/lms/djangoapps/bulk_email/models.py @@ -146,7 +146,7 @@ def get_users(self, course_id, user_id=None): User.objects.filter( models.Q(courseenrollment__mode=self.coursemodetarget.track.mode_slug) & enrollment_query - ) + ).exclude(id__in=staff_instructor_qset) ) else: raise ValueError(f"Unrecognized target type {self.target_type}") diff --git a/lms/djangoapps/bulk_email/signals.py b/lms/djangoapps/bulk_email/signals.py index fb8749bf45a9..8ada8760cdcf 100644 --- a/lms/djangoapps/bulk_email/signals.py +++ b/lms/djangoapps/bulk_email/signals.py @@ -1,6 +1,8 @@ """ Signal handlers for the bulk_email app """ +import logging + from django.dispatch import receiver from eventtracking import tracker @@ -10,6 +12,8 @@ from .models import Optout +log = logging.getLogger(__name__) + @receiver(USER_RETIRE_MAILINGS) def force_optout_all(sender, **kwargs): # lint-amnesty, pylint: disable=unused-argument @@ -43,7 +47,7 @@ def ace_email_sent_handler(sender, **kwargs): if not course_id: course_email = context.get('course_email', None) course_id = course_email.course_id if course_email else None - + log.info(f'Email sent for {message_name} for course {course_id} using channel {channel}') tracker.emit( 'edx.ace.message_sent', { diff --git a/lms/djangoapps/bulk_email/tests/test_models.py b/lms/djangoapps/bulk_email/tests/test_models.py index 1f7dc0c85641..43062492e842 100644 --- a/lms/djangoapps/bulk_email/tests/test_models.py +++ b/lms/djangoapps/bulk_email/tests/test_models.py @@ -15,7 +15,7 @@ from pytz import UTC from common.djangoapps.course_modes.models import CourseMode -from common.djangoapps.student.tests.factories import CourseEnrollmentFactory, UserFactory +from common.djangoapps.student.tests.factories import CourseEnrollmentFactory, UserFactory, StaffFactory from lms.djangoapps.bulk_email.api import is_bulk_email_feature_enabled from lms.djangoapps.bulk_email.models import ( SEND_TO_COHORT, @@ -25,8 +25,10 @@ CourseAuthorization, CourseEmail, CourseEmailTemplate, + CourseModeTarget, DisabledCourse, - Optout + Optout, + Target, ) from lms.djangoapps.bulk_email.models_api import is_bulk_email_disabled_for_course from lms.djangoapps.bulk_email.tests.factories import TargetFactory @@ -366,6 +368,7 @@ def setUp(self): course_id=self.course.id, user=self.user3 ) + self.staff_user = StaffFactory.create(course_key=self.course.id) self.target = TargetFactory() @override_settings(BULK_COURSE_EMAIL_LAST_LOGIN_ELIGIBILITY_PERIOD=None) @@ -391,3 +394,40 @@ def test_target_last_login_eligibility_set(self): assert result.count() == 1 assert result.filter(id=self.user1.id).exists() + + def test_filtering_of_recipients_target_for_audit_track(self): + """ + Verifies the default behavior. + + This test ensures that when the `BULK_COURSE_EMAIL_LAST_LOGIN_ELIGIBILITY_PERIOD` + setting is not defined, all users enrolled in the course are included in the results. + """ + target = Target.objects.create(target_type=SEND_TO_TRACK) + course_mode = CourseMode.objects.create( + mode_slug=CourseMode.AUDIT, + mode_display_name=CourseMode.AUDIT.capitalize(), + course_id=self.course.id, + ) + course_mode_target = CourseModeTarget.objects.create(track=course_mode) + target.coursemodetarget = course_mode_target + result = target.get_users(self.course.id) + + assert result.count() == 1 + assert result.filter(id=self.user2.id).exists() + + # Ensure staff user is not included + assert not result.filter(id=self.staff_user.id).exists() + + def test_filtering_of_recipients_target_for_staff(self): + """ + Test filtering of recipients for a target of type SEND_TO_STAFF. + + This test verifies that only staff users are returned for the given target. + It creates a target of type SEND_TO_STAFF and ensures that the correct users + are retrieved. + """ + self.target = TargetFactory(target_type=SEND_TO_STAFF) + result = self.target.get_users(self.course.id) + + assert result.count() == 1 + assert result.filter(id=self.staff_user.id).exists() diff --git a/lms/djangoapps/course_goals/management/commands/goal_reminder_email.py b/lms/djangoapps/course_goals/management/commands/goal_reminder_email.py index b49d79976c06..af48472886bb 100644 --- a/lms/djangoapps/course_goals/management/commands/goal_reminder_email.py +++ b/lms/djangoapps/course_goals/management/commands/goal_reminder_email.py @@ -1,6 +1,7 @@ """ Command to trigger sending reminder emails for learners to achieve their Course Goals """ +import time from datetime import date, datetime, timedelta from eventtracking import tracker import logging @@ -119,7 +120,11 @@ def send_ace_message(goal, session_id): with emulate_http_request(site, user): try: + start_time = time.perf_counter() ace.send(msg) + end_time = time.perf_counter() + log.info(f"Goal Reminder for {user.id} for course {goal.course_key} sent in {end_time - start_time} " + f"using {'SES' if is_ses_enabled else 'others'}") except Exception as exc: # pylint: disable=broad-except log.error(f"Goal Reminder for {user.id} for course {goal.course_key} could not send: {exc}") tracker.emit( @@ -279,7 +284,7 @@ def handle_goal(goal, today, sunday_date, monday_date, session_id): 'uuid': session_id, 'timestamp': datetime.now(), 'reason': 'User time zone', - 'user_timezone': user_timezone, + 'user_timezone': str(user_timezone), 'now_in_users_timezone': now_in_users_timezone, } ) diff --git a/lms/djangoapps/discussion/django_comment_client/base/tests.py b/lms/djangoapps/discussion/django_comment_client/base/tests.py index 62af24f0ee37..df087fdc533e 100644 --- a/lms/djangoapps/discussion/django_comment_client/base/tests.py +++ b/lms/djangoapps/discussion/django_comment_client/base/tests.py @@ -82,6 +82,7 @@ def _set_mock_request_data(self, mock_request, data): @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) +@patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) class CreateThreadGroupIdTestCase( MockRequestSetupMixin, CohortedTestCase, @@ -90,7 +91,21 @@ class CreateThreadGroupIdTestCase( ): cs_endpoint = "/threads" - def call_view(self, mock_request, commentable_id, user, group_id, pass_group_id=True): + def setUp(self): + super().setUp() + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) + + def call_view(self, mock_is_forum_v2_enabled, mock_request, commentable_id, user, group_id, pass_group_id=True): + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, {}) request_data = {"body": "body", "title": "title", "thread_type": "discussion"} if pass_group_id: @@ -105,8 +120,9 @@ def call_view(self, mock_request, commentable_id, user, group_id, pass_group_id= commentable_id=commentable_id ) - def test_group_info_in_response(self, mock_request): + def test_group_info_in_response(self, mock_is_forum_v2_enabled, mock_request): response = self.call_view( + mock_is_forum_v2_enabled, mock_request, "cohorted_topic", self.student, @@ -116,6 +132,7 @@ def test_group_info_in_response(self, mock_request): @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) +@patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) @disable_signal(views, 'thread_edited') @disable_signal(views, 'thread_voted') @disable_signal(views, 'thread_deleted') @@ -127,11 +144,18 @@ class ThreadActionGroupIdTestCase( def call_view( self, view_name, + mock_is_forum_v2_enabled, mock_request, user=None, post_params=None, view_args=None ): + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data( mock_request, { @@ -154,53 +178,58 @@ def call_view( **(view_args or {}) ) - def test_update(self, mock_request): + def test_update(self, mock_is_forum_v2_enabled, mock_request): response = self.call_view( "update_thread", + mock_is_forum_v2_enabled, mock_request, post_params={"body": "body", "title": "title"} ) self._assert_json_response_contains_group_info(response) - def test_delete(self, mock_request): - response = self.call_view("delete_thread", mock_request) + def test_delete(self, mock_is_forum_v2_enabled, mock_request): + response = self.call_view("delete_thread", mock_is_forum_v2_enabled, mock_request) self._assert_json_response_contains_group_info(response) - def test_vote(self, mock_request): + def test_vote(self, mock_is_forum_v2_enabled, mock_request): response = self.call_view( "vote_for_thread", + mock_is_forum_v2_enabled, mock_request, view_args={"value": "up"} ) self._assert_json_response_contains_group_info(response) - response = self.call_view("undo_vote_for_thread", mock_request) + response = self.call_view("undo_vote_for_thread", mock_is_forum_v2_enabled, mock_request) self._assert_json_response_contains_group_info(response) - def test_flag(self, mock_request): + def test_flag(self, mock_is_forum_v2_enabled, mock_request): with mock.patch('openedx.core.djangoapps.django_comment_common.signals.thread_flagged.send') as signal_mock: - response = self.call_view("flag_abuse_for_thread", mock_request) + response = self.call_view("flag_abuse_for_thread", mock_is_forum_v2_enabled, mock_request) self._assert_json_response_contains_group_info(response) self.assertEqual(signal_mock.call_count, 1) - response = self.call_view("un_flag_abuse_for_thread", mock_request) + response = self.call_view("un_flag_abuse_for_thread", mock_is_forum_v2_enabled, mock_request) self._assert_json_response_contains_group_info(response) - def test_pin(self, mock_request): + def test_pin(self, mock_is_forum_v2_enabled, mock_request): response = self.call_view( "pin_thread", + mock_is_forum_v2_enabled, mock_request, user=self.moderator ) self._assert_json_response_contains_group_info(response) response = self.call_view( "un_pin_thread", + mock_is_forum_v2_enabled, mock_request, user=self.moderator ) self._assert_json_response_contains_group_info(response) - def test_openclose(self, mock_request): + def test_openclose(self, mock_is_forum_v2_enabled, mock_request): response = self.call_view( "openclose_thread", + mock_is_forum_v2_enabled, mock_request, user=self.moderator ) @@ -280,10 +309,11 @@ def _setup_mock_request(self, mock_request, include_depth=False): data["depth"] = 0 self._set_mock_request_data(mock_request, data) - def create_thread_helper(self, mock_request, extra_request_data=None, extra_response_data=None): + def create_thread_helper(self, mock_is_forum_v2_enabled, mock_request, extra_request_data=None, extra_response_data=None): """ Issues a request to create a thread and verifies the result. """ + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, { "thread_type": "discussion", "title": "Hello", @@ -350,10 +380,11 @@ def create_thread_helper(self, mock_request, extra_request_data=None, extra_resp ) assert response.status_code == 200 - def update_thread_helper(self, mock_request): + def update_thread_helper(self, mock_is_forum_v2_enabled, mock_request): """ Issues a request to update a thread and verifies the result. """ + mock_is_forum_v2_enabled.return_value = False self._setup_mock_request(mock_request) # Mock out saving in order to test that content is correctly # updated. Otherwise, the call to thread.save() receives the @@ -376,6 +407,7 @@ def update_thread_helper(self, mock_request): @ddt.ddt @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) +@patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) @disable_signal(views, 'thread_created') @disable_signal(views, 'thread_edited') class ViewsQueryCountTestCase( @@ -393,6 +425,11 @@ class ViewsQueryCountTestCase( @patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) def setUp(self): super().setUp() + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) def count_queries(func): # pylint: disable=no-self-argument """ @@ -414,22 +451,23 @@ def inner(self, default_store, block_count, mongo_calls, sql_queries, *args, **k ) @ddt.unpack @count_queries - def test_create_thread(self, mock_request): - self.create_thread_helper(mock_request) + def test_create_thread(self, mock_is_forum_v2_enabled, mock_request): + self.create_thread_helper(mock_is_forum_v2_enabled, mock_request) @ddt.data( (ModuleStoreEnum.Type.split, 3, 6, 41), ) @ddt.unpack @count_queries - def test_update_thread(self, mock_request): - self.update_thread_helper(mock_request) + def test_update_thread(self, mock_is_forum_v2_enabled, mock_request): + self.update_thread_helper(mock_is_forum_v2_enabled, mock_request) @ddt.ddt @disable_signal(views, 'comment_flagged') @disable_signal(views, 'thread_flagged') @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) +@patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) class ViewsTestCase( ForumsEnableMixin, UrlResetMixin, @@ -464,7 +502,16 @@ def setUp(self): # so we need to call super.setUp() which reloads urls.py (because # of the UrlResetMixin) super().setUp() - + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) # Patch the comment client user save method so it does not try # to create a new cc user when creating a django user with patch('common.djangoapps.student.models.user.cc.User.save'): @@ -497,11 +544,11 @@ def assert_discussion_signals(self, signal, user=None): with self.assert_signal_sent(views, signal, sender=None, user=user, exclude_args=('post',)): yield - def test_create_thread(self, mock_request): + def test_create_thread(self, mock_is_forum_v2_enabled, mock_request): with self.assert_discussion_signals('thread_created'): - self.create_thread_helper(mock_request) + self.create_thread_helper(mock_is_forum_v2_enabled, mock_request) - def test_create_thread_standalone(self, mock_request): + def test_create_thread_standalone(self, mock_is_forum_v2_enabled, mock_request): team = CourseTeamFactory.create( name="A Team", course_id=self.course_id, @@ -513,15 +560,15 @@ def test_create_thread_standalone(self, mock_request): team.add_user(self.student) # create_thread_helper verifies that extra data are passed through to the comments service - self.create_thread_helper(mock_request, extra_response_data={'context': ThreadContext.STANDALONE}) + self.create_thread_helper(mock_is_forum_v2_enabled, mock_request, extra_response_data={'context': ThreadContext.STANDALONE}) @ddt.data( ('follow_thread', 'thread_followed'), ('unfollow_thread', 'thread_unfollowed'), ) @ddt.unpack - def test_follow_unfollow_thread_signals(self, view_name, signal, mock_request): - self.create_thread_helper(mock_request) + def test_follow_unfollow_thread_signals(self, view_name, signal, mock_is_forum_v2_enabled, mock_request): + self.create_thread_helper(mock_is_forum_v2_enabled, mock_request) with self.assert_discussion_signals(signal): response = self.client.post( @@ -532,7 +579,8 @@ def test_follow_unfollow_thread_signals(self, view_name, signal, mock_request): ) assert response.status_code == 200 - def test_delete_thread(self, mock_request): + def test_delete_thread(self, mock_is_forum_v2_enabled, mock_request): + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, { "user_id": str(self.student.id), "closed": False, @@ -551,7 +599,8 @@ def test_delete_thread(self, mock_request): assert response.status_code == 200 assert mock_request.called - def test_delete_comment(self, mock_request): + def test_delete_comment(self, mock_is_forum_v2_enabled, mock_request): + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, { "user_id": str(self.student.id), "closed": False, @@ -573,12 +622,13 @@ def test_delete_comment(self, mock_request): assert args[0] == 'delete' assert args[1].endswith(f"/{test_comment_id}") - def _test_request_error(self, view_name, view_kwargs, data, mock_request): + def _test_request_error(self, view_name, view_kwargs, data, mock_is_forum_v2_enabled, mock_request): """ Submit a request against the given view with the given data and ensure that the result is a 400 error and that no data was posted using mock_request """ + mock_is_forum_v2_enabled.return_value = False self._setup_mock_request(mock_request, include_depth=(view_name == "create_sub_comment")) response = self.client.post(reverse(view_name, kwargs=view_kwargs), data=data) @@ -586,87 +636,97 @@ def _test_request_error(self, view_name, view_kwargs, data, mock_request): for call in mock_request.call_args_list: assert call[0][0].lower() == 'get' - def test_create_thread_no_title(self, mock_request): + def test_create_thread_no_title(self, mock_is_forum_v2_enabled, mock_request): self._test_request_error( "create_thread", {"commentable_id": "dummy", "course_id": str(self.course_id)}, {"body": "foo"}, + mock_is_forum_v2_enabled, mock_request ) - def test_create_thread_empty_title(self, mock_request): + def test_create_thread_empty_title(self, mock_is_forum_v2_enabled, mock_request): self._test_request_error( "create_thread", {"commentable_id": "dummy", "course_id": str(self.course_id)}, {"body": "foo", "title": " "}, + mock_is_forum_v2_enabled, mock_request ) - def test_create_thread_no_body(self, mock_request): + def test_create_thread_no_body(self, mock_is_forum_v2_enabled, mock_request): self._test_request_error( "create_thread", {"commentable_id": "dummy", "course_id": str(self.course_id)}, {"title": "foo"}, + mock_is_forum_v2_enabled, mock_request ) - def test_create_thread_empty_body(self, mock_request): + def test_create_thread_empty_body(self, mock_is_forum_v2_enabled, mock_request): self._test_request_error( "create_thread", {"commentable_id": "dummy", "course_id": str(self.course_id)}, {"body": " ", "title": "foo"}, + mock_is_forum_v2_enabled, mock_request ) - def test_update_thread_no_title(self, mock_request): + def test_update_thread_no_title(self, mock_is_forum_v2_enabled, mock_request): self._test_request_error( "update_thread", {"thread_id": "dummy", "course_id": str(self.course_id)}, {"body": "foo"}, + mock_is_forum_v2_enabled, mock_request ) - def test_update_thread_empty_title(self, mock_request): + def test_update_thread_empty_title(self, mock_is_forum_v2_enabled, mock_request): self._test_request_error( "update_thread", {"thread_id": "dummy", "course_id": str(self.course_id)}, {"body": "foo", "title": " "}, + mock_is_forum_v2_enabled, mock_request ) - def test_update_thread_no_body(self, mock_request): + def test_update_thread_no_body(self, mock_is_forum_v2_enabled, mock_request): self._test_request_error( "update_thread", {"thread_id": "dummy", "course_id": str(self.course_id)}, {"title": "foo"}, + mock_is_forum_v2_enabled, mock_request ) - def test_update_thread_empty_body(self, mock_request): + def test_update_thread_empty_body(self, mock_is_forum_v2_enabled, mock_request): self._test_request_error( "update_thread", {"thread_id": "dummy", "course_id": str(self.course_id)}, {"body": " ", "title": "foo"}, + mock_is_forum_v2_enabled, mock_request ) - def test_update_thread_course_topic(self, mock_request): + def test_update_thread_course_topic(self, mock_is_forum_v2_enabled, mock_request): with self.assert_discussion_signals('thread_edited'): - self.update_thread_helper(mock_request) + self.update_thread_helper(mock_is_forum_v2_enabled, mock_request) @patch( 'lms.djangoapps.discussion.django_comment_client.utils.get_discussion_categories_ids', return_value=["test_commentable"], ) - def test_update_thread_wrong_commentable_id(self, mock_get_discussion_id_map, mock_request): + def test_update_thread_wrong_commentable_id(self, mock_get_discussion_id_map, mock_is_forum_v2_enabled, mock_request): self._test_request_error( "update_thread", {"thread_id": "dummy", "course_id": str(self.course_id)}, {"body": "foo", "title": "foo", "commentable_id": "wrong_commentable"}, + mock_is_forum_v2_enabled, mock_request ) - def test_create_comment(self, mock_request): + def test_create_comment(self, mock_is_forum_v2_enabled, mock_request): + mock_is_forum_v2_enabled.return_value = False self._setup_mock_request(mock_request) with self.assert_discussion_signals('comment_created'): response = self.client.post( @@ -678,55 +738,62 @@ def test_create_comment(self, mock_request): ) assert response.status_code == 200 - def test_create_comment_no_body(self, mock_request): + def test_create_comment_no_body(self, mock_is_forum_v2_enabled, mock_request): self._test_request_error( "create_comment", {"thread_id": "dummy", "course_id": str(self.course_id)}, {}, + mock_is_forum_v2_enabled, mock_request ) - def test_create_comment_empty_body(self, mock_request): + def test_create_comment_empty_body(self, mock_is_forum_v2_enabled, mock_request): self._test_request_error( "create_comment", {"thread_id": "dummy", "course_id": str(self.course_id)}, {"body": " "}, + mock_is_forum_v2_enabled, mock_request ) - def test_create_sub_comment_no_body(self, mock_request): + def test_create_sub_comment_no_body(self, mock_is_forum_v2_enabled, mock_request): self._test_request_error( "create_sub_comment", {"comment_id": "dummy", "course_id": str(self.course_id)}, {}, + mock_is_forum_v2_enabled, mock_request ) - def test_create_sub_comment_empty_body(self, mock_request): + def test_create_sub_comment_empty_body(self, mock_is_forum_v2_enabled, mock_request): self._test_request_error( "create_sub_comment", {"comment_id": "dummy", "course_id": str(self.course_id)}, {"body": " "}, + mock_is_forum_v2_enabled, mock_request ) - def test_update_comment_no_body(self, mock_request): + def test_update_comment_no_body(self, mock_is_forum_v2_enabled, mock_request): self._test_request_error( "update_comment", {"comment_id": "dummy", "course_id": str(self.course_id)}, {}, + mock_is_forum_v2_enabled, mock_request ) - def test_update_comment_empty_body(self, mock_request): + def test_update_comment_empty_body(self, mock_is_forum_v2_enabled, mock_request): self._test_request_error( "update_comment", {"comment_id": "dummy", "course_id": str(self.course_id)}, {"body": " "}, + mock_is_forum_v2_enabled, mock_request ) - def test_update_comment_basic(self, mock_request): + def test_update_comment_basic(self, mock_is_forum_v2_enabled, mock_request): + mock_is_forum_v2_enabled.return_value = False self._setup_mock_request(mock_request) comment_id = "test_comment_id" updated_body = "updated body" @@ -748,13 +815,14 @@ def test_update_comment_basic(self, mock_request): data={"body": updated_body} ) - def test_flag_thread_open(self, mock_request): - self.flag_thread(mock_request, False) + def test_flag_thread_open(self, mock_is_forum_v2_enabled, mock_request): + self.flag_thread(mock_is_forum_v2_enabled, mock_request, False) - def test_flag_thread_close(self, mock_request): - self.flag_thread(mock_request, True) + def test_flag_thread_close(self, mock_is_forum_v2_enabled, mock_request): + self.flag_thread(mock_is_forum_v2_enabled, mock_request, True) - def flag_thread(self, mock_request, is_closed): + def flag_thread(self, mock_is_forum_v2_enabled, mock_request, is_closed): + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, { "title": "Hello", "body": "this is a post", @@ -826,13 +894,14 @@ def flag_thread(self, mock_request, is_closed): assert response.status_code == 200 - def test_un_flag_thread_open(self, mock_request): - self.un_flag_thread(mock_request, False) + def test_un_flag_thread_open(self, mock_is_forum_v2_enabled, mock_request): + self.un_flag_thread(mock_is_forum_v2_enabled, mock_request, False) - def test_un_flag_thread_close(self, mock_request): - self.un_flag_thread(mock_request, True) + def test_un_flag_thread_close(self, mock_is_forum_v2_enabled, mock_request): + self.un_flag_thread(mock_is_forum_v2_enabled, mock_request, True) - def un_flag_thread(self, mock_request, is_closed): + def un_flag_thread(self, mock_is_forum_v2_enabled, mock_request, is_closed): + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, { "title": "Hello", "body": "this is a post", @@ -905,13 +974,14 @@ def un_flag_thread(self, mock_request, is_closed): assert response.status_code == 200 - def test_flag_comment_open(self, mock_request): - self.flag_comment(mock_request, False) + def test_flag_comment_open(self, mock_is_forum_v2_enabled, mock_request): + self.flag_comment(mock_is_forum_v2_enabled, mock_request, False) - def test_flag_comment_close(self, mock_request): - self.flag_comment(mock_request, True) + def test_flag_comment_close(self, mock_is_forum_v2_enabled, mock_request): + self.flag_comment(mock_is_forum_v2_enabled, mock_request, True) - def flag_comment(self, mock_request, is_closed): + def flag_comment(self, mock_is_forum_v2_enabled, mock_request, is_closed): + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, { "body": "this is a comment", "course_id": "MITx/999/Robot_Super_Course", @@ -976,13 +1046,14 @@ def flag_comment(self, mock_request, is_closed): assert response.status_code == 200 - def test_un_flag_comment_open(self, mock_request): - self.un_flag_comment(mock_request, False) + def test_un_flag_comment_open(self, mock_is_forum_v2_enabled, mock_request): + self.un_flag_comment(mock_is_forum_v2_enabled, mock_request, False) - def test_un_flag_comment_close(self, mock_request): - self.un_flag_comment(mock_request, True) + def test_un_flag_comment_close(self, mock_is_forum_v2_enabled, mock_request): + self.un_flag_comment(mock_is_forum_v2_enabled, mock_request, True) - def un_flag_comment(self, mock_request, is_closed): + def un_flag_comment(self, mock_is_forum_v2_enabled, mock_request, is_closed): + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, { "body": "this is a comment", "course_id": "MITx/999/Robot_Super_Course", @@ -1054,7 +1125,8 @@ def un_flag_comment(self, mock_request, is_closed): ('downvote_comment', 'comment_id', 'comment_voted') ) @ddt.unpack - def test_voting(self, view_name, item_id, signal, mock_request): + def test_voting(self, view_name, item_id, signal, mock_is_forum_v2_enabled, mock_request): + mock_is_forum_v2_enabled.return_value = False self._setup_mock_request(mock_request) with self.assert_discussion_signals(signal): response = self.client.post( @@ -1065,7 +1137,8 @@ def test_voting(self, view_name, item_id, signal, mock_request): ) assert response.status_code == 200 - def test_endorse_comment(self, mock_request): + def test_endorse_comment(self, mock_is_forum_v2_enabled, mock_request): + mock_is_forum_v2_enabled.return_value = False self._setup_mock_request(mock_request) self.client.login(username=self.moderator.username, password=self.password) with self.assert_discussion_signals('comment_endorsed', user=self.moderator): @@ -1079,6 +1152,7 @@ def test_endorse_comment(self, mock_request): @patch("openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request", autospec=True) +@patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) @disable_signal(views, 'comment_endorsed') class ViewPermissionsTestCase(ForumsEnableMixin, UrlResetMixin, SharedModuleStoreTestCase, MockRequestSetupMixin): @@ -1106,8 +1180,19 @@ def setUpTestData(cls): @patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) def setUp(self): super().setUp() + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) - def test_pin_thread_as_student(self, mock_request): + def test_pin_thread_as_student(self, mock_is_forum_v2_enabled, mock_request): + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, {}) self.client.login(username=self.student.username, password=self.password) response = self.client.post( @@ -1115,7 +1200,8 @@ def test_pin_thread_as_student(self, mock_request): ) assert response.status_code == 401 - def test_pin_thread_as_moderator(self, mock_request): + def test_pin_thread_as_moderator(self, mock_is_forum_v2_enabled, mock_request): + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, {}) self.client.login(username=self.moderator.username, password=self.password) response = self.client.post( @@ -1123,7 +1209,8 @@ def test_pin_thread_as_moderator(self, mock_request): ) assert response.status_code == 200 - def test_un_pin_thread_as_student(self, mock_request): + def test_un_pin_thread_as_student(self, mock_is_forum_v2_enabled, mock_request): + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, {}) self.client.login(username=self.student.username, password=self.password) response = self.client.post( @@ -1131,7 +1218,8 @@ def test_un_pin_thread_as_student(self, mock_request): ) assert response.status_code == 401 - def test_un_pin_thread_as_moderator(self, mock_request): + def test_un_pin_thread_as_moderator(self, mock_is_forum_v2_enabled, mock_request): + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, {}) self.client.login(username=self.moderator.username, password=self.password) response = self.client.post( @@ -1139,7 +1227,7 @@ def test_un_pin_thread_as_moderator(self, mock_request): ) assert response.status_code == 200 - def _set_mock_request_thread_and_comment(self, mock_request, thread_data, comment_data): + def _set_mock_request_thread_and_comment(self, mock_is_forum_v2_enabled, mock_request, thread_data, comment_data): def handle_request(*args, **kwargs): url = args[1] if "/threads/" in url: @@ -1148,10 +1236,12 @@ def handle_request(*args, **kwargs): return self._create_response_mock(comment_data) else: raise ArgumentError("Bad url to mock request") + mock_is_forum_v2_enabled.return_value = False mock_request.side_effect = handle_request - def test_endorse_response_as_staff(self, mock_request): + def test_endorse_response_as_staff(self, mock_is_forum_v2_enabled, mock_request): self._set_mock_request_thread_and_comment( + mock_is_forum_v2_enabled, mock_request, {"type": "thread", "thread_type": "question", "user_id": str(self.student.id), "commentable_id": "course"}, {"type": "comment", "thread_id": "dummy"} @@ -1162,8 +1252,9 @@ def test_endorse_response_as_staff(self, mock_request): ) assert response.status_code == 200 - def test_endorse_response_as_student(self, mock_request): + def test_endorse_response_as_student(self, mock_is_forum_v2_enabled, mock_request): self._set_mock_request_thread_and_comment( + mock_is_forum_v2_enabled, mock_request, {"type": "thread", "thread_type": "question", "user_id": str(self.moderator.id), "commentable_id": "course"}, @@ -1175,8 +1266,9 @@ def test_endorse_response_as_student(self, mock_request): ) assert response.status_code == 401 - def test_endorse_response_as_student_question_author(self, mock_request): + def test_endorse_response_as_student_question_author(self, mock_is_forum_v2_enabled, mock_request): self._set_mock_request_thread_and_comment( + mock_is_forum_v2_enabled, mock_request, {"type": "thread", "thread_type": "question", "user_id": str(self.student.id), "commentable_id": "course"}, {"type": "comment", "thread_id": "dummy"} @@ -1209,10 +1301,12 @@ def setUpTestData(cls): CourseEnrollmentFactory(user=cls.student, course_id=cls.course.id) @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) - def _test_unicode_data(self, text, mock_request,): + @patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) + def _test_unicode_data(self, text, mock_is_forum_v2_enabled, mock_request,): """ Test to make sure unicode data in a thread doesn't break it. """ + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, {}) request = RequestFactory().post("dummy_url", {"thread_type": "discussion", "body": text, "title": text}) request.user = self.student @@ -1235,6 +1329,13 @@ class UpdateThreadUnicodeTestCase( UnicodeTestMixin, MockRequestSetupMixin ): + def setUp(self): + super().setUp() + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) @classmethod def setUpClass(cls): @@ -1255,7 +1356,9 @@ def setUpTestData(cls): return_value=["test_commentable"], ) @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) - def _test_unicode_data(self, text, mock_request, mock_get_discussion_id_map): + @patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) + def _test_unicode_data(self, text, mock_is_forum_v2_enabled, mock_request, mock_get_discussion_id_map): + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, { "user_id": str(self.student.id), "closed": False, @@ -1280,6 +1383,13 @@ class CreateCommentUnicodeTestCase( UnicodeTestMixin, MockRequestSetupMixin ): + def setUp(self): + super().setUp() + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) @classmethod def setUpClass(cls): @@ -1296,7 +1406,9 @@ def setUpTestData(cls): CourseEnrollmentFactory(user=cls.student, course_id=cls.course.id) @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) - def _test_unicode_data(self, text, mock_request): + @patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) + def _test_unicode_data(self, text, mock_is_forum_v2_enabled, mock_request): + mock_is_forum_v2_enabled.return_value = False commentable_id = "non_team_dummy_id" self._set_mock_request_data(mock_request, { "closed": False, @@ -1327,6 +1439,13 @@ class UpdateCommentUnicodeTestCase( UnicodeTestMixin, MockRequestSetupMixin ): + def setUp(self): + super().setUp() + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) @classmethod def setUpClass(cls): @@ -1343,7 +1462,9 @@ def setUpTestData(cls): CourseEnrollmentFactory(user=cls.student, course_id=cls.course.id) @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) - def _test_unicode_data(self, text, mock_request): + @patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) + def _test_unicode_data(self, text, mock_is_forum_v2_enabled, mock_request): + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, { "user_id": str(self.student.id), "closed": False, @@ -1359,6 +1480,7 @@ def _test_unicode_data(self, text, mock_request): @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) +@patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) class CommentActionTestCase( MockRequestSetupMixin, CohortedTestCase, @@ -1367,11 +1489,18 @@ class CommentActionTestCase( def call_view( self, view_name, + mock_is_forum_v2_enabled, mock_request, user=None, post_params=None, view_args=None ): + mock_is_forum_v2_enabled.return_value = False + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) self._set_mock_request_data( mock_request, { @@ -1394,9 +1523,9 @@ def call_view( **(view_args or {}) ) - def test_flag(self, mock_request): + def test_flag(self, mock_is_forum_v2_enabled, mock_request): with mock.patch('openedx.core.djangoapps.django_comment_common.signals.comment_flagged.send') as signal_mock: - self.call_view("flag_abuse_for_comment", mock_request) + self.call_view("flag_abuse_for_comment", mock_is_forum_v2_enabled, mock_request) self.assertEqual(signal_mock.call_count, 1) @@ -1410,6 +1539,14 @@ class CreateSubCommentUnicodeTestCase( """ Make sure comments under a response can handle unicode. """ + def setUp(self): + super().setUp() + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) + @classmethod def setUpClass(cls): # pylint: disable=super-method-not-called @@ -1425,10 +1562,12 @@ def setUpTestData(cls): CourseEnrollmentFactory(user=cls.student, course_id=cls.course.id) @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) - def _test_unicode_data(self, text, mock_request): + @patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) + def _test_unicode_data(self, text, mock_is_forum_v2_enabled, mock_request): """ Create a comment with unicode in it. """ + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, { "closed": False, "depth": 1, @@ -1453,6 +1592,7 @@ def _test_unicode_data(self, text, mock_request): @ddt.ddt @patch("openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request", autospec=True) +@patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) @disable_signal(views, 'thread_voted') @disable_signal(views, 'thread_edited') @disable_signal(views, 'comment_created') @@ -1562,13 +1702,24 @@ def create_users_and_enroll(coursemode): users=[cls.group_moderator, cls.cohorted] ) - @patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) + @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) def setUp(self): super().setUp() - def _setup_mock(self, user, mock_request, data): + def _setup_mock(self, user, mock_is_forum_v2_enabled, mock_request, data): user = getattr(self, user) + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, data) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) self.client.login(username=user.username, password=self.password) @ddt.data( @@ -1593,7 +1744,7 @@ def _setup_mock(self, user, mock_request, data): ('group_moderator', 'cohorted', 'course_commentable_id', 401, CourseDiscussionSettings.NONE) ) @ddt.unpack - def test_update_thread(self, user, thread_author, commentable_id, status_code, division_scheme, mock_request): + def test_update_thread(self, user, thread_author, commentable_id, status_code, division_scheme, mock_is_forum_v2_enabled, mock_request): """ Verify that update_thread is limited to thread authors and privileged users (team membership does not matter). """ @@ -1603,7 +1754,7 @@ def test_update_thread(self, user, thread_author, commentable_id, status_code, d thread_author = getattr(self, thread_author) self._setup_mock( - user, mock_request, # user is the person making the request. + user, mock_is_forum_v2_enabled, mock_request, # user is the person making the request. { "user_id": str(thread_author.id), "closed": False, "commentable_id": commentable_id, @@ -1643,12 +1794,12 @@ def test_update_thread(self, user, thread_author, commentable_id, status_code, d ('group_moderator', 'cohorted', 'team_commentable_id', 401, CourseDiscussionSettings.NONE) ) @ddt.unpack - def test_delete_comment(self, user, comment_author, commentable_id, status_code, division_scheme, mock_request): + def test_delete_comment(self, user, comment_author, commentable_id, status_code, division_scheme, mock_is_forum_v2_enabled, mock_request): commentable_id = getattr(self, commentable_id) comment_author = getattr(self, comment_author) self.change_divided_discussion_settings(division_scheme) - self._setup_mock(user, mock_request, { + self._setup_mock(user, mock_is_forum_v2_enabled, mock_request, { "closed": False, "commentable_id": commentable_id, "user_id": str(comment_author.id), @@ -1671,12 +1822,12 @@ def test_delete_comment(self, user, comment_author, commentable_id, status_code, @ddt.data(*ddt_permissions_args) @ddt.unpack - def test_create_comment(self, user, commentable_id, status_code, mock_request): + def test_create_comment(self, user, commentable_id, status_code, mock_is_forum_v2_enabled, mock_request): """ Verify that create_comment is limited to members of the team or users with 'edit_content' permission. """ commentable_id = getattr(self, commentable_id) - self._setup_mock(user, mock_request, {"closed": False, "commentable_id": commentable_id}) + self._setup_mock(user, mock_is_forum_v2_enabled, mock_request, {"closed": False, "commentable_id": commentable_id}) response = self.client.post( reverse( @@ -1692,13 +1843,13 @@ def test_create_comment(self, user, commentable_id, status_code, mock_request): @ddt.data(*ddt_permissions_args) @ddt.unpack - def test_create_sub_comment(self, user, commentable_id, status_code, mock_request): + def test_create_sub_comment(self, user, commentable_id, status_code, mock_is_forum_v2_enabled, mock_request): """ Verify that create_subcomment is limited to members of the team or users with 'edit_content' permission. """ commentable_id = getattr(self, commentable_id) self._setup_mock( - user, mock_request, + user, mock_is_forum_v2_enabled, mock_request, {"closed": False, "commentable_id": commentable_id, "thread_id": "dummy_thread"}, ) response = self.client.post( @@ -1715,14 +1866,14 @@ def test_create_sub_comment(self, user, commentable_id, status_code, mock_reques @ddt.data(*ddt_permissions_args) @ddt.unpack - def test_comment_actions(self, user, commentable_id, status_code, mock_request): + def test_comment_actions(self, user, commentable_id, status_code, mock_is_forum_v2_enabled, mock_request): """ Verify that voting and flagging of comments is limited to members of the team or users with 'edit_content' permission. """ commentable_id = getattr(self, commentable_id) self._setup_mock( - user, mock_request, + user, mock_is_forum_v2_enabled, mock_request, { "closed": False, "commentable_id": commentable_id, @@ -1742,14 +1893,14 @@ def test_comment_actions(self, user, commentable_id, status_code, mock_request): @ddt.data(*ddt_permissions_args) @ddt.unpack - def test_threads_actions(self, user, commentable_id, status_code, mock_request): + def test_threads_actions(self, user, commentable_id, status_code, mock_is_forum_v2_enabled, mock_request): """ Verify that voting, flagging, and following of threads is limited to members of the team or users with 'edit_content' permission. """ commentable_id = getattr(self, commentable_id) self._setup_mock( - user, mock_request, + user, mock_is_forum_v2_enabled, mock_request, {"closed": False, "commentable_id": commentable_id, "body": "dummy body", "course_id": str(self.course.id)} ) for action in ["upvote_thread", "downvote_thread", "un_flag_abuse_for_thread", "flag_abuse_for_thread", @@ -1772,6 +1923,19 @@ class ForumEventTestCase(ForumsEnableMixin, SharedModuleStoreTestCase, MockReque """ Forum actions are expected to launch analytics events. Test these here. """ + def setUp(self): + super().setUp() + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) + @classmethod def setUpClass(cls): # pylint: disable=super-method-not-called @@ -1791,12 +1955,14 @@ def setUpTestData(cls): @patch('eventtracking.tracker.emit') @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) - def test_response_event(self, mock_request, mock_emit): + @patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) + def test_response_event(self, mock_is_forum_v2_enabled, mock_request, mock_emit): """ Check to make sure an event is fired when a user responds to a thread. """ event_receiver = Mock() FORUM_THREAD_RESPONSE_CREATED.connect(event_receiver) + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, { "closed": False, "commentable_id": 'test_commentable_id', @@ -1833,12 +1999,14 @@ def test_response_event(self, mock_request, mock_emit): @patch('eventtracking.tracker.emit') @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) - def test_comment_event(self, mock_request, mock_emit): + @patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) + def test_comment_event(self, mock_is_forum_v2_enabled, mock_request, mock_emit): """ Ensure an event is fired when someone comments on a response. """ event_receiver = Mock() FORUM_RESPONSE_COMMENT_CREATED.connect(event_receiver) + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, { "closed": False, "depth": 1, @@ -1875,6 +2043,7 @@ def test_comment_event(self, mock_request, mock_emit): @patch('eventtracking.tracker.emit') @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) + @patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) @ddt.data(( 'create_thread', 'edx.forum.thread.created', { @@ -1896,7 +2065,7 @@ def test_comment_event(self, mock_request, mock_emit): {'comment_id': 'dummy_comment_id'} )) @ddt.unpack - def test_team_events(self, view_name, event_name, view_data, view_kwargs, mock_request, mock_emit): + def test_team_events(self, view_name, event_name, view_data, view_kwargs, mock_is_forum_v2_enabled, mock_request, mock_emit): user = self.student team = CourseTeamFactory.create(discussion_topic_id=TEAM_COMMENTABLE_ID) CourseTeamMembershipFactory.create(team=team, user=user) @@ -1905,6 +2074,7 @@ def test_team_events(self, view_name, event_name, view_data, view_kwargs, mock_r forum_event = views.TRACKING_LOG_TO_EVENT_MAPS.get(event_name) forum_event.connect(event_receiver) + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, { 'closed': False, 'commentable_id': TEAM_COMMENTABLE_ID, @@ -1943,9 +2113,11 @@ def test_team_events(self, view_name, event_name, view_data, view_kwargs, mock_r @ddt.unpack @patch('eventtracking.tracker.emit') @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) - def test_thread_voted_event(self, view_name, obj_id_name, obj_type, mock_request, mock_emit): + @patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) + def test_thread_voted_event(self, view_name, obj_id_name, obj_type, mock_is_forum_v2_enabled, mock_request, mock_emit): undo = view_name.startswith('undo') + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, { 'closed': False, 'commentable_id': 'test_commentable_id', @@ -1971,11 +2143,13 @@ def test_thread_voted_event(self, view_name, obj_id_name, obj_type, mock_request @ddt.data('follow_thread', 'unfollow_thread',) @patch('eventtracking.tracker.emit') @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) - def test_thread_followed_event(self, view_name, mock_request, mock_emit): + @patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) + def test_thread_followed_event(self, view_name, mock_is_forum_v2_enabled, mock_request, mock_emit): event_receiver = Mock() for signal in views.TRACKING_LOG_TO_EVENT_MAPS.values(): signal.connect(event_receiver) + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, { 'closed': False, 'commentable_id': 'test_commentable_id', @@ -2025,10 +2199,11 @@ def setUpTestData(cls): cls.other_user = UserFactory.create(username="other") CourseEnrollmentFactory(user=cls.other_user, course_id=cls.course.id) - def set_post_counts(self, mock_request, threads_count=1, comments_count=1): + def set_post_counts(self, mock_is_forum_v2_enabled, mock_request, threads_count=1, comments_count=1): """ sets up a mock response from the comments service for getting post counts for our other_user """ + mock_is_forum_v2_enabled.return_value = False self._set_mock_request_data(mock_request, { "threads_count": threads_count, "comments_count": comments_count, @@ -2042,15 +2217,17 @@ def make_request(self, method='get', course_id=None, **kwargs): return views.users(request, course_id=str(course_id)) @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) - def test_finds_exact_match(self, mock_request): - self.set_post_counts(mock_request) + @patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) + def test_finds_exact_match(self, mock_is_forum_v2_enabled, mock_request): + self.set_post_counts(mock_is_forum_v2_enabled, mock_request) response = self.make_request(username="other") assert response.status_code == 200 assert json.loads(response.content.decode('utf-8'))['users'] == [{'id': self.other_user.id, 'username': self.other_user.username}] @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) - def test_finds_no_match(self, mock_request): - self.set_post_counts(mock_request) + @patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) + def test_finds_no_match(self, mock_is_forum_v2_enabled, mock_request): + self.set_post_counts(mock_is_forum_v2_enabled, mock_request) response = self.make_request(username="othor") assert response.status_code == 200 assert json.loads(response.content.decode('utf-8'))['users'] == [] @@ -2086,8 +2263,9 @@ def test_requires_requestor_enrolled_in_course(self): assert 'users' not in content @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) - def test_requires_matched_user_has_forum_content(self, mock_request): - self.set_post_counts(mock_request, 0, 0) + @patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) + def test_requires_matched_user_has_forum_content(self, mock_is_forum_v2_enabled, mock_request): + self.set_post_counts(mock_is_forum_v2_enabled, mock_request, 0, 0) response = self.make_request(username="other") assert response.status_code == 200 assert json.loads(response.content.decode('utf-8'))['users'] == [] diff --git a/lms/djangoapps/discussion/django_comment_client/base/views.py b/lms/djangoapps/discussion/django_comment_client/base/views.py index e3e52a5400a4..3df362bdf6d2 100644 --- a/lms/djangoapps/discussion/django_comment_client/base/views.py +++ b/lms/djangoapps/discussion/django_comment_client/base/views.py @@ -562,7 +562,6 @@ def create_thread(request, course_id, commentable_id): params['context'] = ThreadContext.STANDALONE else: params['context'] = ThreadContext.COURSE - thread = cc.Thread(**params) # Divide the thread if required diff --git a/lms/djangoapps/discussion/django_comment_client/tests/group_id.py b/lms/djangoapps/discussion/django_comment_client/tests/group_id.py index 78853293ec46..0a5fbe491930 100644 --- a/lms/djangoapps/discussion/django_comment_client/tests/group_id.py +++ b/lms/djangoapps/discussion/django_comment_client/tests/group_id.py @@ -60,51 +60,76 @@ class CohortedTopicGroupIdTestMixin(GroupIdAssertionMixin): Provides test cases to verify that views pass the correct `group_id` to the comments service when requesting content in cohorted discussions. """ - def call_view(self, mock_request, commentable_id, user, group_id, pass_group_id=True): + def call_view(self, mock_is_forum_v2_enabled, mock_request, commentable_id, user, group_id, pass_group_id=True): """ Call the view for the implementing test class, constructing a request from the parameters. """ pass # lint-amnesty, pylint: disable=unnecessary-pass - def test_cohorted_topic_student_without_group_id(self, mock_request): - self.call_view(mock_request, "cohorted_topic", self.student, '', pass_group_id=False) + def test_cohorted_topic_student_without_group_id(self, mock_is_forum_v2_enabled, mock_request): + self.call_view(mock_is_forum_v2_enabled, mock_request, "cohorted_topic", self.student, '', pass_group_id=False) self._assert_comments_service_called_with_group_id(mock_request, self.student_cohort.id) - def test_cohorted_topic_student_none_group_id(self, mock_request): - self.call_view(mock_request, "cohorted_topic", self.student, "") + def test_cohorted_topic_student_none_group_id(self, mock_is_forum_v2_enabled, mock_request): + self.call_view(mock_is_forum_v2_enabled, mock_request, "cohorted_topic", self.student, "") self._assert_comments_service_called_with_group_id(mock_request, self.student_cohort.id) - def test_cohorted_topic_student_with_own_group_id(self, mock_request): - self.call_view(mock_request, "cohorted_topic", self.student, self.student_cohort.id) + def test_cohorted_topic_student_with_own_group_id(self, mock_is_forum_v2_enabled, mock_request): + self.call_view(mock_is_forum_v2_enabled, mock_request, "cohorted_topic", self.student, self.student_cohort.id) self._assert_comments_service_called_with_group_id(mock_request, self.student_cohort.id) - def test_cohorted_topic_student_with_other_group_id(self, mock_request): - self.call_view(mock_request, "cohorted_topic", self.student, self.moderator_cohort.id) + def test_cohorted_topic_student_with_other_group_id(self, mock_is_forum_v2_enabled, mock_request): + self.call_view( + mock_is_forum_v2_enabled, + mock_request, + "cohorted_topic", + self.student, + self.moderator_cohort.id + ) self._assert_comments_service_called_with_group_id(mock_request, self.student_cohort.id) - def test_cohorted_topic_moderator_without_group_id(self, mock_request): - self.call_view(mock_request, "cohorted_topic", self.moderator, '', pass_group_id=False) + def test_cohorted_topic_moderator_without_group_id(self, mock_is_forum_v2_enabled, mock_request): + self.call_view( + mock_is_forum_v2_enabled, + mock_request, + "cohorted_topic", + self.moderator, + '', + pass_group_id=False + ) self._assert_comments_service_called_without_group_id(mock_request) - def test_cohorted_topic_moderator_none_group_id(self, mock_request): - self.call_view(mock_request, "cohorted_topic", self.moderator, "") + def test_cohorted_topic_moderator_none_group_id(self, mock_is_forum_v2_enabled, mock_request): + self.call_view(mock_is_forum_v2_enabled, mock_request, "cohorted_topic", self.moderator, "") self._assert_comments_service_called_without_group_id(mock_request) - def test_cohorted_topic_moderator_with_own_group_id(self, mock_request): - self.call_view(mock_request, "cohorted_topic", self.moderator, self.moderator_cohort.id) + def test_cohorted_topic_moderator_with_own_group_id(self, mock_is_forum_v2_enabled, mock_request): + self.call_view( + mock_is_forum_v2_enabled, + mock_request, + "cohorted_topic", + self.moderator, + self.moderator_cohort.id + ) self._assert_comments_service_called_with_group_id(mock_request, self.moderator_cohort.id) - def test_cohorted_topic_moderator_with_other_group_id(self, mock_request): - self.call_view(mock_request, "cohorted_topic", self.moderator, self.student_cohort.id) + def test_cohorted_topic_moderator_with_other_group_id(self, mock_is_forum_v2_enabled, mock_request): + self.call_view( + mock_is_forum_v2_enabled, + mock_request, + "cohorted_topic", + self.moderator, + self.student_cohort.id + ) self._assert_comments_service_called_with_group_id(mock_request, self.student_cohort.id) - def test_cohorted_topic_moderator_with_invalid_group_id(self, mock_request): + def test_cohorted_topic_moderator_with_invalid_group_id(self, mock_is_forum_v2_enabled, mock_request): invalid_id = self.student_cohort.id + self.moderator_cohort.id - response = self.call_view(mock_request, "cohorted_topic", self.moderator, invalid_id) # lint-amnesty, pylint: disable=assignment-from-no-return + response = self.call_view(mock_is_forum_v2_enabled, mock_request, "cohorted_topic", self.moderator, invalid_id) # lint-amnesty, pylint: disable=assignment-from-no-return assert response.status_code == 500 - def test_cohorted_topic_enrollment_track_invalid_group_id(self, mock_request): + def test_cohorted_topic_enrollment_track_invalid_group_id(self, mock_is_forum_v2_enabled, mock_request): CourseModeFactory.create(course_id=self.course.id, mode_slug=CourseMode.AUDIT) CourseModeFactory.create(course_id=self.course.id, mode_slug=CourseMode.VERIFIED) discussion_settings = CourseDiscussionSettings.get(self.course.id) @@ -115,7 +140,7 @@ def test_cohorted_topic_enrollment_track_invalid_group_id(self, mock_request): }) invalid_id = -1000 - response = self.call_view(mock_request, "cohorted_topic", self.moderator, invalid_id) # lint-amnesty, pylint: disable=assignment-from-no-return + response = self.call_view(mock_is_forum_v2_enabled, mock_request, "cohorted_topic", self.moderator, invalid_id) # lint-amnesty, pylint: disable=assignment-from-no-return assert response.status_code == 500 @@ -124,57 +149,95 @@ class NonCohortedTopicGroupIdTestMixin(GroupIdAssertionMixin): Provides test cases to verify that views pass the correct `group_id` to the comments service when requesting content in non-cohorted discussions. """ - def call_view(self, mock_request, commentable_id, user, group_id, pass_group_id=True): + def call_view(self, mock_is_forum_v2_enabled, mock_request, commentable_id, user, group_id, pass_group_id=True): """ Call the view for the implementing test class, constructing a request from the parameters. """ pass # lint-amnesty, pylint: disable=unnecessary-pass - def test_non_cohorted_topic_student_without_group_id(self, mock_request): - self.call_view(mock_request, "non_cohorted_topic", self.student, '', pass_group_id=False) + def test_non_cohorted_topic_student_without_group_id(self, mock_is_forum_v2_enabled, mock_request): + self.call_view( + mock_is_forum_v2_enabled, + mock_request, + "non_cohorted_topic", + self.student, + '', + pass_group_id=False + ) self._assert_comments_service_called_without_group_id(mock_request) - def test_non_cohorted_topic_student_none_group_id(self, mock_request): - self.call_view(mock_request, "non_cohorted_topic", self.student, '') + def test_non_cohorted_topic_student_none_group_id(self, mock_is_forum_v2_enabled, mock_request): + self.call_view(mock_is_forum_v2_enabled, mock_request, "non_cohorted_topic", self.student, '') self._assert_comments_service_called_without_group_id(mock_request) - def test_non_cohorted_topic_student_with_own_group_id(self, mock_request): - self.call_view(mock_request, "non_cohorted_topic", self.student, self.student_cohort.id) + def test_non_cohorted_topic_student_with_own_group_id(self, mock_is_forum_v2_enabled, mock_request): + self.call_view( + mock_is_forum_v2_enabled, + mock_request, + "non_cohorted_topic", + self.student, + self.student_cohort.id + ) self._assert_comments_service_called_without_group_id(mock_request) - def test_non_cohorted_topic_student_with_other_group_id(self, mock_request): - self.call_view(mock_request, "non_cohorted_topic", self.student, self.moderator_cohort.id) + def test_non_cohorted_topic_student_with_other_group_id(self, mock_is_forum_v2_enabled, mock_request): + self.call_view( + mock_is_forum_v2_enabled, + mock_request, + "non_cohorted_topic", + self.student, + self.moderator_cohort.id + ) self._assert_comments_service_called_without_group_id(mock_request) - def test_non_cohorted_topic_moderator_without_group_id(self, mock_request): - self.call_view(mock_request, "non_cohorted_topic", self.moderator, '', pass_group_id=False) + def test_non_cohorted_topic_moderator_without_group_id(self, mock_is_forum_v2_enabled, mock_request): + self.call_view( + mock_is_forum_v2_enabled, + mock_request, + "non_cohorted_topic", + self.moderator, + "", + pass_group_id=False, + ) self._assert_comments_service_called_without_group_id(mock_request) - def test_non_cohorted_topic_moderator_none_group_id(self, mock_request): - self.call_view(mock_request, "non_cohorted_topic", self.moderator, '') + def test_non_cohorted_topic_moderator_none_group_id(self, mock_is_forum_v2_enabled, mock_request): + self.call_view(mock_is_forum_v2_enabled, mock_request, "non_cohorted_topic", self.moderator, '') self._assert_comments_service_called_without_group_id(mock_request) - def test_non_cohorted_topic_moderator_with_own_group_id(self, mock_request): - self.call_view(mock_request, "non_cohorted_topic", self.moderator, self.moderator_cohort.id) + def test_non_cohorted_topic_moderator_with_own_group_id(self, mock_is_forum_v2_enabled, mock_request): + self.call_view( + mock_is_forum_v2_enabled, + mock_request, + "non_cohorted_topic", + self.moderator, + self.moderator_cohort.id, + ) self._assert_comments_service_called_without_group_id(mock_request) - def test_non_cohorted_topic_moderator_with_other_group_id(self, mock_request): - self.call_view(mock_request, "non_cohorted_topic", self.moderator, self.student_cohort.id) + def test_non_cohorted_topic_moderator_with_other_group_id(self, mock_is_forum_v2_enabled, mock_request): + self.call_view( + mock_is_forum_v2_enabled, + mock_request, + "non_cohorted_topic", + self.moderator, + self.student_cohort.id, + ) self._assert_comments_service_called_without_group_id(mock_request) - def test_non_cohorted_topic_moderator_with_invalid_group_id(self, mock_request): + def test_non_cohorted_topic_moderator_with_invalid_group_id(self, mock_is_forum_v2_enabled, mock_request): invalid_id = self.student_cohort.id + self.moderator_cohort.id - self.call_view(mock_request, "non_cohorted_topic", self.moderator, invalid_id) + self.call_view(mock_is_forum_v2_enabled, mock_request, "non_cohorted_topic", self.moderator, invalid_id) self._assert_comments_service_called_without_group_id(mock_request) - def test_team_discussion_id_not_cohorted(self, mock_request): + def test_team_discussion_id_not_cohorted(self, mock_is_forum_v2_enabled, mock_request): team = CourseTeamFactory( course_id=self.course.id, topic_id='topic-id' ) team.add_user(self.student) - self.call_view(mock_request, team.discussion_topic_id, self.student, '') + self.call_view(mock_is_forum_v2_enabled, mock_request, team.discussion_topic_id, self.student, '') self._assert_comments_service_called_without_group_id(mock_request) diff --git a/lms/djangoapps/discussion/rest_api/api.py b/lms/djangoapps/discussion/rest_api/api.py index 19ccf26d19a4..a517e00dff34 100644 --- a/lms/djangoapps/discussion/rest_api/api.py +++ b/lms/djangoapps/discussion/rest_api/api.py @@ -199,7 +199,7 @@ def _get_course(course_key: CourseKey, user: User, check_tab: bool = True) -> Co return course -def _get_thread_and_context(request, thread_id, retrieve_kwargs=None): +def _get_thread_and_context(request, thread_id, retrieve_kwargs=None, course_id=None): """ Retrieve the given thread and build a serializer context for it, returning both. This function also enforces access control for the thread (checking @@ -213,7 +213,7 @@ def _get_thread_and_context(request, thread_id, retrieve_kwargs=None): retrieve_kwargs["with_responses"] = False if "mark_as_read" not in retrieve_kwargs: retrieve_kwargs["mark_as_read"] = False - cc_thread = Thread(id=thread_id).retrieve(**retrieve_kwargs) + cc_thread = Thread(id=thread_id).retrieve(course_id=course_id, **retrieve_kwargs) course_key = CourseKey.from_string(cc_thread["course_id"]) course = _get_course(course_key, request.user) context = get_context(course, request, cc_thread) @@ -1645,7 +1645,8 @@ def get_thread(request, thread_id, requested_fields=None, course_id=None): retrieve_kwargs={ "with_responses": True, "user_id": str(request.user.id), - } + }, + course_id=course_id, ) if course_id and course_id != cc_thread.course_id: raise ThreadNotFoundError("Thread not found.") diff --git a/lms/djangoapps/discussion/rest_api/discussions_notifications.py b/lms/djangoapps/discussion/rest_api/discussions_notifications.py index 88c7fea558c1..f6572c829c7a 100644 --- a/lms/djangoapps/discussion/rest_api/discussions_notifications.py +++ b/lms/djangoapps/discussion/rest_api/discussions_notifications.py @@ -117,6 +117,7 @@ def send_new_response_notification(self): context = { 'email_content': clean_thread_html_body(self.comment.body), } + self._populate_context_with_ids_for_mobile(context) self._send_notification([self.thread.user_id], "new_response", extra_context=context) def _response_and_thread_has_same_creator(self) -> bool: @@ -156,6 +157,7 @@ def send_new_comment_notification(self): "author_pronoun": str(author_pronoun), "email_content": clean_thread_html_body(self.comment.body), } + self._populate_context_with_ids_for_mobile(context) self._send_notification([self.thread.user_id], "new_comment", extra_context=context) def send_new_comment_on_response_notification(self): @@ -171,6 +173,7 @@ def send_new_comment_on_response_notification(self): context = { "email_content": clean_thread_html_body(self.comment.body), } + self._populate_context_with_ids_for_mobile(context) self._send_notification( [self.parent_response.user_id], "new_comment_on_response", @@ -202,7 +205,7 @@ def send_response_on_followed_post_notification(self): while has_more_subscribers: - subscribers = Subscription.fetch(self.thread.id, query_params={'page': page}) + subscribers = Subscription.fetch(self.thread.id, self.course.id, query_params={'page': page}) if page <= subscribers.num_pages: for subscriber in subscribers.collection: # Check if the subscriber is not the thread creator or response creator @@ -216,12 +219,15 @@ def send_response_on_followed_post_notification(self): # Remove duplicate users from the list of users to send notification users = list(set(users)) if not self.parent_id: + context = { + "email_content": clean_thread_html_body(self.comment.body), + } + self._populate_context_with_ids_for_mobile(context) self._send_notification( users, "response_on_followed_post", - extra_context={ - "email_content": clean_thread_html_body(self.comment.body), - }) + extra_context=context + ) else: author_name = f"{self.parent_response.username}'s" # use 'their' if comment author is also response author. @@ -231,14 +237,16 @@ def send_response_on_followed_post_notification(self): if self._response_and_comment_has_same_creator() else f"{self.parent_response.username}'s" ) + context = { + "author_name": str(author_name), + "author_pronoun": str(author_pronoun), + "email_content": clean_thread_html_body(self.comment.body), + } + self._populate_context_with_ids_for_mobile(context) self._send_notification( users, "comment_on_followed_post", - extra_context={ - "author_name": str(author_name), - "author_pronoun": str(author_pronoun), - "email_content": clean_thread_html_body(self.comment.body), - } + extra_context=context ) def _create_cohort_course_audience(self): @@ -290,6 +298,7 @@ def send_response_endorsed_on_thread_notification(self): context = { "email_content": clean_thread_html_body(self.comment.body) } + self._populate_context_with_ids_for_mobile(context) self._send_notification([self.thread.user_id], "response_endorsed_on_thread", extra_context=context) def send_response_endorsed_notification(self): @@ -299,6 +308,7 @@ def send_response_endorsed_notification(self): context = { "email_content": clean_thread_html_body(self.comment.body) } + self._populate_context_with_ids_for_mobile(context) self._send_notification([self.creator.id], "response_endorsed", extra_context=context) def send_new_thread_created_notification(self): @@ -330,6 +340,7 @@ def send_new_thread_created_notification(self): 'post_title': self.thread.title, "email_content": clean_thread_html_body(self.thread.body), } + self._populate_context_with_ids_for_mobile(context) self._send_course_wide_notification(notification_type, audience_filters, context) def send_reported_content_notification(self): @@ -362,6 +373,12 @@ def send_reported_content_notification(self): ]} self._send_course_wide_notification("content_reported", audience_filters, context) + def _populate_context_with_ids_for_mobile(self, context): + context['thread_id'] = self.thread.id + context['topic_id'] = self.thread.commentable_id + context['comment_id'] = self.comment_id + context['parent_id'] = self.parent_id + def is_discussion_cohorted(course_key_str): """ diff --git a/lms/djangoapps/discussion/rest_api/serializers.py b/lms/djangoapps/discussion/rest_api/serializers.py index f8868cbed8c8..ff0c656baf28 100644 --- a/lms/djangoapps/discussion/rest_api/serializers.py +++ b/lms/djangoapps/discussion/rest_api/serializers.py @@ -68,7 +68,7 @@ def get_context(course, request, thread=None): moderator_user_ids = get_moderator_users_list(course.id) ta_user_ids = get_course_ta_users_list(course.id) requester = request.user - cc_requester = CommentClientUser.from_django_user(requester).retrieve() + cc_requester = CommentClientUser.from_django_user(requester).retrieve(course_id=course.id) cc_requester["course_id"] = course.id course_discussion_settings = CourseDiscussionSettings.get(course.id) is_global_staff = GlobalStaff().has_user(requester) diff --git a/lms/djangoapps/discussion/rest_api/tests/test_api.py b/lms/djangoapps/discussion/rest_api/tests/test_api.py index 9a9041fd5fa4..62725cc47466 100644 --- a/lms/djangoapps/discussion/rest_api/tests/test_api.py +++ b/lms/djangoapps/discussion/rest_api/tests/test_api.py @@ -1248,6 +1248,22 @@ def setUp(self): httpretty.enable() self.addCleanup(httpretty.reset) self.addCleanup(httpretty.disable) + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) self.maxDiff = None # pylint: disable=invalid-name self.user = UserFactory.create() self.register_get_user_response(self.user) @@ -1872,6 +1888,12 @@ def setUp(self): httpretty.enable() self.addCleanup(httpretty.reset) self.addCleanup(httpretty.disable) + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) self.user = UserFactory.create() self.register_get_user_response(self.user) self.request = RequestFactory().get("/test_path") @@ -2198,6 +2220,22 @@ def setUp(self): self.course = CourseFactory.create() self.addCleanup(httpretty.reset) self.addCleanup(httpretty.disable) + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) self.user = UserFactory.create() self.register_get_user_response(self.user) self.request = RequestFactory().get("/test_path") @@ -2589,6 +2627,17 @@ def setUp(self): httpretty.enable() self.addCleanup(httpretty.reset) self.addCleanup(httpretty.disable) + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) self.user = UserFactory.create() self.register_get_user_response(self.user) @@ -3153,6 +3202,22 @@ def setUp(self): self.addCleanup(httpretty.reset) self.addCleanup(httpretty.disable) + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) self.user = UserFactory.create() self.register_get_user_response(self.user) self.request = RequestFactory().get("/test_path") @@ -3670,6 +3735,22 @@ def setUp(self): httpretty.enable() self.addCleanup(httpretty.reset) self.addCleanup(httpretty.disable) + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) self.user = UserFactory.create() self.register_get_user_response(self.user) self.request = RequestFactory().get("/test_path") @@ -3823,6 +3904,22 @@ def setUp(self): httpretty.enable() self.addCleanup(httpretty.reset) self.addCleanup(httpretty.disable) + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) self.user = UserFactory.create() self.register_get_user_response(self.user) self.request = RequestFactory().get("/test_path") @@ -3991,6 +4088,17 @@ def setUp(self): httpretty.enable() self.addCleanup(httpretty.reset) self.addCleanup(httpretty.disable) + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) self.user = UserFactory.create() self.register_get_user_response(self.user) self.request = RequestFactory().get("/test_path") diff --git a/lms/djangoapps/discussion/rest_api/tests/test_serializers.py b/lms/djangoapps/discussion/rest_api/tests/test_serializers.py index 8103eb692791..73b195e02fa6 100644 --- a/lms/djangoapps/discussion/rest_api/tests/test_serializers.py +++ b/lms/djangoapps/discussion/rest_api/tests/test_serializers.py @@ -54,6 +54,12 @@ def setUp(self): httpretty.enable() self.addCleanup(httpretty.reset) self.addCleanup(httpretty.disable) + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) self.maxDiff = None # pylint: disable=invalid-name self.user = UserFactory.create() self.register_get_user_response(self.user) @@ -571,6 +577,12 @@ def setUp(self): httpretty.enable() self.addCleanup(httpretty.reset) self.addCleanup(httpretty.disable) + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) self.user = UserFactory.create() self.register_get_user_response(self.user) self.request = RequestFactory().get("/dummy") @@ -802,6 +814,22 @@ def setUp(self): httpretty.enable() self.addCleanup(httpretty.reset) self.addCleanup(httpretty.disable) + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) self.user = UserFactory.create() self.register_get_user_response(self.user) self.request = RequestFactory().get("/dummy") diff --git a/lms/djangoapps/discussion/rest_api/tests/test_tasks.py b/lms/djangoapps/discussion/rest_api/tests/test_tasks.py index ddfc120a8e4b..8171ca7e6a71 100644 --- a/lms/djangoapps/discussion/rest_api/tests/test_tasks.py +++ b/lms/djangoapps/discussion/rest_api/tests/test_tasks.py @@ -58,10 +58,27 @@ def setUp(self): Setup test case """ super().setUp() - + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) # Creating a course self.course = CourseFactory.create() + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread", + return_value=self.course.id + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment", + return_value=self.course.id + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) # Creating relative discussion and cohort settings CourseCohortsSettings.objects.create(course_id=str(self.course.id)) CourseDiscussionSettings.objects.create(course_id=str(self.course.id), _divided_discussions='[]') @@ -250,8 +267,26 @@ def setUp(self): super().setUp() httpretty.reset() httpretty.enable() + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) self.course = CourseFactory.create() + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread", + return_value=self.course.id + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment", + return_value=self.course.id + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) self.user_1 = UserFactory.create() CourseEnrollment.enroll(self.user_1, self.course.id) self.user_2 = UserFactory.create() @@ -270,6 +305,8 @@ def setUp(self): "username": thread.username, "thread_type": 'discussion', "title": thread.title, + "commentable_id": thread.commentable_id, + }) self._register_subscriptions_endpoint() @@ -319,7 +356,11 @@ def test_send_notification_to_thread_creator(self): 'post_title': 'test thread', 'email_content': self.comment.body, 'course_name': self.course.display_name, - 'sender_id': self.user_2.id + 'sender_id': self.user_2.id, + 'parent_id': None, + 'topic_id': None, + 'thread_id': 1, + 'comment_id': 4, } self.assertDictEqual(args.context, expected_context) self.assertEqual( @@ -365,7 +406,11 @@ def test_send_notification_to_parent_threads(self): 'author_name': 'dummy\'s', 'author_pronoun': 'dummy\'s', 'course_name': self.course.display_name, - 'sender_id': self.user_3.id + 'sender_id': self.user_3.id, + 'parent_id': 2, + 'topic_id': None, + 'thread_id': 1, + 'comment_id': 4, } self.assertDictEqual(args_comment.context, expected_context) self.assertEqual( @@ -382,7 +427,11 @@ def test_send_notification_to_parent_threads(self): 'post_title': self.thread.title, 'email_content': self.comment.body, 'course_name': self.course.display_name, - 'sender_id': self.user_3.id + 'sender_id': self.user_3.id, + 'parent_id': 2, + 'topic_id': None, + 'thread_id': 1, + 'comment_id': 4, } self.assertDictEqual(args_comment_on_response.context, expected_context) self.assertEqual( @@ -442,7 +491,11 @@ def test_comment_creators_own_response(self): 'author_pronoun': 'your', 'course_name': self.course.display_name, 'sender_id': self.user_3.id, - 'email_content': self.comment.body + 'email_content': self.comment.body, + 'parent_id': 2, + 'topic_id': None, + 'thread_id': 1, + 'comment_id': 4, } self.assertDictEqual(args_comment.context, expected_context) self.assertEqual( @@ -487,6 +540,10 @@ def test_send_notification_to_followers(self, parent_id, notification_type): 'email_content': self.comment.body, 'course_name': self.course.display_name, 'sender_id': self.user_2.id, + 'parent_id': parent_id, + 'topic_id': None, + 'thread_id': 1, + 'comment_id': 4, } if parent_id: expected_context['author_name'] = 'dummy\'s' @@ -536,8 +593,26 @@ def setUp(self): super().setUp() httpretty.reset() httpretty.enable() + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) self.course = CourseFactory.create() + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread", + return_value=self.course.id + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment", + return_value=self.course.id + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) self.user_1 = UserFactory.create() CourseEnrollment.enroll(self.user_1, self.course.id) self.user_2 = UserFactory.create() @@ -571,6 +646,8 @@ def test_new_comment_notification(self): "username": thread.username, "thread_type": 'discussion', "title": thread.title, + "commentable_id": thread.commentable_id, + }) self.register_get_comment_response({ 'id': response.id, @@ -603,8 +680,26 @@ def setUp(self): super().setUp() httpretty.reset() httpretty.enable() + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) self.course = CourseFactory.create() + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread", + return_value=self.course.id + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment", + return_value=self.course.id + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) self.user_1 = UserFactory.create() CourseEnrollment.enroll(self.user_1, self.course.id) self.user_2 = UserFactory.create() @@ -636,6 +731,7 @@ def test_response_endorsed_notifications(self): "username": thread.username, "thread_type": 'discussion', "title": thread.title, + "commentable_id": thread.commentable_id, }) self.register_get_comment_response({ 'id': 1, @@ -663,7 +759,11 @@ def test_response_endorsed_notifications(self): 'post_title': 'test thread', 'course_name': self.course.display_name, 'sender_id': int(self.user_2.id), - 'email_content': 'dummy' + 'email_content': 'dummy', + 'parent_id': None, + 'topic_id': None, + 'thread_id': 1, + 'comment_id': 2, } self.assertDictEqual(notification_data.context, expected_context) self.assertEqual(notification_data.content_url, _get_mfe_url(self.course.id, thread.id)) @@ -681,7 +781,11 @@ def test_response_endorsed_notifications(self): 'post_title': 'test thread', 'course_name': self.course.display_name, 'sender_id': int(response.user_id), - 'email_content': 'dummy' + 'email_content': 'dummy', + 'parent_id': None, + 'topic_id': None, + 'thread_id': 1, + 'comment_id': 2, } self.assertDictEqual(notification_data.context, expected_context) self.assertEqual(notification_data.content_url, _get_mfe_url(self.course.id, thread.id)) diff --git a/lms/djangoapps/discussion/rest_api/tests/test_views.py b/lms/djangoapps/discussion/rest_api/tests/test_views.py index 283117000712..9ae03986bb93 100644 --- a/lms/djangoapps/discussion/rest_api/tests/test_views.py +++ b/lms/djangoapps/discussion/rest_api/tests/test_views.py @@ -171,6 +171,12 @@ def setUp(self): self.user = UserFactory.create(password=self.TEST_PASSWORD) self.course = CourseFactory.create(org='a', course='b', run='c', start=datetime.now(UTC)) self.url = reverse("upload_file", kwargs={"course_id": str(self.course.id)}) + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) def user_login(self): """ @@ -301,6 +307,7 @@ def test_file_upload_with_no_data(self): @ddt.ddt @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) +@mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_FORUM_V2": False}) class CommentViewSetListByUserTest( ForumsEnableMixin, CommentsServiceMockMixin, @@ -319,6 +326,12 @@ def setUp(self): httpretty.enable() self.addCleanup(httpretty.reset) self.addCleanup(httpretty.disable) + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) self.user = UserFactory.create(password=self.TEST_PASSWORD) self.register_get_user_response(self.user) @@ -500,6 +513,12 @@ class CourseViewTest(DiscussionAPIViewTestMixin, ModuleStoreTestCase): def setUp(self): super().setUp() self.url = reverse("discussion_course", kwargs={"course_id": str(self.course.id)}) + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) def test_404(self): response = self.client.get( @@ -561,6 +580,12 @@ def setUp(self): self.superuser_client = APIClient() self.retired_username = get_retired_username_by_username(self.user.username) self.url = reverse("retire_discussion_user") + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) def assert_response_correct(self, response, expected_status, expected_content): """ @@ -631,6 +656,12 @@ def setUp(self): self.worker_client = APIClient() self.new_username = "test_username_replacement" self.url = reverse("replace_discussion_username") + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) def assert_response_correct(self, response, expected_status, expected_content): """ @@ -733,6 +764,12 @@ def setUp(self): "courseware-3": {"discussion": 7, "question": 2}, } self.register_get_course_commentable_counts_response(self.course.id, self.thread_counts_map) + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) def create_course(self, blocks_count, module_store, topics): """ @@ -988,6 +1025,12 @@ def setUp(self) -> None: patcher.start() self.addCleanup(patcher.stop) self.url = reverse("course_topics_v3", kwargs={"course_id": str(self.course.id)}) + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) def test_basic(self): response = self.client.get(self.url) @@ -1024,6 +1067,12 @@ def setUp(self): super().setUp() self.author = UserFactory.create() self.url = reverse("thread-list") + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) def create_source_thread(self, overrides=None): """ @@ -1365,6 +1414,12 @@ class ThreadViewSetCreateTest(DiscussionAPIViewTestMixin, ModuleStoreTestCase): def setUp(self): super().setUp() self.url = reverse("thread-list") + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) def test_basic(self): self.register_get_user_response(self.user) @@ -1437,6 +1492,17 @@ def setUp(self): self.unsupported_media_type = JSONParser.media_type super().setUp() self.url = reverse("thread-detail", kwargs={"thread_id": "test_thread"}) + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) def test_basic(self): self.register_get_user_response(self.user) @@ -1581,6 +1647,17 @@ def setUp(self): super().setUp() self.url = reverse("thread-detail", kwargs={"thread_id": "test_thread"}) self.thread_id = "test_thread" + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) def test_basic(self): self.register_get_user_response(self.user) @@ -1681,6 +1758,12 @@ def setUp(self): ] self.url = reverse("discussion_learner_threads", kwargs={'course_id': str(self.course.id)}) + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) def update_thread(self, thread): """ @@ -1923,6 +2006,17 @@ def setUp(self): self.url = reverse("comment-list") self.thread_id = "test_thread" self.storage = get_profile_image_storage() + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) def create_source_comment(self, overrides=None): """ @@ -2377,6 +2471,22 @@ def setUp(self): super().setUp() self.url = reverse("comment-detail", kwargs={"comment_id": "test_comment"}) self.comment_id = "test_comment" + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) def test_basic(self): self.register_get_user_response(self.user) @@ -2416,6 +2526,23 @@ class CommentViewSetCreateTest(DiscussionAPIViewTestMixin, ModuleStoreTestCase): def setUp(self): super().setUp() self.url = reverse("comment-list") + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) def test_basic(self): self.register_get_user_response(self.user) @@ -2518,6 +2645,22 @@ def setUp(self): httpretty.enable() self.addCleanup(httpretty.reset) self.addCleanup(httpretty.disable) + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) self.register_get_user_response(self.user) self.url = reverse("comment-detail", kwargs={"comment_id": "test_comment"}) @@ -2640,6 +2783,22 @@ def setUp(self): super().setUp() self.url = reverse("thread-detail", kwargs={"thread_id": "test_thread"}) self.thread_id = "test_thread" + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) def test_basic(self): self.register_get_user_response(self.user) @@ -2693,6 +2852,22 @@ def setUp(self): self.url = reverse("comment-detail", kwargs={"comment_id": "test_comment"}) self.thread_id = "test_thread" self.comment_id = "test_comment" + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) def make_comment_data(self, comment_id, parent_id=None, children=[]): # pylint: disable=W0102 """ @@ -2838,6 +3013,12 @@ def setUp(self): self.path = reverse('discussion_course_settings', kwargs={'course_id': str(self.course.id)}) self.password = self.TEST_PASSWORD self.user = UserFactory(username='staff', password=self.password, is_staff=True) + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) def _get_oauth_headers(self, user): """Return the OAuth headers for testing OAuth authentication""" @@ -3127,6 +3308,12 @@ class CourseDiscussionRolesAPIViewTest(APITestCase, UrlResetMixin, ModuleStoreTe @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) def setUp(self): super().setUp() + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) self.course = CourseFactory.create( org="x", course="y", @@ -3318,6 +3505,12 @@ class CourseActivityStatsTest(ForumsEnableMixin, UrlResetMixin, CommentsServiceM @mock.patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) def setUp(self) -> None: super().setUp() + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) self.course = CourseFactory.create() self.course_key = str(self.course.id) seed_permissions_roles(self.course.id) diff --git a/lms/djangoapps/discussion/rest_api/tests/utils.py b/lms/djangoapps/discussion/rest_api/tests/utils.py index 27e34705f5df..496f8723acfb 100644 --- a/lms/djangoapps/discussion/rest_api/tests/utils.py +++ b/lms/djangoapps/discussion/rest_api/tests/utils.py @@ -675,13 +675,14 @@ class ThreadMock(object): A mock thread object """ - def __init__(self, thread_id, creator, title, parent_id=None, body=''): + def __init__(self, thread_id, creator, title, parent_id=None, body='', commentable_id=None): self.id = thread_id self.user_id = str(creator.id) self.username = creator.username self.title = title self.parent_id = parent_id self.body = body + self.commentable_id = commentable_id def url_with_id(self, params): return f"http://example.com/{params['id']}" diff --git a/lms/djangoapps/discussion/tests/test_tasks.py b/lms/djangoapps/discussion/tests/test_tasks.py index 92dadac9d9ee..952a6c567a52 100644 --- a/lms/djangoapps/discussion/tests/test_tasks.py +++ b/lms/djangoapps/discussion/tests/test_tasks.py @@ -232,6 +232,22 @@ def setUp(self): thread_permalink = '/courses/discussion/dummy_discussion_id' self.permalink_patcher = mock.patch('lms.djangoapps.discussion.tasks.permalink', return_value=thread_permalink) self.mock_permalink = self.permalink_patcher.start() + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) def tearDown(self): super().tearDown() diff --git a/lms/djangoapps/discussion/tests/test_views.py b/lms/djangoapps/discussion/tests/test_views.py index e0d3b869da3d..facdb368f14f 100644 --- a/lms/djangoapps/discussion/tests/test_views.py +++ b/lms/djangoapps/discussion/tests/test_views.py @@ -4,6 +4,7 @@ import json import logging from datetime import datetime +from unittest import mock from unittest.mock import ANY, Mock, call, patch import ddt @@ -109,9 +110,20 @@ def setUp(self): config = ForumsConfig.current() config.enabled = True config.save() + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) @patch('common.djangoapps.student.models.user.cc.User.from_django_user') - @patch('common.djangoapps.student.models.user.cc.User.active_threads') + @patch('openedx.core.djangoapps.django_comment_common.comment_client.user.User.active_threads') def test_user_profile_exception(self, mock_threads, mock_from_django_user): # Mock the code that makes the HTTP requests to the cs_comment_service app @@ -323,6 +335,17 @@ class SingleThreadTestCase(ForumsEnableMixin, ModuleStoreTestCase): # lint-amne def setUp(self): super().setUp() + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) self.course = CourseFactory.create(discussion_topics={'dummy discussion': {'id': 'dummy_discussion_id'}}) self.student = UserFactory.create() @@ -513,6 +536,20 @@ class SingleThreadQueryCountTestCase(ForumsEnableMixin, ModuleStoreTestCase): Ensures the number of modulestore queries and number of sql queries are independent of the number of responses retrieved for a given discussion thread. """ + def setUp(self): + super().setUp() + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) + @ddt.data( # split mongo: 3 queries, regardless of thread response size. (False, 1, 2, 2, 21, 8), @@ -582,6 +619,20 @@ def call_single_thread(): @patch('requests.request', autospec=True) class SingleCohortedThreadTestCase(CohortedTestCase): # lint-amnesty, pylint: disable=missing-class-docstring + def setUp(self): + super().setUp() + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) + def _create_mock_cohorted_thread(self, mock_request): # lint-amnesty, pylint: disable=missing-function-docstring mock_text = "dummy content" mock_thread_id = "test_thread_id" @@ -644,6 +695,20 @@ def test_html(self, mock_request): @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) class SingleThreadAccessTestCase(CohortedTestCase): # lint-amnesty, pylint: disable=missing-class-docstring + def setUp(self): + super().setUp() + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) + def call_view(self, mock_request, commentable_id, user, group_id, thread_group_id=None, pass_group_id=True): # lint-amnesty, pylint: disable=missing-function-docstring thread_id = "test_thread_id" mock_request.side_effect = make_mock_request_impl( @@ -746,6 +811,20 @@ def test_private_team_thread(self, mock_request): class SingleThreadGroupIdTestCase(CohortedTestCase, GroupIdAssertionMixin): # lint-amnesty, pylint: disable=missing-class-docstring cs_endpoint = "/threads/dummy_thread_id" + def setUp(self): + super().setUp() + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) + def call_view(self, mock_request, commentable_id, user, group_id, pass_group_id=True, is_ajax=False): # lint-amnesty, pylint: disable=missing-function-docstring mock_request.side_effect = make_mock_request_impl( course=self.course, text="dummy context", group_id=self.student_cohort.id @@ -881,6 +960,22 @@ class SingleThreadContentGroupTestCase(ForumsEnableMixin, UrlResetMixin, Content @patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) def setUp(self): super().setUp() + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) def assert_can_access(self, user, discussion_id, thread_id, should_have_access): """ @@ -1046,6 +1141,7 @@ def test_private_team_discussion(self, mock_request): @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) +@patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) class InlineDiscussionGroupIdTestCase( # lint-amnesty, pylint: disable=missing-class-docstring CohortedTestCase, CohortedTopicGroupIdTestMixin, @@ -1056,8 +1152,22 @@ class InlineDiscussionGroupIdTestCase( # lint-amnesty, pylint: disable=missing- def setUp(self): super().setUp() self.cohorted_commentable_id = 'cohorted_topic' - - def call_view(self, mock_request, commentable_id, user, group_id, pass_group_id=True): + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) + + def call_view( + self, + mock_is_forum_v2_enabled, + mock_request, + commentable_id, + user, + group_id, + pass_group_id=True + ): # pylint: disable=arguments-differ + mock_is_forum_v2_enabled.return_value = False kwargs = {'commentable_id': self.cohorted_commentable_id} if group_id: # avoid causing a server error when the LMS chokes attempting @@ -1084,8 +1194,9 @@ def call_view(self, mock_request, commentable_id, user, group_id, pass_group_id= commentable_id ) - def test_group_info_in_ajax_response(self, mock_request): + def test_group_info_in_ajax_response(self, mock_is_forum_v2_enabled, mock_request): response = self.call_view( + mock_is_forum_v2_enabled, mock_request, self.cohorted_commentable_id, self.student, @@ -1097,10 +1208,29 @@ def test_group_info_in_ajax_response(self, mock_request): @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) +@patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) class ForumFormDiscussionGroupIdTestCase(CohortedTestCase, CohortedTopicGroupIdTestMixin): # lint-amnesty, pylint: disable=missing-class-docstring cs_endpoint = "/threads" - def call_view(self, mock_request, commentable_id, user, group_id, pass_group_id=True, is_ajax=False): # pylint: disable=arguments-differ + def setUp(self): + super().setUp() + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) + + def call_view( + self, + mock_is_forum_v2_enabled, + mock_request, + commentable_id, + user, + group_id, + pass_group_id=True, + is_ajax=False + ): # pylint: disable=arguments-differ + mock_is_forum_v2_enabled.return_value = False kwargs = {} if group_id: kwargs['group_id'] = group_id @@ -1120,8 +1250,9 @@ def call_view(self, mock_request, commentable_id, user, group_id, pass_group_id= **headers ) - def test_group_info_in_html_response(self, mock_request): + def test_group_info_in_html_response(self, mock_is_forum_v2_enabled, mock_request): response = self.call_view( + mock_is_forum_v2_enabled, mock_request, "cohorted_topic", self.student, @@ -1129,8 +1260,9 @@ def test_group_info_in_html_response(self, mock_request): ) self._assert_html_response_contains_group_info(response) - def test_group_info_in_ajax_response(self, mock_request): + def test_group_info_in_ajax_response(self, mock_is_forum_v2_enabled, mock_request): response = self.call_view( + mock_is_forum_v2_enabled, mock_request, "cohorted_topic", self.student, @@ -1143,16 +1275,38 @@ def test_group_info_in_ajax_response(self, mock_request): @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) +@patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) class UserProfileDiscussionGroupIdTestCase(CohortedTestCase, CohortedTopicGroupIdTestMixin): # lint-amnesty, pylint: disable=missing-class-docstring cs_endpoint = "/active_threads" + def setUp(self): + super().setUp() + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) + def call_view_for_profiled_user( - self, mock_request, requesting_user, profiled_user, group_id, pass_group_id, is_ajax=False + self, + mock_is_forum_v2_enabled, + mock_request, + requesting_user, + profiled_user, + group_id, + pass_group_id, + is_ajax=False ): """ Calls "user_profile" view method on behalf of "requesting_user" to get information about the user "profiled_user". """ + mock_is_forum_v2_enabled.return_value = False kwargs = {} if group_id: kwargs['group_id'] = group_id @@ -1172,13 +1326,23 @@ def call_view_for_profiled_user( **headers ) - def call_view(self, mock_request, _commentable_id, user, group_id, pass_group_id=True, is_ajax=False): # pylint: disable=arguments-differ + def call_view( + self, + mock_is_forum_v2_enabled, + mock_request, + _commentable_id, + user, + group_id, + pass_group_id=True, + is_ajax=False + ): # pylint: disable=arguments-differ return self.call_view_for_profiled_user( - mock_request, user, user, group_id, pass_group_id=pass_group_id, is_ajax=is_ajax + mock_is_forum_v2_enabled, mock_request, user, user, group_id, pass_group_id=pass_group_id, is_ajax=is_ajax ) - def test_group_info_in_html_response(self, mock_request): + def test_group_info_in_html_response(self, mock_is_forum_v2_enabled, mock_request): response = self.call_view( + mock_is_forum_v2_enabled, mock_request, "cohorted_topic", self.student, @@ -1187,8 +1351,9 @@ def test_group_info_in_html_response(self, mock_request): ) self._assert_html_response_contains_group_info(response) - def test_group_info_in_ajax_response(self, mock_request): + def test_group_info_in_ajax_response(self, mock_is_forum_v2_enabled, mock_request): response = self.call_view( + mock_is_forum_v2_enabled, mock_request, "cohorted_topic", self.student, @@ -1200,7 +1365,14 @@ def test_group_info_in_ajax_response(self, mock_request): ) def _test_group_id_passed_to_user_profile( - self, mock_request, expect_group_id_in_request, requesting_user, profiled_user, group_id, pass_group_id + self, + mock_is_forum_v2_enabled, + mock_request, + expect_group_id_in_request, + requesting_user, + profiled_user, + group_id, + pass_group_id ): """ Helper method for testing whether or not group_id was passed to the user_profile request. @@ -1221,10 +1393,11 @@ def get_params_from_user_info_call(for_specific_course): has_course_id = "course_id" in params if (for_specific_course and has_course_id) or (not for_specific_course and not has_course_id): return params - pytest.fail("Did not find appropriate user_profile call for 'for_specific_course'=" + for_specific_course) + pytest.fail(f"Did not find appropriate user_profile call for 'for_specific_course'={for_specific_course}") mock_request.reset_mock() self.call_view_for_profiled_user( + mock_is_forum_v2_enabled, mock_request, requesting_user, profiled_user, @@ -1243,7 +1416,7 @@ def get_params_from_user_info_call(for_specific_course): else: assert 'group_id' not in params_with_course_id - def test_group_id_passed_to_user_profile_student(self, mock_request): + def test_group_id_passed_to_user_profile_student(self, mock_is_forum_v2_enabled, mock_request): """ Test that the group id is always included when requesting user profile information for a particular course if the requester does not have discussion moderation privileges. @@ -1254,7 +1427,13 @@ def verify_group_id_always_present(profiled_user, pass_group_id): (non-privileged user). """ self._test_group_id_passed_to_user_profile( - mock_request, True, self.student, profiled_user, self.student_cohort.id, pass_group_id + mock_is_forum_v2_enabled, + mock_request, + True, + self.student, + profiled_user, + self.student_cohort.id, + pass_group_id ) # In all these test cases, the requesting_user is the student (non-privileged user). @@ -1264,7 +1443,7 @@ def verify_group_id_always_present(profiled_user, pass_group_id): verify_group_id_always_present(profiled_user=self.moderator, pass_group_id=True) verify_group_id_always_present(profiled_user=self.moderator, pass_group_id=False) - def test_group_id_user_profile_moderator(self, mock_request): + def test_group_id_user_profile_moderator(self, mock_is_forum_v2_enabled, mock_request): """ Test that the group id is only included when a privileged user requests user profile information for a particular course and user if the group_id is explicitly passed in. @@ -1274,7 +1453,13 @@ def verify_group_id_present(profiled_user, pass_group_id, requested_cohort=self. Helper method to verify that group_id is present. """ self._test_group_id_passed_to_user_profile( - mock_request, True, self.moderator, profiled_user, requested_cohort.id, pass_group_id + mock_is_forum_v2_enabled, + mock_request, + True, + self.moderator, + profiled_user, + requested_cohort.id, + pass_group_id ) def verify_group_id_not_present(profiled_user, pass_group_id, requested_cohort=self.moderator_cohort): @@ -1282,7 +1467,13 @@ def verify_group_id_not_present(profiled_user, pass_group_id, requested_cohort=s Helper method to verify that group_id is not present. """ self._test_group_id_passed_to_user_profile( - mock_request, False, self.moderator, profiled_user, requested_cohort.id, pass_group_id + mock_is_forum_v2_enabled, + mock_request, + False, + self.moderator, + profiled_user, + requested_cohort.id, + pass_group_id ) # In all these test cases, the requesting_user is the moderator (privileged user). @@ -1301,10 +1492,28 @@ def verify_group_id_not_present(profiled_user, pass_group_id, requested_cohort=s @patch('openedx.core.djangoapps.django_comment_common.comment_client.utils.requests.request', autospec=True) +@patch('openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', autospec=True) class FollowedThreadsDiscussionGroupIdTestCase(CohortedTestCase, CohortedTopicGroupIdTestMixin): # lint-amnesty, pylint: disable=missing-class-docstring cs_endpoint = "/subscribed_threads" - def call_view(self, mock_request, commentable_id, user, group_id, pass_group_id=True): + def setUp(self): + super().setUp() + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) + + def call_view( + self, + mock_is_forum_v2_enabled, + mock_request, + commentable_id, + user, + group_id, + pass_group_id=True + ): # pylint: disable=arguments-differ + mock_is_forum_v2_enabled.return_value = False kwargs = {} if group_id: kwargs['group_id'] = group_id @@ -1325,8 +1534,9 @@ def call_view(self, mock_request, commentable_id, user, group_id, pass_group_id= user.id ) - def test_group_info_in_ajax_response(self, mock_request): + def test_group_info_in_ajax_response(self, mock_is_forum_v2_enabled, mock_request): response = self.call_view( + mock_is_forum_v2_enabled, mock_request, "cohorted_topic", self.student, @@ -1528,6 +1738,22 @@ class CommentsServiceRequestHeadersTestCase(ForumsEnableMixin, UrlResetMixin, Mo @patch.dict("django.conf.settings.FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) def setUp(self): super().setUp() + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.models.forum_api.get_course_id_by_comment" + ) + self.mock_get_course_id_by_comment = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) username = "foo" password = "bar" @@ -1742,6 +1968,20 @@ def setUpClass(cls): with super().setUpClassAndTestData(): cls.course = CourseFactory.create(discussion_topics={'dummy_discussion_id': {'id': 'dummy_discussion_id'}}) + def setUp(self): + super().setUp() + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) + @classmethod def setUpTestData(cls): super().setUpTestData() @@ -1858,7 +2098,17 @@ class EnterpriseConsentTestCase(EnterpriseTestConsentRequired, ForumsEnableMixin def setUp(self): # Invoke UrlResetMixin setUp super().setUp() - + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) username = "foo" password = "bar" @@ -2195,6 +2445,17 @@ class ThreadViewedEventTestCase(EventTestMixin, ForumsEnableMixin, UrlResetMixin def setUp(self): # pylint: disable=arguments-differ super().setUp('lms.djangoapps.discussion.django_comment_client.base.views.tracker') + patcher = mock.patch( + 'openedx.core.djangoapps.discussions.config.waffle.ENABLE_FORUM_V2.is_enabled', + return_value=False + ) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch( + "openedx.core.djangoapps.django_comment_common.comment_client.thread.forum_api.get_course_id_by_thread" + ) + self.mock_get_course_id_by_thread = patcher.start() + self.addCleanup(patcher.stop) self.course = CourseFactory.create( teams_configuration=TeamsConfig({ 'topics': [{ diff --git a/lms/djangoapps/discussion/toggles.py b/lms/djangoapps/discussion/toggles.py index a1c292a4734f..a01a3b6a0a59 100644 --- a/lms/djangoapps/discussion/toggles.py +++ b/lms/djangoapps/discussion/toggles.py @@ -1,6 +1,7 @@ """ Discussions feature toggles """ + from openedx.core.djangoapps.discussions.config.waffle import WAFFLE_FLAG_NAMESPACE from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag @@ -11,4 +12,6 @@ # .. toggle_use_cases: temporary, open_edx # .. toggle_creation_date: 2021-11-05 # .. toggle_target_removal_date: 2022-12-05 -ENABLE_DISCUSSIONS_MFE = CourseWaffleFlag(f'{WAFFLE_FLAG_NAMESPACE}.enable_discussions_mfe', __name__) +ENABLE_DISCUSSIONS_MFE = CourseWaffleFlag( + f"{WAFFLE_FLAG_NAMESPACE}.enable_discussions_mfe", __name__ +) diff --git a/lms/djangoapps/grades/models.py b/lms/djangoapps/grades/models.py index a5608eb39afd..4dd5fb51f4ef 100644 --- a/lms/djangoapps/grades/models.py +++ b/lms/djangoapps/grades/models.py @@ -467,12 +467,6 @@ def update_or_create_grade(cls, **params): defaults=params, ) - # TODO: Remove as part of EDUCATOR-4602. - if str(usage_key.course_key) == 'course-v1:UQx+BUSLEAD5x+2T2019': - log.info('Created/updated grade ***{}*** for user ***{}*** in course ***{}***' - 'for subsection ***{}*** with default params ***{}***' - .format(grade, user_id, usage_key.course_key, usage_key, params)) - grade.override = PersistentSubsectionGradeOverride.get_override(user_id, usage_key) if first_attempted is not None and grade.first_attempted is None: grade.first_attempted = first_attempted @@ -822,11 +816,6 @@ def update_or_create_override( grade_defaults['override_reason'] = override_data['comment'] if 'comment' in override_data else None grade_defaults['system'] = override_data['system'] if 'system' in override_data else None - # TODO: Remove as part of EDUCATOR-4602. - if str(subsection_grade_model.course_id) == 'course-v1:UQx+BUSLEAD5x+2T2019': - log.info('Creating override for user ***{}*** for PersistentSubsectionGrade' - '***{}*** with override data ***{}*** and derived grade_defaults ***{}***.' - .format(requesting_user, subsection_grade_model, override_data, grade_defaults)) try: override = PersistentSubsectionGradeOverride.objects.get(grade=subsection_grade_model) for key, value in grade_defaults.items(): diff --git a/lms/djangoapps/grades/rest_api/v1/gradebook_views.py b/lms/djangoapps/grades/rest_api/v1/gradebook_views.py index 26df7ce2583d..c295563da565 100644 --- a/lms/djangoapps/grades/rest_api/v1/gradebook_views.py +++ b/lms/djangoapps/grades/rest_api/v1/gradebook_views.py @@ -859,11 +859,6 @@ def post(self, request, course_key): subsection = course.get_child(usage_key) if subsection: subsection_grade_model = self._create_subsection_grade(user, course, subsection) - # TODO: Remove as part of EDUCATOR-4602. - if str(course_key) == 'course-v1:UQx+BUSLEAD5x+2T2019': - log.info('PersistentSubsectionGrade ***{}*** created for' - ' subsection ***{}*** in course ***{}*** for user ***{}***.' - .format(subsection_grade_model, subsection.location, course, user.id)) else: self._log_update_result(request.user, requested_user_id, requested_usage_id, success=False) result.append(GradebookUpdateResponseItem( diff --git a/lms/djangoapps/grades/scores.py b/lms/djangoapps/grades/scores.py index 38dd0dc18926..7a89a88c794b 100644 --- a/lms/djangoapps/grades/scores.py +++ b/lms/djangoapps/grades/scores.py @@ -102,10 +102,6 @@ def get_score(submissions_scores, csm_scores, persisted_block, block): weight, graded - retrieved from the latest block content """ weight = _get_weight_from_block(persisted_block, block) - # TODO: Remove as part of EDUCATOR-4602. - if str(block.location.course_key) == 'course-v1:UQx+BUSLEAD5x+2T2019': - log.info('Weight for block: ***{}*** is {}' - .format(str(block.location), weight)) # Priority order for retrieving the scores: # submissions API -> CSM -> grades persisted block -> latest block content @@ -115,13 +111,6 @@ def get_score(submissions_scores, csm_scores, persisted_block, block): _get_score_from_persisted_or_latest_block(persisted_block, block, weight) ) - # TODO: Remove as part of EDUCATOR-4602. - if str(block.location.course_key) == 'course-v1:UQx+BUSLEAD5x+2T2019': - log.info('Calculated raw-earned: {}, raw_possible: {}, weighted_earned: ' - '{}, weighted_possible: {}, first_attempted: {} for block: ***{}***.' - .format(raw_earned, raw_possible, weighted_earned, - weighted_possible, first_attempted, str(block.location))) - if weighted_possible is None or weighted_earned is None: return None @@ -219,11 +208,6 @@ def _get_score_from_persisted_or_latest_block(persisted_block, block, weight): Uses the raw_possible value from the persisted_block if found, else from the latest block content. """ - # TODO: Remove as part of EDUCATOR-4602. - if str(block.location.course_key) == 'course-v1:UQx+BUSLEAD5x+2T2019': - log.info('Using _get_score_from_persisted_or_latest_block to calculate score for block: ***{}***.'.format( - str(block.location) - )) raw_earned = 0.0 first_attempted = None @@ -231,10 +215,6 @@ def _get_score_from_persisted_or_latest_block(persisted_block, block, weight): raw_possible = persisted_block.raw_possible else: raw_possible = block.transformer_data[GradesTransformer].max_score - # TODO: Remove as part of EDUCATOR-4602. - if str(block.location.course_key) == 'course-v1:UQx+BUSLEAD5x+2T2019': - log.info('Using latest block content to calculate score for block: ***{}***.') - log.info(f'weight for block: ***{str(block.location)}*** is {raw_possible}.') # TODO TNL-5982 remove defensive code for scorables without max_score if raw_possible is None: diff --git a/lms/djangoapps/grades/subsection_grade.py b/lms/djangoapps/grades/subsection_grade.py index ba098a92a417..4ce0a1f3a463 100644 --- a/lms/djangoapps/grades/subsection_grade.py +++ b/lms/djangoapps/grades/subsection_grade.py @@ -170,39 +170,21 @@ def _compute_block_score( # lint-amnesty, pylint: disable=missing-function-docs csm_scores, persisted_block=None, ): - # TODO: Remove as part of EDUCATOR-4602. - if str(block_key.course_key) == 'course-v1:UQx+BUSLEAD5x+2T2019': - log.info('Computing block score for block: ***{}*** in course: ***{}***.'.format( - str(block_key), - str(block_key.course_key), - )) try: block = course_structure[block_key] except KeyError: - # TODO: Remove as part of EDUCATOR-4602. - if str(block_key.course_key) == 'course-v1:UQx+BUSLEAD5x+2T2019': - log.info('User\'s access to block: ***{}*** in course: ***{}*** has changed. ' - 'No block score calculated.'.format(str(block_key), str(block_key.course_key))) # It's possible that the user's access to that # block has changed since the subsection grade # was last persisted. + pass else: if getattr(block, 'has_score', False): - # TODO: Remove as part of EDUCATOR-4602. - if str(block_key.course_key) == 'course-v1:UQx+BUSLEAD5x+2T2019': - log.info('Block: ***{}*** in course: ***{}*** HAS has_score attribute. Continuing.' - .format(str(block_key), str(block_key.course_key))) return get_score( submissions_scores, csm_scores, persisted_block, block, ) - # TODO: Remove as part of EDUCATOR-4602. - if str(block_key.course_key) == 'course-v1:UQx+BUSLEAD5x+2T2019': - log.info('Block: ***{}*** in course: ***{}*** DOES NOT HAVE has_score attribute. ' - 'No block score calculated.' - .format(str(block_key), str(block_key.course_key))) @staticmethod def _aggregated_score_from_model(grade_model, is_graded): @@ -283,23 +265,11 @@ def __init__(self, subsection, course_structure, submissions_scores, csm_scores) start_node=subsection.location, ): problem_score = self._compute_block_score(block_key, course_structure, submissions_scores, csm_scores) - - # TODO: Remove as part of EDUCATOR-4602. - if str(block_key.course_key) == 'course-v1:UQx+BUSLEAD5x+2T2019': - log.info('Calculated problem score ***{}*** for block ***{!s}***' - ' in subsection ***{}***.' - .format(problem_score, block_key, subsection.location)) if problem_score: self.problem_scores[block_key] = problem_score all_total, graded_total = graders.aggregate_scores(list(self.problem_scores.values())) - # TODO: Remove as part of EDUCATOR-4602. - if str(subsection.location.course_key) == 'course-v1:UQx+BUSLEAD5x+2T2019': - log.info('Calculated aggregate all_total ***{}***' - ' and grade_total ***{}*** for subsection ***{}***' - .format(all_total, graded_total, subsection.location)) - super().__init__(subsection, all_total, graded_total) def update_or_create_model(self, student, score_deleted=False, force_update_subsections=False): @@ -307,11 +277,6 @@ def update_or_create_model(self, student, score_deleted=False, force_update_subs Saves or updates the subsection grade in a persisted model. """ if self._should_persist_per_attempted(score_deleted, force_update_subsections): - # TODO: Remove as part of EDUCATOR-4602. - if str(self.location.course_key) == 'course-v1:UQx+BUSLEAD5x+2T2019': - log.info('Updating PersistentSubsectionGrade for student ***{}*** in' - ' subsection ***{}*** with params ***{}***.' - .format(student.id, self.location, self._persisted_model_params(student))) model = PersistentSubsectionGrade.update_or_create_grade(**self._persisted_model_params(student)) if hasattr(model, 'override'): diff --git a/lms/djangoapps/instructor/tests/test_api.py b/lms/djangoapps/instructor/tests/test_api.py index 4d3e413f5a2a..f1e7322abc6b 100644 --- a/lms/djangoapps/instructor/tests/test_api.py +++ b/lms/djangoapps/instructor/tests/test_api.py @@ -1990,6 +1990,15 @@ def test_add_notenrolled_username_autoenroll(self): self.add_notenrolled(response, self.notenrolled_student.username) assert CourseEnrollment.is_enrolled(self.notenrolled_student, self.course.id) + def test_add_notenrolled_username_autoenroll_with_multiple_users(self): + url = reverse('bulk_beta_modify_access', kwargs={'course_id': str(self.course.id)}) + identifiers = (f"Lorem@ipsum.dolor, " + f"sit@amet.consectetur\nadipiscing@elit.Aenean\r convallis@at.lacus\r, ut@lacinia.Sed, " + f"{self.notenrolled_student.username}" + ) + response = self.client.post(url, {'identifiers': identifiers, 'action': 'add', 'email_students': False, 'auto_enroll': True}) # lint-amnesty, pylint: disable=line-too-long + assert 6, len(json.loads(response.content.decode())['results']) + @ddt.data('http', 'https') def test_add_notenrolled_with_email(self, protocol): url = reverse('bulk_beta_modify_access', kwargs={'course_id': str(self.course.id)}) diff --git a/lms/djangoapps/instructor/views/api.py b/lms/djangoapps/instructor/views/api.py index 631500fe3246..43c9c1947c71 100644 --- a/lms/djangoapps/instructor/views/api.py +++ b/lms/djangoapps/instructor/views/api.py @@ -109,6 +109,7 @@ CertificateSerializer, CertificateStatusesSerializer, ListInstructorTaskInputSerializer, + ModifyAccessSerializer, RoleNameSerializer, SendEmailSerializer, ShowStudentExtensionSerializer, @@ -914,88 +915,91 @@ def students_update_enrollment(request, course_id): # lint-amnesty, pylint: dis return JsonResponse(response_payload) -@require_POST -@ensure_csrf_cookie -@cache_control(no_cache=True, no_store=True, must_revalidate=True) -@require_course_permission(permissions.CAN_BETATEST) -@common_exceptions_400 -@require_post_params( - identifiers="stringified list of emails and/or usernames", - action="add or remove", -) -def bulk_beta_modify_access(request, course_id): +@method_decorator(cache_control(no_cache=True, no_store=True, must_revalidate=True), name='dispatch') +class BulkBetaModifyAccess(DeveloperErrorViewMixin, APIView): """ Enroll or unenroll users in beta testing program. - - Query parameters: - - identifiers is string containing a list of emails and/or usernames separated by - anything split_input_list can handle. - - action is one of ['add', 'remove'] """ - course_id = CourseKey.from_string(course_id) - action = request.POST.get('action') - identifiers_raw = request.POST.get('identifiers') - identifiers = _split_input_list(identifiers_raw) - email_students = _get_boolean_param(request, 'email_students') - auto_enroll = _get_boolean_param(request, 'auto_enroll') - results = [] - rolename = 'beta' - course = get_course_by_id(course_id) + permission_classes = (IsAuthenticated, permissions.InstructorPermission) + permission_name = permissions.CAN_BETATEST + serializer_class = ModifyAccessSerializer - email_params = {} - if email_students: - secure = request.is_secure() - email_params = get_email_params(course, auto_enroll=auto_enroll, secure=secure) + @method_decorator(ensure_csrf_cookie) + def post(self, request, course_id): + """ + Query parameters: + - identifiers is string containing a list of emails and/or usernames separated by + anything split_input_list can handle. + - action is one of ['add', 'remove'] + """ + course_id = CourseKey.from_string(course_id) + serializer = self.serializer_class(data=request.data) + if not serializer.is_valid(): + return JsonResponse({'message': serializer.errors}, status=400) - for identifier in identifiers: - try: - error = False - user_does_not_exist = False - user = get_student_from_identifier(identifier) - user_active = user.is_active + action = serializer.validated_data['action'] + identifiers = serializer.validated_data['identifiers'] + email_students = serializer.validated_data['email_students'] + auto_enroll = serializer.validated_data['auto_enroll'] - if action == 'add': - allow_access(course, user, rolename) - elif action == 'remove': - revoke_access(course, user, rolename) + results = [] + rolename = 'beta' + course = get_course_by_id(course_id) + + email_params = {} + if email_students: + secure = request.is_secure() + email_params = get_email_params(course, auto_enroll=auto_enroll, secure=secure) + + for identifier in identifiers: + try: + error = False + user_does_not_exist = False + user = get_student_from_identifier(identifier) + user_active = user.is_active + + if action == 'add': + allow_access(course, user, rolename) + elif action == 'remove': + revoke_access(course, user, rolename) + else: + return HttpResponseBadRequest(strip_tags( + f"Unrecognized action '{action}'" + )) + except User.DoesNotExist: + error = True + user_does_not_exist = True + user_active = None + # catch and log any unexpected exceptions + # so that one error doesn't cause a 500. + except Exception as exc: # pylint: disable=broad-except + log.exception("Error while #{}ing student") + log.exception(exc) + error = True else: - return HttpResponseBadRequest(strip_tags( - f"Unrecognized action '{action}'" - )) - except User.DoesNotExist: - error = True - user_does_not_exist = True - user_active = None - # catch and log any unexpected exceptions - # so that one error doesn't cause a 500. - except Exception as exc: # pylint: disable=broad-except - log.exception("Error while #{}ing student") - log.exception(exc) - error = True - else: - # If no exception thrown, see if we should send an email - if email_students: - send_beta_role_email(action, user, email_params) - # See if we should autoenroll the student - if auto_enroll: - # Check if student is already enrolled - if not is_user_enrolled_in_course(user, course_id): - CourseEnrollment.enroll(user, course_id) + # If no exception thrown, see if we should send an email + if email_students: + send_beta_role_email(action, user, email_params) + # See if we should autoenroll the student + if auto_enroll: + # Check if student is already enrolled + if not is_user_enrolled_in_course(user, course_id): + CourseEnrollment.enroll(user, course_id) - finally: - # Tabulate the action result of this email address - results.append({ - 'identifier': identifier, - 'error': error, # pylint: disable=used-before-assignment - 'userDoesNotExist': user_does_not_exist, # pylint: disable=used-before-assignment - 'is_active': user_active # pylint: disable=used-before-assignment - }) + finally: + # Tabulate the action result of this email address + results.append({ + 'identifier': identifier, + 'error': error, # pylint: disable=used-before-assignment + 'userDoesNotExist': user_does_not_exist, # pylint: disable=used-before-assignment + 'is_active': user_active # pylint: disable=used-before-assignment + }) - response_payload = { - 'action': action, - 'results': results, - } - return JsonResponse(response_payload) + response_payload = { + 'action': action, + 'results': results, + } + return JsonResponse(response_payload) @method_decorator(cache_control(no_cache=True, no_store=True, must_revalidate=True), name='dispatch') @@ -1025,7 +1029,6 @@ def post(self, request, course_id): course = get_course_with_access( request.user, 'instructor', course_id, depth=None ) - serializer_data = AccessSerializer(data=request.data) if not serializer_data.is_valid(): return HttpResponseBadRequest(reason=serializer_data.errors) diff --git a/lms/djangoapps/instructor/views/api_urls.py b/lms/djangoapps/instructor/views/api_urls.py index ea27034d0942..6f67a1a6863c 100644 --- a/lms/djangoapps/instructor/views/api_urls.py +++ b/lms/djangoapps/instructor/views/api_urls.py @@ -25,7 +25,7 @@ path('register_and_enroll_students', api.RegisterAndEnrollStudents.as_view(), name='register_and_enroll_students'), path('list_course_role_members', api.ListCourseRoleMembersView.as_view(), name='list_course_role_members'), path('modify_access', api.ModifyAccess.as_view(), name='modify_access'), - path('bulk_beta_modify_access', api.bulk_beta_modify_access, name='bulk_beta_modify_access'), + path('bulk_beta_modify_access', api.BulkBetaModifyAccess.as_view(), name='bulk_beta_modify_access'), path('get_problem_responses', api.get_problem_responses, name='get_problem_responses'), path('get_issued_certificates/', api.GetIssuedCertificates.as_view(), name='get_issued_certificates'), re_path(r'^get_students_features(?P/csv)?$', api.GetStudentsFeatures.as_view(), name='get_students_features'), diff --git a/lms/djangoapps/instructor/views/serializer.py b/lms/djangoapps/instructor/views/serializer.py index f7fc685f658c..7b20eed32f60 100644 --- a/lms/djangoapps/instructor/views/serializer.py +++ b/lms/djangoapps/instructor/views/serializer.py @@ -1,4 +1,5 @@ """ Instructor apis serializers. """ +import re from django.contrib.auth.models import User # lint-amnesty, pylint: disable=imported-auth-user from django.core.exceptions import ValidationError @@ -232,6 +233,76 @@ def __init__(self, *args, **kwargs): self.fields['due_datetime'].required = False +class ModifyAccessSerializer(serializers.Serializer): + """ + serializers for enroll or un-enroll users in beta testing program. + """ + identifiers = serializers.CharField( + help_text="A comma separated list of emails or usernames.", + required=True + ) + action = serializers.ChoiceField( + choices=["add", "remove"], + help_text="Action to perform: add or remove.", + required=True + ) + + email_students = serializers.BooleanField( + default=False, + help_text="Boolean flag to indicate if students should be emailed." + ) + + auto_enroll = serializers.BooleanField( + default=False, + help_text="Boolean flag to indicate if the user should be auto-enrolled." + ) + + def validate_identifiers(self, value): + """ + Validate the 'identifiers' field which is now a list of strings. + """ + # Iterate over the list of identifiers and validate each one + validated_list = _split_input_list(value) + if not validated_list: + raise serializers.ValidationError("The identifiers list cannot be empty.") + + return validated_list + + def validate_email_students(self, value): + """ + handle string values like 'true' or 'false'. + """ + if isinstance(value, str): + return value.lower() == 'true' + return bool(value) + + def validate_auto_enroll(self, value): + """ + handle string values like 'true' or 'false'. + """ + if isinstance(value, str): + return value.lower() == 'true' + return bool(value) + + +def _split_input_list(str_list): + """ + Separate out individual student email from the comma, or space separated string. + + e.g. + in: "Lorem@ipsum.dolor, sit@amet.consectetur\nadipiscing@elit.Aenean\r convallis@at.lacus\r, ut@lacinia.Sed" + out: ['Lorem@ipsum.dolor', 'sit@amet.consectetur', 'adipiscing@elit.Aenean', 'convallis@at.lacus', 'ut@lacinia.Sed'] + + `str_list` is a string coming from an input text area + returns a list of separated values + """ + new_list = re.split(r'[,\s\n\r]+', str_list) + new_list = [s.strip() for s in new_list] + new_list = [s for s in new_list if s != ''] + + return new_list + + class CertificateStatusesSerializer(serializers.Serializer): """ Serializer for validating and serializing certificate status inputs. diff --git a/lms/envs/common.py b/lms/envs/common.py index 4fcb0a8126bc..e354f75a8530 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -1059,18 +1059,6 @@ # .. toggle_creation_date: 2024-04-24 'ENABLE_COURSEWARE_SEARCH_VERIFIED_ENROLLMENT_REQUIRED': False, - # .. toggle_name: FEATURES['ENABLE_BLAKE2B_HASHING'] - # .. toggle_implementation: DjangoSetting - # .. toggle_default: False - # .. toggle_description: Enables the memcache to use the blake2b hash algorithm instead of depreciated md4 for keys - # exceeding 250 characters - # .. toggle_use_cases: open_edx - # .. toggle_creation_date: 2024-04-02 - # .. toggle_target_removal_date: 2024-12-09 - # .. toggle_warning: For consistency, keep the value in sync with the setting of the same name in the LMS and CMS. - # .. toggle_tickets: https://github.com/openedx/edx-platform/pull/34442 - 'ENABLE_BLAKE2B_HASHING': False, - # .. toggle_name: FEATURES['BADGES_ENABLED'] # .. toggle_implementation: DjangoSetting # .. toggle_default: False diff --git a/lms/envs/minimal.yml b/lms/envs/minimal.yml index 51d7bbf499c4..003bc243764f 100644 --- a/lms/envs/minimal.yml +++ b/lms/envs/minimal.yml @@ -2,7 +2,8 @@ # # This is the minimal settings you need to set to be able to get django to # load when using the production.py settings files. It's useful to point -# LMS_CFG and CMS_CFG to this file to be able to run various paver commands +# LMS_CFG and CMS_CFG to this file to be able to run various npm commands +# and make targets (e.g.: building assets, pulling translations, etc.) # without needing a full docker setup. # # Follow up work will likely be done as a part of diff --git a/lms/templates/courseware/static_tab.html b/lms/templates/courseware/static_tab.html index 38b08f85ee76..cf79fb8948f7 100644 --- a/lms/templates/courseware/static_tab.html +++ b/lms/templates/courseware/static_tab.html @@ -33,7 +33,6 @@ lang=${course.language} % endif > -
${HTML(fragment.body_html())}
diff --git a/lms/templates/vert_module.html b/lms/templates/vert_module.html index 7df5b8b5b46e..909e7db829c1 100644 --- a/lms/templates/vert_module.html +++ b/lms/templates/vert_module.html @@ -62,7 +62,7 @@

${unit_title}

% for idx, item in enumerate(items): % if item['content']: -
+
${HTML(item['content'])}
%endif diff --git a/openedx/core/djangoapps/content/learning_sequences/models.py b/openedx/core/djangoapps/content/learning_sequences/models.py index 5d2ee34bbc9f..924403f408aa 100644 --- a/openedx/core/djangoapps/content/learning_sequences/models.py +++ b/openedx/core/djangoapps/content/learning_sequences/models.py @@ -37,6 +37,8 @@ yourself to the LearningContext and LearningSequence models. Other tables are not guaranteed to stick around, and values may be deleted unexpectedly. """ +from __future__ import annotations + from django.db import models from model_utils.models import TimeStampedModel @@ -214,7 +216,7 @@ class CourseSection(CourseContentVisibilityMixin, TimeStampedModel): # What is our position within the Course? (starts with 0) ordering = models.PositiveIntegerField(null=False) - new_user_partition_groups = models.ManyToManyField( + new_user_partition_groups: models.ManyToManyField[UserPartitionGroup, models.Model] = models.ManyToManyField( UserPartitionGroup, db_index=True, related_name='sec_user_partition_groups', @@ -280,7 +282,7 @@ class CourseSectionSequence(CourseContentVisibilityMixin, TimeStampedModel): # sequences across 20 sections, the numbering here would be 0-199. ordering = models.PositiveIntegerField(null=False) - new_user_partition_groups = models.ManyToManyField( + new_user_partition_groups: models.ManyToManyField[UserPartitionGroup, models.Model] = models.ManyToManyField( UserPartitionGroup, db_index=True, related_name='secseq_user_partition_groups', diff --git a/openedx/core/djangoapps/content/search/documents.py b/openedx/core/djangoapps/content/search/documents.py index 0dd02683ceea..40fe4529272b 100644 --- a/openedx/core/djangoapps/content/search/documents.py +++ b/openedx/core/djangoapps/content/search/documents.py @@ -306,7 +306,10 @@ def _collections_for_content_object(object_id: UsageKey | LearningContextKey) -> If the object is in no collections, returns: { - "collections": {}, + "collections": { + "display_name": [], + "key": [], + }, } """ diff --git a/openedx/core/djangoapps/content/search/tests/test_handlers.py b/openedx/core/djangoapps/content/search/tests/test_handlers.py index 3577cbfc5692..dc274d182696 100644 --- a/openedx/core/djangoapps/content/search/tests/test_handlers.py +++ b/openedx/core/djangoapps/content/search/tests/test_handlers.py @@ -185,3 +185,13 @@ def test_create_delete_library_block(self, meilisearch_client): meilisearch_client.return_value.index.return_value.delete_document.assert_called_with( "lborgalib_aproblemproblem1-ca3186e9" ) + + # Restore the Library Block + library_api.restore_library_block(problem.usage_key) + meilisearch_client.return_value.index.return_value.update_documents.assert_any_call([doc_problem]) + meilisearch_client.return_value.index.return_value.update_documents.assert_any_call( + [{'id': doc_problem['id'], 'collections': {'display_name': [], 'key': []}}] + ) + meilisearch_client.return_value.index.return_value.update_documents.assert_any_call( + [{'id': doc_problem['id'], 'tags': {}}] + ) diff --git a/openedx/core/djangoapps/content_libraries/api.py b/openedx/core/djangoapps/content_libraries/api.py index 5195826468c2..c51c707fc470 100644 --- a/openedx/core/djangoapps/content_libraries/api.py +++ b/openedx/core/djangoapps/content_libraries/api.py @@ -1133,6 +1133,46 @@ def delete_library_block(usage_key, remove_from_parent=True): ) +def restore_library_block(usage_key): + """ + Restore the specified library block. + """ + component = get_component_from_usage_key(usage_key) + library_key = usage_key.context_key + affected_collections = authoring_api.get_entity_collections(component.learning_package_id, component.key) + + # Set draft version back to the latest available component version id. + authoring_api.set_draft_version(component.pk, component.versioning.latest.pk) + + LIBRARY_BLOCK_CREATED.send_event( + library_block=LibraryBlockData( + library_key=library_key, + usage_key=usage_key + ) + ) + + # Add tags and collections back to index + CONTENT_OBJECT_ASSOCIATIONS_CHANGED.send_event( + content_object=ContentObjectChangedData( + object_id=str(usage_key), + changes=["collections", "tags"], + ), + ) + + # For each collection, trigger LIBRARY_COLLECTION_UPDATED signal and set background=True to trigger + # collection indexing asynchronously. + # + # To restore the component in the collections + for collection in affected_collections: + LIBRARY_COLLECTION_UPDATED.send_event( + library_collection=LibraryCollectionData( + library_key=library_key, + collection_key=collection.key, + background=True, + ) + ) + + def get_library_block_static_asset_files(usage_key) -> list[LibraryXBlockStaticFile]: """ Given an XBlock in a content library, list all the static asset files diff --git a/openedx/core/djangoapps/content_libraries/serializers.py b/openedx/core/djangoapps/content_libraries/serializers.py index d639ed63afb0..c2a26220d4c7 100644 --- a/openedx/core/djangoapps/content_libraries/serializers.py +++ b/openedx/core/djangoapps/content_libraries/serializers.py @@ -276,7 +276,7 @@ class ContentLibraryCollectionUpdateSerializer(serializers.Serializer): description = serializers.CharField(allow_blank=True) -class UsageKeyV2Serializer(serializers.Serializer): +class UsageKeyV2Serializer(serializers.BaseSerializer): """ Serializes a UsageKeyV2. """ diff --git a/openedx/core/djangoapps/content_libraries/tests/test_api.py b/openedx/core/djangoapps/content_libraries/tests/test_api.py index 7be3e592ba9d..203cc7a9397a 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_api.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_api.py @@ -591,6 +591,35 @@ def test_delete_library_block(self): event_receiver.call_args_list[0].kwargs, ) + def test_restore_library_block(self): + api.update_library_collection_components( + self.lib1.library_key, + self.col1.key, + usage_keys=[ + UsageKey.from_string(self.lib1_problem_block["id"]), + UsageKey.from_string(self.lib1_html_block["id"]), + ], + ) + + event_receiver = mock.Mock() + LIBRARY_COLLECTION_UPDATED.connect(event_receiver) + + api.restore_library_block(UsageKey.from_string(self.lib1_problem_block["id"])) + + assert event_receiver.call_count == 1 + self.assertDictContainsSubset( + { + "signal": LIBRARY_COLLECTION_UPDATED, + "sender": None, + "library_collection": LibraryCollectionData( + self.lib1.library_key, + collection_key=self.col1.key, + background=True, + ), + }, + event_receiver.call_args_list[0].kwargs, + ) + def test_add_component_and_revert(self): # Add component and publish api.update_library_collection_components( diff --git a/openedx/core/djangoapps/content_libraries/tests/test_static_assets.py b/openedx/core/djangoapps/content_libraries/tests/test_static_assets.py index 69f5cd2d797c..91d9556b01f1 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_static_assets.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_static_assets.py @@ -63,6 +63,13 @@ def test_asset_filenames(self): file_name = "a////////b" self._set_library_block_asset(block_id, file_name, SVG_DATA, expect_response=400) + # Names with spaces are allowed but replaced with underscores + file_name_with_space = "o w o.svg" + self._set_library_block_asset(block_id, file_name_with_space, SVG_DATA) + file_name = "o_w_o.svg" + assert self._get_library_block_asset(block_id, file_name)['path'] == file_name + assert self._get_library_block_asset(block_id, file_name)['size'] == file_size + def test_video_transcripts(self): """ Test that video blocks can read transcript files out of learning core. diff --git a/openedx/core/djangoapps/content_libraries/urls.py b/openedx/core/djangoapps/content_libraries/urls.py index 857126eef7c9..1272d79b3873 100644 --- a/openedx/core/djangoapps/content_libraries/urls.py +++ b/openedx/core/djangoapps/content_libraries/urls.py @@ -57,6 +57,7 @@ path('blocks//', include([ # Get metadata about a specific XBlock in this library, or delete the block: path('', views.LibraryBlockView.as_view()), + path('restore/', views.LibraryBlockRestore.as_view()), # Update collections for a given component path('collections/', views.LibraryBlockCollectionsView.as_view(), name='update-collections'), # Get the LTI URL of a specific XBlock diff --git a/openedx/core/djangoapps/content_libraries/views.py b/openedx/core/djangoapps/content_libraries/views.py index 3dc7f538df86..4c14651c7961 100644 --- a/openedx/core/djangoapps/content_libraries/views.py +++ b/openedx/core/djangoapps/content_libraries/views.py @@ -122,6 +122,7 @@ from openedx.core.lib.api.view_utils import view_auth_classes from openedx.core.djangoapps.safe_sessions.middleware import mark_user_change_as_expected from openedx.core.djangoapps.xblock import api as xblock_api +from openedx.core.types.http import RestRequest from .models import ContentLibrary, LtiGradedResource, LtiProfile @@ -644,6 +645,22 @@ def delete(self, request, usage_key_str): # pylint: disable=unused-argument return Response({}) +@view_auth_classes() +class LibraryBlockRestore(APIView): + """ + View to restore soft-deleted library xblocks. + """ + @convert_exceptions + def post(self, request, usage_key_str) -> Response: + """ + Restores a soft-deleted library block that belongs to a Content Library + """ + key = LibraryUsageLocatorV2.from_string(usage_key_str) + api.require_permission_for_library_key(key.lib_key, request.user, permissions.CAN_EDIT_THIS_CONTENT_LIBRARY) + api.restore_library_block(key) + return Response(None, status=status.HTTP_204_NO_CONTENT) + + @method_decorator(non_atomic_requests, name="dispatch") @view_auth_classes() class LibraryBlockCollectionsView(APIView): @@ -651,7 +668,7 @@ class LibraryBlockCollectionsView(APIView): View to set collections for a component. """ @convert_exceptions - def patch(self, request, usage_key_str) -> Response: + def patch(self, request: RestRequest, usage_key_str) -> Response: """ Sets Collections for a Component. @@ -672,7 +689,7 @@ def patch(self, request, usage_key_str) -> Response: library_key=key.lib_key, component=component, collection_keys=collection_keys, - created_by=self.request.user.id, + created_by=request.user.id, content_library=content_library, ) @@ -783,6 +800,7 @@ def put(self, request, usage_key_str, file_path): """ Replace a static asset file belonging to this block. """ + file_path = file_path.replace(" ", "_") # Messes up url/name correspondence due to URL encoding. usage_key = LibraryUsageLocatorV2.from_string(usage_key_str) api.require_permission_for_library_key( usage_key.lib_key, request.user, permissions.CAN_EDIT_THIS_CONTENT_LIBRARY, diff --git a/openedx/core/djangoapps/content_libraries/views_collections.py b/openedx/core/djangoapps/content_libraries/views_collections.py index b6c1c999ba94..21c4b12dd3da 100644 --- a/openedx/core/djangoapps/content_libraries/views_collections.py +++ b/openedx/core/djangoapps/content_libraries/views_collections.py @@ -25,6 +25,7 @@ ContentLibraryCollectionComponentsUpdateSerializer, ContentLibraryCollectionUpdateSerializer, ) +from openedx.core.types.http import RestRequest class LibraryCollectionsView(ModelViewSet): @@ -89,7 +90,7 @@ def get_object(self) -> Collection: return collection @convert_exceptions - def retrieve(self, request, *args, **kwargs) -> Response: + def retrieve(self, request: RestRequest, *args, **kwargs) -> Response: """ Retrieve the Content Library Collection """ @@ -97,7 +98,7 @@ def retrieve(self, request, *args, **kwargs) -> Response: return super().retrieve(request, *args, **kwargs) @convert_exceptions - def list(self, request, *args, **kwargs) -> Response: + def list(self, request: RestRequest, *args, **kwargs) -> Response: """ List Collections that belong to Content Library """ @@ -105,7 +106,7 @@ def list(self, request, *args, **kwargs) -> Response: return super().list(request, *args, **kwargs) @convert_exceptions - def create(self, request, *args, **kwargs) -> Response: + def create(self, request: RestRequest, *args, **kwargs) -> Response: """ Create a Collection that belongs to a Content Library """ @@ -139,7 +140,7 @@ def create(self, request, *args, **kwargs) -> Response: return Response(serializer.data) @convert_exceptions - def partial_update(self, request, *args, **kwargs) -> Response: + def partial_update(self, request: RestRequest, *args, **kwargs) -> Response: """ Update a Collection that belongs to a Content Library """ @@ -161,7 +162,7 @@ def partial_update(self, request, *args, **kwargs) -> Response: return Response(serializer.data) @convert_exceptions - def destroy(self, request, *args, **kwargs) -> Response: + def destroy(self, request: RestRequest, *args, **kwargs) -> Response: """ Soft-deletes a Collection that belongs to a Content Library """ @@ -176,7 +177,7 @@ def destroy(self, request, *args, **kwargs) -> Response: @convert_exceptions @action(detail=True, methods=['post'], url_path='restore', url_name='collection-restore') - def restore(self, request, *args, **kwargs) -> Response: + def restore(self, request: RestRequest, *args, **kwargs) -> Response: """ Restores a soft-deleted Collection that belongs to a Content Library """ @@ -191,7 +192,7 @@ def restore(self, request, *args, **kwargs) -> Response: @convert_exceptions @action(detail=True, methods=['delete', 'patch'], url_path='components', url_name='components-update') - def update_components(self, request, *args, **kwargs) -> Response: + def update_components(self, request: RestRequest, *args, **kwargs) -> Response: """ Adds (PATCH) or removes (DELETE) Components to/from a Collection. @@ -209,7 +210,7 @@ def update_components(self, request, *args, **kwargs) -> Response: content_library=content_library, collection_key=collection_key, usage_keys=usage_keys, - created_by=self.request.user.id, + created_by=request.user.id, remove=(request.method == "DELETE"), ) diff --git a/openedx/core/djangoapps/content_staging/api.py b/openedx/core/djangoapps/content_staging/api.py index f0432922dcb0..5f85d701faa5 100644 --- a/openedx/core/djangoapps/content_staging/api.py +++ b/openedx/core/djangoapps/content_staging/api.py @@ -66,7 +66,7 @@ def save_xblock_to_user_clipboard(block: XBlock, user_id: int, version_num: int olx=block_data.olx_str, display_name=block_metadata_utils.display_name_with_default(block), suggested_url_name=usage_key.block_id, - tags=block_data.tags, + tags=block_data.tags or {}, version_num=(version_num or 0), ) (clipboard, _created) = _UserClipboard.objects.update_or_create(user_id=user_id, defaults={ @@ -209,7 +209,7 @@ def _user_clipboard_model_to_data(clipboard: _UserClipboard) -> UserClipboardDat status=content.status, block_type=content.block_type, display_name=content.display_name, - tags=content.tags, + tags=content.tags or {}, version_num=content.version_num, ), source_usage_key=clipboard.source_usage_key, diff --git a/openedx/core/djangoapps/content_staging/models.py b/openedx/core/djangoapps/content_staging/models.py index 2eab7954e826..5e007bc4485a 100644 --- a/openedx/core/djangoapps/content_staging/models.py +++ b/openedx/core/djangoapps/content_staging/models.py @@ -67,7 +67,9 @@ class Meta: version_num = models.PositiveIntegerField(default=0) # Tags applied to the original source block(s) will be copied to the new block(s) on paste. - tags = models.JSONField(null=True, help_text=_("Content tags applied to these blocks")) + tags: models.JSONField[dict | None, dict | None] = models.JSONField( + null=True, help_text=_("Content tags applied to these blocks") + ) @property def olx_filename(self) -> str: diff --git a/openedx/core/djangoapps/content_tagging/handlers.py b/openedx/core/djangoapps/content_tagging/handlers.py index cc86f7e0dcd6..86cbb7167cbe 100644 --- a/openedx/core/djangoapps/content_tagging/handlers.py +++ b/openedx/core/djangoapps/content_tagging/handlers.py @@ -20,7 +20,6 @@ XBLOCK_DUPLICATED, LIBRARY_BLOCK_CREATED, LIBRARY_BLOCK_UPDATED, - LIBRARY_BLOCK_DELETED, ) from .api import copy_object_tags @@ -30,7 +29,6 @@ update_course_tags, update_xblock_tags, update_library_block_tags, - delete_library_block_tags, ) from .toggles import CONTENT_TAGGING_AUTO @@ -119,22 +117,6 @@ def auto_tag_library_block(**kwargs): ) -@receiver(LIBRARY_BLOCK_DELETED) -def delete_tag_library_block(**kwargs): - """ - Delete tags associated with a Library XBlock whenever the block is deleted. - """ - library_block_data = kwargs.get("library_block", None) - if not library_block_data or not isinstance(library_block_data, LibraryBlockData): - log.error("Received null or incorrect data for event") - return - - try: - delete_library_block_tags(str(library_block_data.usage_key)) - except Exception as err: # pylint: disable=broad-except - log.error(f"Failed to delete library block tags: {err}") - - @receiver(XBLOCK_DUPLICATED) def duplicate_tags(**kwargs): """ diff --git a/openedx/core/djangoapps/content_tagging/rest_api/v1/views.py b/openedx/core/djangoapps/content_tagging/rest_api/v1/views.py index c2f79ef677db..615406ffccc5 100644 --- a/openedx/core/djangoapps/content_tagging/rest_api/v1/views.py +++ b/openedx/core/djangoapps/content_tagging/rest_api/v1/views.py @@ -10,7 +10,6 @@ from rest_framework import status from rest_framework.decorators import action from rest_framework.exceptions import PermissionDenied, ValidationError -from rest_framework.request import Request from rest_framework.response import Response from rest_framework.views import APIView from openedx_events.content_authoring.data import ContentObjectData, ContentObjectChangedData @@ -19,6 +18,8 @@ CONTENT_OBJECT_TAGS_CHANGED, ) +from openedx.core.types.http import RestRequest + from ...auth import has_view_object_tags_access from ...api import ( create_taxonomy, @@ -99,7 +100,7 @@ def perform_create(self, serializer): serializer.instance = create_taxonomy(**serializer.validated_data, orgs=user_admin_orgs) @action(detail=False, url_path="import", methods=["post"]) - def create_import(self, request: Request, **kwargs) -> Response: # type: ignore + def create_import(self, request: RestRequest, **kwargs) -> Response: # type: ignore """ Creates a new taxonomy with the given orgs and imports the tags from the uploaded file. """ @@ -183,7 +184,7 @@ class ObjectTagExportView(APIView): """" View to export a CSV with all children and tags for a given course/context. """ - def get(self, request: Request, **kwargs) -> StreamingHttpResponse: + def get(self, request: RestRequest, **kwargs) -> StreamingHttpResponse: """ Export a CSV with all children and tags for a given course/context. """ diff --git a/openedx/core/djangoapps/content_tagging/tests/test_tasks.py b/openedx/core/djangoapps/content_tagging/tests/test_tasks.py index c14adfcce13a..d0e10ecfb7ae 100644 --- a/openedx/core/djangoapps/content_tagging/tests/test_tasks.py +++ b/openedx/core/djangoapps/content_tagging/tests/test_tasks.py @@ -14,7 +14,9 @@ from common.djangoapps.student.tests.factories import UserFactory from openedx.core.djangolib.testing.utils import skip_unless_cms from xmodule.modulestore.tests.django_utils import TEST_DATA_SPLIT_MODULESTORE, ModuleStoreTestCase -from openedx.core.djangoapps.content_libraries.api import create_library, create_library_block, delete_library_block +from openedx.core.djangoapps.content_libraries.api import ( + create_library, create_library_block, delete_library_block, restore_library_block +) from .. import api from ..models.base import TaxonomyOrg @@ -267,7 +269,7 @@ def test_waffle_disabled_create_delete_xblock(self): # Still no tags assert self._check_tag(usage_key_str, LANGUAGE_TAXONOMY_ID, None) - def test_create_delete_library_block(self): + def test_create_delete_restore_library_block(self): # Create library library = create_library( org=self.orgA, @@ -287,11 +289,17 @@ def test_create_delete_library_block(self): # Check if the tags are created in the Library Block with the user's preferred language assert self._check_tag(usage_key_str, LANGUAGE_TAXONOMY_ID, 'Português (Brasil)') - # Delete the XBlock + # Soft delete the XBlock delete_library_block(library_block.usage_key) - # Check if the tags are deleted - assert self._check_tag(usage_key_str, LANGUAGE_TAXONOMY_ID, None) + # Check that the tags are not deleted + assert self._check_tag(usage_key_str, LANGUAGE_TAXONOMY_ID, 'Português (Brasil)') + + # Restore the XBlock + restore_library_block(library_block.usage_key) + + # Check if the tags are still present in the Library Block with the user's preferred language + assert self._check_tag(usage_key_str, LANGUAGE_TAXONOMY_ID, 'Português (Brasil)') @override_waffle_flag(CONTENT_TAGGING_AUTO, active=False) def test_waffle_disabled_create_delete_library_block(self): @@ -319,3 +327,10 @@ def test_waffle_disabled_create_delete_library_block(self): # Still no tags assert self._check_tag(usage_key_str, LANGUAGE_TAXONOMY_ID, None) + + # Restore the XBlock + with patch('crum.get_current_request', return_value=fake_request): + restore_library_block(library_block.usage_key) + + # Still no tags + assert self._check_tag(usage_key_str, LANGUAGE_TAXONOMY_ID, None) diff --git a/openedx/core/djangoapps/discussions/config/waffle.py b/openedx/core/djangoapps/discussions/config/waffle.py index 1d4c67e9e17b..eca6fc970856 100644 --- a/openedx/core/djangoapps/discussions/config/waffle.py +++ b/openedx/core/djangoapps/discussions/config/waffle.py @@ -2,6 +2,8 @@ This module contains various configuration settings via waffle switches for the discussions app. """ +from django.conf import settings + from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag WAFFLE_FLAG_NAMESPACE = "discussions" @@ -43,3 +45,31 @@ ENABLE_NEW_STRUCTURE_DISCUSSIONS = CourseWaffleFlag( f"{WAFFLE_FLAG_NAMESPACE}.enable_new_structure_discussions", __name__ ) + +# .. toggle_name: discussions.enable_forum_v2 +# .. toggle_implementation: CourseWaffleFlag +# .. toggle_default: False +# .. toggle_description: Waffle flag to use the forum v2 instead of v1(cs_comment_service) +# .. toggle_use_cases: temporary, open_edx +# .. toggle_creation_date: 2024-9-26 +# .. toggle_target_removal_date: 2025-12-05 +ENABLE_FORUM_V2 = CourseWaffleFlag(f"{WAFFLE_FLAG_NAMESPACE}.enable_forum_v2", __name__) + + +def is_forum_v2_enabled(course_key): + """ + Returns whether forum V2 is enabled on the course. This is a 2-step check: + + 1. Check value of settings.DISABLE_FORUM_V2: if it exists and is true, this setting overrides any course flag. + 2. Else, check the value of the corresponding course waffle flag. + """ + if is_forum_v2_disabled_globally(): + return False + return ENABLE_FORUM_V2.is_enabled(course_key) + + +def is_forum_v2_disabled_globally() -> bool: + """ + Return True if DISABLE_FORUM_V2 is defined and true-ish. + """ + return getattr(settings, "DISABLE_FORUM_V2", False) diff --git a/openedx/core/djangoapps/django_comment_common/comment_client/comment.py b/openedx/core/djangoapps/django_comment_common/comment_client/comment.py index c86f7eb40515..ba95c620496d 100644 --- a/openedx/core/djangoapps/django_comment_common/comment_client/comment.py +++ b/openedx/core/djangoapps/django_comment_common/comment_client/comment.py @@ -4,7 +4,9 @@ from openedx.core.djangoapps.django_comment_common.comment_client import models, settings from .thread import Thread, _url_for_flag_abuse_thread, _url_for_unflag_abuse_thread -from .utils import CommentClientRequestError, perform_request +from .utils import CommentClientRequestError, get_course_key, perform_request +from forum import api as forum_api +from openedx.core.djangoapps.discussions.config.waffle import is_forum_v2_enabled class Comment(models.Model): @@ -68,14 +70,21 @@ def flagAbuse(self, user, voteable): url = _url_for_flag_abuse_comment(voteable.id) else: raise CommentClientRequestError("Can only flag/unflag threads or comments") - params = {'user_id': user.id} - response = perform_request( - 'put', - url, - params, - metric_tags=self._metric_tags, - metric_action='comment.abuse.flagged' - ) + course_key = get_course_key(self.attributes.get("course_id")) + if is_forum_v2_enabled(course_key): + if voteable.type == 'thread': + response = forum_api.update_thread_flag(voteable.id, "flag", user.id, str(course_key)) + else: + response = forum_api.update_comment_flag(voteable.id, "flag", user.id, str(course_key)) + else: + params = {'user_id': user.id} + response = perform_request( + 'put', + url, + params, + metric_tags=self._metric_tags, + metric_action='comment.abuse.flagged' + ) voteable._update_from_response(response) def unFlagAbuse(self, user, voteable, removeAll): @@ -85,18 +94,37 @@ def unFlagAbuse(self, user, voteable, removeAll): url = _url_for_unflag_abuse_comment(voteable.id) else: raise CommentClientRequestError("Can flag/unflag for threads or comments") - params = {'user_id': user.id} - - if removeAll: - params['all'] = True - - response = perform_request( - 'put', - url, - params, - metric_tags=self._metric_tags, - metric_action='comment.abuse.unflagged' - ) + course_key = get_course_key(self.attributes.get("course_id")) + if is_forum_v2_enabled(course_key): + if voteable.type == "thread": + response = forum_api.update_thread_flag( + thread_id=voteable.id, + action="unflag", + user_id=user.id, + update_all=bool(removeAll), + course_id=str(course_key) + ) + else: + response = forum_api.update_comment_flag( + comment_id=voteable.id, + action="unflag", + user_id=user.id, + update_all=bool(removeAll), + course_id=str(course_key) + ) + else: + params = {'user_id': user.id} + + if removeAll: + params['all'] = True + + response = perform_request( + 'put', + url, + params, + metric_tags=self._metric_tags, + metric_action='comment.abuse.unflagged' + ) voteable._update_from_response(response) @property diff --git a/openedx/core/djangoapps/django_comment_common/comment_client/course.py b/openedx/core/djangoapps/django_comment_common/comment_client/course.py index 67d7efd22838..8cbb580e7831 100644 --- a/openedx/core/djangoapps/django_comment_common/comment_client/course.py +++ b/openedx/core/djangoapps/django_comment_common/comment_client/course.py @@ -7,8 +7,10 @@ from edx_django_utils.monitoring import function_trace from opaque_keys.edx.keys import CourseKey +from forum import api as forum_api from openedx.core.djangoapps.django_comment_common.comment_client import settings from openedx.core.djangoapps.django_comment_common.comment_client.utils import perform_request +from openedx.core.djangoapps.discussions.config.waffle import is_forum_v2_enabled def get_course_commentable_counts(course_key: CourseKey) -> Dict[str, Dict[str, int]]: @@ -29,17 +31,20 @@ def get_course_commentable_counts(course_key: CourseKey) -> Dict[str, Dict[str, } """ - url = f"{settings.PREFIX}/commentables/{course_key}/counts" - response = perform_request( - 'get', - url, - metric_tags=[ - f"course_key:{course_key}", - "function:get_course_commentable_counts", - ], - metric_action='commentable_stats.retrieve', - ) - return response + if is_forum_v2_enabled(course_key): + commentable_stats = forum_api.get_commentables_stats(str(course_key)) + else: + url = f"{settings.PREFIX}/commentables/{course_key}/counts" + commentable_stats = perform_request( + 'get', + url, + metric_tags=[ + f"course_key:{course_key}", + "function:get_course_commentable_counts", + ], + metric_action='commentable_stats.retrieve', + ) + return commentable_stats @function_trace("get_course_user_stats") @@ -76,17 +81,21 @@ def get_course_user_stats(course_key: CourseKey, params: Optional[Dict] = None) """ if params is None: params = {} - url = f"{settings.PREFIX}/users/{course_key}/stats" - return perform_request( - 'get', - url, - params, - metric_action='user.course_stats', - metric_tags=[ - f"course_key:{course_key}", - "function:get_course_user_stats", - ], - ) + if is_forum_v2_enabled(course_key): + course_stats = forum_api.get_user_course_stats(str(course_key), **params) + else: + url = f"{settings.PREFIX}/users/{course_key}/stats" + course_stats = perform_request( + 'get', + url, + params, + metric_action='user.course_stats', + metric_tags=[ + f"course_key:{course_key}", + "function:get_course_user_stats", + ], + ) + return course_stats @function_trace("update_course_users_stats") @@ -100,13 +109,17 @@ def update_course_users_stats(course_key: CourseKey) -> Dict: Returns: dict: data returned by API. Contains count of users updated. """ - url = f"{settings.PREFIX}/users/{course_key}/update_stats" - return perform_request( - 'post', - url, - metric_action='user.update_course_stats', - metric_tags=[ - f"course_key:{course_key}", - "function:update_course_users_stats", - ], - ) + if is_forum_v2_enabled(course_key): + course_stats = forum_api.update_users_in_course(str(course_key)) + else: + url = f"{settings.PREFIX}/users/{course_key}/update_stats" + course_stats = perform_request( + 'post', + url, + metric_action='user.update_course_stats', + metric_tags=[ + f"course_key:{course_key}", + "function:update_course_users_stats", + ], + ) + return course_stats diff --git a/openedx/core/djangoapps/django_comment_common/comment_client/models.py b/openedx/core/djangoapps/django_comment_common/comment_client/models.py index 4e602809c82a..9b6c9ca03f3d 100644 --- a/openedx/core/djangoapps/django_comment_common/comment_client/models.py +++ b/openedx/core/djangoapps/django_comment_common/comment_client/models.py @@ -2,8 +2,11 @@ import logging +import typing as t -from .utils import CommentClientRequestError, extract, perform_request +from .utils import CommentClientRequestError, extract, perform_request, get_course_key +from forum import api as forum_api +from openedx.core.djangoapps.discussions.config.waffle import is_forum_v2_enabled, is_forum_v2_disabled_globally log = logging.getLogger(__name__) @@ -69,14 +72,27 @@ def retrieve(self, *args, **kwargs): return self def _retrieve(self, *args, **kwargs): - url = self.url(action='get', params=self.attributes) - response = perform_request( - 'get', - url, - self.default_retrieve_params, - metric_tags=self._metric_tags, - metric_action='model.retrieve' - ) + course_id = self.attributes.get("course_id") or kwargs.get("course_id") + if course_id: + course_key = get_course_key(course_id) + use_forumv2 = is_forum_v2_enabled(course_key) + else: + use_forumv2, course_id = is_forum_v2_enabled_for_comment(self.id) + response = None + if use_forumv2: + if self.type == "comment": + response = forum_api.get_parent_comment(comment_id=self.attributes["id"], course_id=course_id) + if response is None: + raise CommentClientRequestError("Forum v2 API call is missing") + else: + url = self.url(action='get', params=self.attributes) + response = perform_request( + 'get', + url, + self.default_retrieve_params, + metric_tags=self._metric_tags, + metric_action='model.retrieve' + ) self._update_from_response(response) @property @@ -151,33 +167,27 @@ def save(self, params=None): """ self.before_save(self) if self.id: # if we have id already, treat this as an update - request_params = self.updatable_attributes() - if params: - request_params.update(params) - url = self.url(action='put', params=self.attributes) - response = perform_request( - 'put', - url, - request_params, - metric_tags=self._metric_tags, - metric_action='model.update' - ) - else: # otherwise, treat this as an insert - url = self.url(action='post', params=self.attributes) - response = perform_request( - 'post', - url, - self.initializable_attributes(), - metric_tags=self._metric_tags, - metric_action='model.insert' - ) + response = self.handle_update(params) + else: # otherwise, treat this as an insert + response = self.handle_create(params) + self.retrieved = True self._update_from_response(response) self.after_save(self) def delete(self): - url = self.url(action='delete', params=self.attributes) - response = perform_request('delete', url, metric_tags=self._metric_tags, metric_action='model.delete') + course_key = get_course_key(self.attributes.get("course_id")) + if is_forum_v2_enabled(course_key): + response = None + if self.type == "comment": + response = forum_api.delete_comment(comment_id=self.attributes["id"], course_id=str(course_key)) + elif self.type == "thread": + response = forum_api.delete_thread(thread_id=self.attributes["id"], course_id=str(course_key)) + if response is None: + raise CommentClientRequestError("Forum v2 API call is missing") + else: + url = self.url(action='delete', params=self.attributes) + response = perform_request('delete', url, metric_tags=self._metric_tags, metric_action='model.delete') self.retrieved = True self._update_from_response(response) @@ -208,3 +218,176 @@ def url(cls, action, params=None): raise CommentClientRequestError(f"Cannot perform action {action} without id") # lint-amnesty, pylint: disable=raise-missing-from else: # action must be in DEFAULT_ACTIONS_WITHOUT_ID now return cls.url_without_id() + + def handle_update(self, params=None): + request_params = self.updatable_attributes() + if params: + request_params.update(params) + course_id = self.attributes.get("course_id") or request_params.get("course_id") + course_key = get_course_key(course_id) + if is_forum_v2_enabled(course_key): + response = None + if self.type == "comment": + response = self.handle_update_comment(request_params, str(course_key)) + elif self.type == "thread": + response = self.handle_update_thread(request_params, str(course_key)) + elif self.type == "user": + response = self.handle_update_user(request_params, str(course_key)) + if response is None: + raise CommentClientRequestError("Forum v2 API call is missing") + else: + response = self.perform_http_put_request(request_params) + return response + + def handle_update_user(self, request_params, course_id): + try: + username = request_params["username"] + external_id = str(request_params["external_id"]) + except KeyError as e: + raise e + response = forum_api.update_user( + external_id, + username=username, + course_id=course_id, + ) + return response + + def handle_update_comment(self, request_params, course_id): + request_data = { + "comment_id": self.attributes["id"], + "body": request_params.get("body"), + "course_id": request_params.get("course_id"), + "user_id": request_params.get("user_id"), + "anonymous": request_params.get("anonymous"), + "anonymous_to_peers": request_params.get("anonymous_to_peers"), + "endorsed": request_params.get("endorsed"), + "closed": request_params.get("closed"), + "editing_user_id": request_params.get("editing_user_id"), + "edit_reason_code": request_params.get("edit_reason_code"), + "endorsement_user_id": request_params.get("endorsement_user_id"), + "course_key": course_id + } + request_data = {k: v for k, v in request_data.items() if v is not None} + response = forum_api.update_comment(**request_data) + return response + + def handle_update_thread(self, request_params, course_id): + request_data = { + "thread_id": self.attributes["id"], + "title": request_params.get("title"), + "body": request_params.get("body"), + "course_id": request_params.get("course_id"), + "anonymous": request_params.get("anonymous"), + "anonymous_to_peers": request_params.get("anonymous_to_peers"), + "closed": request_params.get("closed"), + "commentable_id": request_params.get("commentable_id"), + "user_id": request_params.get("user_id"), + "editing_user_id": request_params.get("editing_user_id"), + "pinned": request_params.get("pinned"), + "thread_type": request_params.get("thread_type"), + "edit_reason_code": request_params.get("edit_reason_code"), + "close_reason_code": request_params.get("close_reason_code"), + "closing_user_id": request_params.get("closing_user_id"), + "endorsed": request_params.get("endorsed"), + "course_key": course_id + } + request_data = {k: v for k, v in request_data.items() if v is not None} + response = forum_api.update_thread(**request_data) + return response + + def perform_http_put_request(self, request_params): + url = self.url(action="put", params=self.attributes) + response = perform_request( + "put", + url, + request_params, + metric_tags=self._metric_tags, + metric_action="model.update", + ) + return response + + def perform_http_post_request(self): + url = self.url(action="post", params=self.attributes) + response = perform_request( + "post", + url, + self.initializable_attributes(), + metric_tags=self._metric_tags, + metric_action="model.insert", + ) + return response + + def handle_create(self, params=None): + course_id = self.attributes.get("course_id") or params.get("course_id") + course_key = get_course_key(course_id) + if is_forum_v2_enabled(course_key): + response = None + if self.type == "comment": + response = self.handle_create_comment(str(course_key)) + elif self.type == "thread": + response = self.handle_create_thread(str(course_key)) + if response is None: + raise CommentClientRequestError("Forum v2 API call is missing") + else: + response = self.perform_http_post_request() + return response + + def handle_create_comment(self, course_id): + request_data = self.initializable_attributes() + body = request_data["body"] + user_id = request_data["user_id"] + course_id = course_id or str(request_data["course_id"]) + if parent_id := self.attributes.get("parent_id"): + response = forum_api.create_child_comment( + parent_id, + body, + user_id, + course_id, + request_data.get("anonymous", False), + request_data.get("anonymous_to_peers", False), + ) + else: + response = forum_api.create_parent_comment( + self.attributes["thread_id"], + body, + user_id, + course_id, + request_data.get("anonymous", False), + request_data.get("anonymous_to_peers", False), + ) + return response + + def handle_create_thread(self, course_id): + request_data = self.initializable_attributes() + response = forum_api.create_thread( + title=request_data["title"], + body=request_data["body"], + course_id=course_id or str(request_data["course_id"]), + user_id=str(request_data["user_id"]), + anonymous=request_data.get("anonymous", False), + anonymous_to_peers=request_data.get("anonymous_to_peers", False), + commentable_id=request_data.get("commentable_id", "course"), + thread_type=request_data.get("thread_type", "discussion"), + group_id=request_data.get("group_id", None), + context=request_data.get("context", None), + ) + return response + + +def is_forum_v2_enabled_for_comment(comment_id: str) -> tuple[bool, t.Optional[str]]: + """ + Figure out whether we use forum v2 for a given comment. + + See is_forum_v2_enabled_for_thread. + + Return: + + enabled (bool) + course_id (str or None) + """ + if is_forum_v2_disabled_globally(): + return False, None + + course_id = forum_api.get_course_id_by_comment(comment_id) + course_key = get_course_key(course_id) + return is_forum_v2_enabled(course_key), course_id diff --git a/openedx/core/djangoapps/django_comment_common/comment_client/subscriptions.py b/openedx/core/djangoapps/django_comment_common/comment_client/subscriptions.py index 545948a092cc..2130dfc56be6 100644 --- a/openedx/core/djangoapps/django_comment_common/comment_client/subscriptions.py +++ b/openedx/core/djangoapps/django_comment_common/comment_client/subscriptions.py @@ -4,6 +4,8 @@ import logging from . import models, settings, utils +from forum import api as forum_api +from openedx.core.djangoapps.discussions.config.waffle import is_forum_v2_enabled log = logging.getLogger(__name__) @@ -21,7 +23,7 @@ class Subscription(models.Model): base_url = f"{settings.PREFIX}/threads" @classmethod - def fetch(cls, thread_id, query_params): + def fetch(cls, thread_id, course_id, query_params): """ Fetches the subscriptions for a given thread_id """ @@ -33,14 +35,23 @@ def fetch(cls, thread_id, query_params): params.update( utils.strip_blank(utils.strip_none(query_params)) ) - response = utils.perform_request( - 'get', - cls.url(action='get', params=params) + "/subscriptions", - params, - metric_tags=[], - metric_action='subscription.get', - paged_results=True - ) + course_key = utils.get_course_key(course_id) + if is_forum_v2_enabled(course_key): + response = forum_api.get_thread_subscriptions( + thread_id=thread_id, + page=params["page"], + per_page=params["per_page"], + course_id=str(course_key) + ) + else: + response = utils.perform_request( + 'get', + cls.url(action='get', params=params) + "/subscriptions", + params, + metric_tags=[], + metric_action='subscription.get', + paged_results=True + ) return utils.SubscriptionsPaginatedResult( collection=response.get('collection', []), page=response.get('page', 1), diff --git a/openedx/core/djangoapps/django_comment_common/comment_client/thread.py b/openedx/core/djangoapps/django_comment_common/comment_client/thread.py index ef5accbad25d..ecf420cfae56 100644 --- a/openedx/core/djangoapps/django_comment_common/comment_client/thread.py +++ b/openedx/core/djangoapps/django_comment_common/comment_client/thread.py @@ -2,10 +2,13 @@ import logging +import typing as t from eventtracking import tracker from . import models, settings, utils +from forum import api as forum_api +from openedx.core.djangoapps.discussions.config.waffle import is_forum_v2_enabled, is_forum_v2_disabled_globally log = logging.getLogger(__name__) @@ -59,14 +62,35 @@ def search(cls, query_params): url = cls.url(action='get_all', params=utils.extract(params, 'commentable_id')) if params.get('commentable_id'): del params['commentable_id'] - response = utils.perform_request( - 'get', - url, - params, - metric_tags=['course_id:{}'.format(query_params['course_id'])], - metric_action='thread.search', - paged_results=True - ) + + if is_forum_v2_enabled(utils.get_course_key(query_params['course_id'])): + if query_params.get('text'): + search_params = utils.strip_none(params) + if user_id := search_params.get('user_id'): + search_params['user_id'] = str(user_id) + if group_ids := search_params.get('group_ids'): + search_params['group_ids'] = [int(group_id) for group_id in group_ids.split(',')] + elif group_id := search_params.get('group_id'): + search_params['group_ids'] = [int(group_id)] + search_params.pop('group_id', None) + if commentable_ids := search_params.get('commentable_ids'): + search_params['commentable_ids'] = commentable_ids.split(',') + elif commentable_id := search_params.get('commentable_id'): + search_params['commentable_ids'] = [commentable_id] + search_params.pop('commentable_id', None) + response = forum_api.search_threads(**search_params) + else: + response = forum_api.get_user_threads(**params) + else: + response = utils.perform_request( + 'get', + url, + params, + metric_tags=['course_id:{}'.format(query_params['course_id'])], + metric_action='thread.search', + paged_results=True + ) + if query_params.get('text'): search_query = query_params['text'] course_id = query_params['course_id'] @@ -148,14 +172,28 @@ def _retrieve(self, *args, **kwargs): 'merge_question_type_responses': kwargs.get('merge_question_type_responses', False) } request_params = utils.strip_none(request_params) - - response = utils.perform_request( - 'get', - url, - request_params, - metric_action='model.retrieve', - metric_tags=self._metric_tags - ) + course_id = kwargs.get("course_id") + if course_id: + course_key = utils.get_course_key(course_id) + use_forumv2 = is_forum_v2_enabled(course_key) + else: + use_forumv2, course_id = is_forum_v2_enabled_for_thread(self.id) + if use_forumv2: + if user_id := request_params.get('user_id'): + request_params['user_id'] = str(user_id) + response = forum_api.get_thread( + thread_id=self.id, + params=request_params, + course_id=course_id, + ) + else: + response = utils.perform_request( + 'get', + url, + request_params, + metric_action='model.retrieve', + metric_tags=self._metric_tags + ) self._update_from_response(response) def flagAbuse(self, user, voteable): @@ -163,14 +201,18 @@ def flagAbuse(self, user, voteable): url = _url_for_flag_abuse_thread(voteable.id) else: raise utils.CommentClientRequestError("Can only flag/unflag threads or comments") - params = {'user_id': user.id} - response = utils.perform_request( - 'put', - url, - params, - metric_action='thread.abuse.flagged', - metric_tags=self._metric_tags - ) + course_key = utils.get_course_key(self.attributes.get("course_id")) + if is_forum_v2_enabled(course_key): + response = forum_api.update_thread_flag(voteable.id, "flag", user.id, str(course_key)) + else: + params = {'user_id': user.id} + response = utils.perform_request( + 'put', + url, + params, + metric_action='thread.abuse.flagged', + metric_tags=self._metric_tags + ) voteable._update_from_response(response) def unFlagAbuse(self, user, voteable, removeAll): @@ -178,42 +220,68 @@ def unFlagAbuse(self, user, voteable, removeAll): url = _url_for_unflag_abuse_thread(voteable.id) else: raise utils.CommentClientRequestError("Can only flag/unflag for threads or comments") - params = {'user_id': user.id} - #if you're an admin, when you unflag, remove ALL flags - if removeAll: - params['all'] = True - - response = utils.perform_request( - 'put', - url, - params, - metric_tags=self._metric_tags, - metric_action='thread.abuse.unflagged' - ) + course_key = utils.get_course_key(self.attributes.get("course_id")) + if is_forum_v2_enabled(course_key): + response = forum_api.update_thread_flag( + thread_id=voteable.id, + action="unflag", + user_id=user.id, + update_all=bool(removeAll), + course_id=str(course_key) + ) + else: + params = {'user_id': user.id} + #if you're an admin, when you unflag, remove ALL flags + if removeAll: + params['all'] = True + + response = utils.perform_request( + 'put', + url, + params, + metric_tags=self._metric_tags, + metric_action='thread.abuse.unflagged' + ) voteable._update_from_response(response) def pin(self, user, thread_id): - url = _url_for_pin_thread(thread_id) - params = {'user_id': user.id} - response = utils.perform_request( - 'put', - url, - params, - metric_tags=self._metric_tags, - metric_action='thread.pin' - ) + course_key = utils.get_course_key(self.attributes.get("course_id")) + if is_forum_v2_enabled(course_key): + response = forum_api.pin_thread( + user_id=user.id, + thread_id=thread_id, + course_id=str(course_key) + ) + else: + url = _url_for_pin_thread(thread_id) + params = {'user_id': user.id} + response = utils.perform_request( + 'put', + url, + params, + metric_tags=self._metric_tags, + metric_action='thread.pin' + ) self._update_from_response(response) def un_pin(self, user, thread_id): - url = _url_for_un_pin_thread(thread_id) - params = {'user_id': user.id} - response = utils.perform_request( - 'put', - url, - params, - metric_tags=self._metric_tags, - metric_action='thread.unpin' - ) + course_key = utils.get_course_key(self.attributes.get("course_id")) + if is_forum_v2_enabled(course_key): + response = forum_api.unpin_thread( + user_id=user.id, + thread_id=thread_id, + course_id=str(course_key) + ) + else: + url = _url_for_un_pin_thread(thread_id) + params = {'user_id': user.id} + response = utils.perform_request( + 'put', + url, + params, + metric_tags=self._metric_tags, + metric_action='thread.unpin' + ) self._update_from_response(response) @@ -231,3 +299,28 @@ def _url_for_pin_thread(thread_id): def _url_for_un_pin_thread(thread_id): return f"{settings.PREFIX}/threads/{thread_id}/unpin" + + +def is_forum_v2_enabled_for_thread(thread_id: str) -> tuple[bool, t.Optional[str]]: + """ + Figure out whether we use forum v2 for a given thread. + + This is a complex affair... First, we check the value of the DISABLE_FORUM_V2 + setting, which overrides everything. If this setting does not exist, then we need to + find the course ID that corresponds to the thread ID. Then, we return the value of + the course waffle flag for this course ID. + + Note that to fetch the course ID associated to a thread ID, we need to connect both + to mongodb and mysql. As a consequence, when forum v2 needs adequate connection + strings for both backends. + + Return: + + enabled (bool) + course_id (str or None) + """ + if is_forum_v2_disabled_globally(): + return False, None + course_id = forum_api.get_course_id_by_thread(thread_id) + course_key = utils.get_course_key(course_id) + return is_forum_v2_enabled(course_key), course_id diff --git a/openedx/core/djangoapps/django_comment_common/comment_client/user.py b/openedx/core/djangoapps/django_comment_common/comment_client/user.py index 684469c9e787..2de4fbbfa95a 100644 --- a/openedx/core/djangoapps/django_comment_common/comment_client/user.py +++ b/openedx/core/djangoapps/django_comment_common/comment_client/user.py @@ -1,8 +1,10 @@ # pylint: disable=missing-docstring,protected-access """ User model wrapper for comment service""" - from . import models, settings, utils +from forum import api as forum_api +from forum.utils import ForumV2RequestError, str_to_bool +from openedx.core.djangoapps.discussions.config.waffle import is_forum_v2_enabled class User(models.Model): @@ -34,34 +36,55 @@ def read(self, source): """ Calls cs_comments_service to mark thread as read for the user """ - params = {'source_type': source.type, 'source_id': source.id} - utils.perform_request( - 'post', - _url_for_read(self.id), - params, - metric_action='user.read', - metric_tags=self._metric_tags + [f'target.type:{source.type}'], - ) + course_id = self.attributes.get("course_id") + course_key = utils.get_course_key(course_id) + if is_forum_v2_enabled(course_key): + forum_api.mark_thread_as_read(self.id, source.id, course_id=str(course_id)) + else: + params = {'source_type': source.type, 'source_id': source.id} + utils.perform_request( + 'post', + _url_for_read(self.id), + params, + metric_action='user.read', + metric_tags=self._metric_tags + [f'target.type:{source.type}'], + ) def follow(self, source): - params = {'source_type': source.type, 'source_id': source.id} - utils.perform_request( - 'post', - _url_for_subscription(self.id), - params, - metric_action='user.follow', - metric_tags=self._metric_tags + [f'target.type:{source.type}'], - ) + course_key = utils.get_course_key(self.attributes.get("course_id")) + if is_forum_v2_enabled(course_key): + forum_api.create_subscription( + user_id=self.id, + source_id=source.id, + course_id=str(course_key) + ) + else: + params = {'source_type': source.type, 'source_id': source.id} + utils.perform_request( + 'post', + _url_for_subscription(self.id), + params, + metric_action='user.follow', + metric_tags=self._metric_tags + [f'target.type:{source.type}'], + ) def unfollow(self, source): - params = {'source_type': source.type, 'source_id': source.id} - utils.perform_request( - 'delete', - _url_for_subscription(self.id), - params, - metric_action='user.unfollow', - metric_tags=self._metric_tags + [f'target.type:{source.type}'], - ) + course_key = utils.get_course_key(self.attributes.get("course_id")) + if is_forum_v2_enabled(course_key): + forum_api.delete_subscription( + user_id=self.id, + source_id=source.id, + course_id=str(course_key) + ) + else: + params = {'source_type': source.type, 'source_id': source.id} + utils.perform_request( + 'delete', + _url_for_subscription(self.id), + params, + metric_action='user.unfollow', + metric_tags=self._metric_tags + [f'target.type:{source.type}'], + ) def vote(self, voteable, value): if voteable.type == 'thread': @@ -70,14 +93,31 @@ def vote(self, voteable, value): url = _url_for_vote_comment(voteable.id) else: raise utils.CommentClientRequestError("Can only vote / unvote for threads or comments") - params = {'user_id': self.id, 'value': value} - response = utils.perform_request( - 'put', - url, - params, - metric_action='user.vote', - metric_tags=self._metric_tags + [f'target.type:{voteable.type}'], - ) + course_key = utils.get_course_key(self.attributes.get("course_id")) + if is_forum_v2_enabled(course_key): + if voteable.type == 'thread': + response = forum_api.update_thread_votes( + thread_id=voteable.id, + user_id=self.id, + value=value, + course_id=str(course_key) + ) + else: + response = forum_api.update_comment_votes( + comment_id=voteable.id, + user_id=self.id, + value=value, + course_id=str(course_key) + ) + else: + params = {'user_id': self.id, 'value': value} + response = utils.perform_request( + 'put', + url, + params, + metric_action='user.vote', + metric_tags=self._metric_tags + [f'target.type:{voteable.type}'], + ) voteable._update_from_response(response) def unvote(self, voteable): @@ -87,14 +127,29 @@ def unvote(self, voteable): url = _url_for_vote_comment(voteable.id) else: raise utils.CommentClientRequestError("Can only vote / unvote for threads or comments") - params = {'user_id': self.id} - response = utils.perform_request( - 'delete', - url, - params, - metric_action='user.unvote', - metric_tags=self._metric_tags + [f'target.type:{voteable.type}'], - ) + course_key = utils.get_course_key(self.attributes.get("course_id")) + if is_forum_v2_enabled(course_key): + if voteable.type == 'thread': + response = forum_api.delete_thread_vote( + thread_id=voteable.id, + user_id=self.id, + course_id=str(course_key) + ) + else: + response = forum_api.delete_comment_vote( + comment_id=voteable.id, + user_id=self.id, + course_id=str(course_key) + ) + else: + params = {'user_id': self.id} + response = utils.perform_request( + 'delete', + url, + params, + metric_action='user.unvote', + metric_tags=self._metric_tags + [f'target.type:{voteable.type}'], + ) voteable._update_from_response(response) def active_threads(self, query_params=None): @@ -105,14 +160,28 @@ def active_threads(self, query_params=None): url = _url_for_user_active_threads(self.id) params = {'course_id': str(self.course_id)} params.update(query_params) - response = utils.perform_request( - 'get', - url, - params, - metric_action='user.active_threads', - metric_tags=self._metric_tags, - paged_results=True, - ) + course_key = utils.get_course_key(self.attributes.get("course_id")) + if is_forum_v2_enabled(course_key): + if user_id := params.get("user_id"): + params["user_id"] = str(user_id) + if page := params.get("page"): + params["page"] = int(page) + if per_page := params.get("per_page"): + params["per_page"] = int(per_page) + if count_flagged := params.get("count_flagged", False): + params["count_flagged"] = str_to_bool(count_flagged) + if not params.get("course_id"): + params["course_id"] = str(course_key) + response = forum_api.get_user_active_threads(**params) + else: + response = utils.perform_request( + 'get', + url, + params, + metric_action='user.active_threads', + metric_tags=self._metric_tags, + paged_results=True, + ) return response.get('collection', []), response.get('page', 1), response.get('num_pages', 1) def subscribed_threads(self, query_params=None): @@ -125,14 +194,28 @@ def subscribed_threads(self, query_params=None): url = _url_for_user_subscribed_threads(self.id) params = {'course_id': str(self.course_id)} params.update(query_params) - response = utils.perform_request( - 'get', - url, - params, - metric_action='user.subscribed_threads', - metric_tags=self._metric_tags, - paged_results=True - ) + course_key = utils.get_course_key(self.attributes.get("course_id")) + if is_forum_v2_enabled(course_key): + if user_id := params.get("user_id"): + params["user_id"] = str(user_id) + if page := params.get("page"): + params["page"] = int(page) + if per_page := params.get("per_page"): + params["per_page"] = int(per_page) + if count_flagged := params.get("count_flagged", False): + params["count_flagged"] = str_to_bool(count_flagged) + if not params.get("course_id"): + params["course_id"] = str(course_key) + response = forum_api.get_user_threads(**params) + else: + response = utils.perform_request( + 'get', + url, + params, + metric_action='user.subscribed_threads', + metric_tags=self._metric_tags, + paged_results=True + ) return utils.CommentClientPaginatedResult( collection=response.get('collection', []), page=response.get('page', 1), @@ -144,23 +227,39 @@ def _retrieve(self, *args, **kwargs): url = self.url(action='get', params=self.attributes) retrieve_params = self.default_retrieve_params.copy() retrieve_params.update(kwargs) + if self.attributes.get('course_id'): retrieve_params['course_id'] = str(self.course_id) if self.attributes.get('group_id'): retrieve_params['group_id'] = self.group_id - try: - response = utils.perform_request( - 'get', - url, - retrieve_params, - metric_action='model.retrieve', - metric_tags=self._metric_tags, - ) - except utils.CommentClientRequestError as e: - if e.status_code == 404: - # attempt to gracefully recover from a previous failure - # to sync this user to the comments service. - self.save() + + # course key -> id conversation + course_id = retrieve_params.get('course_id') + if course_id: + course_id = str(course_id) + retrieve_params['course_id'] = course_id + course_key = utils.get_course_key(course_id) + + if is_forum_v2_enabled(course_key): + group_ids = [retrieve_params['group_id']] if 'group_id' in retrieve_params else [] + is_complete = retrieve_params['complete'] + try: + response = forum_api.get_user( + self.attributes["id"], + group_ids=group_ids, + course_id=course_id, + complete=is_complete + ) + except ForumV2RequestError as e: + self.save({"course_id": course_id}) + response = forum_api.get_user( + self.attributes["id"], + group_ids=group_ids, + course_id=course_id, + complete=is_complete + ) + else: + try: response = utils.perform_request( 'get', url, @@ -168,33 +267,52 @@ def _retrieve(self, *args, **kwargs): metric_action='model.retrieve', metric_tags=self._metric_tags, ) - else: - raise + except utils.CommentClientRequestError as e: + if e.status_code == 404: + # attempt to gracefully recover from a previous failure + # to sync this user to the comments service. + self.save() + response = utils.perform_request( + 'get', + url, + retrieve_params, + metric_action='model.retrieve', + metric_tags=self._metric_tags, + ) + else: + raise self._update_from_response(response) def retire(self, retired_username): - url = _url_for_retire(self.id) - params = {'retired_username': retired_username} - - utils.perform_request( - 'post', - url, - params, - raw=True, - metric_action='user.retire', - metric_tags=self._metric_tags - ) + course_key = utils.get_course_key(self.attributes.get("course_id")) + if is_forum_v2_enabled(course_key): + forum_api.retire_user(user_id=self.id, retired_username=retired_username, course_id=str(course_key)) + else: + url = _url_for_retire(self.id) + params = {'retired_username': retired_username} + utils.perform_request( + 'post', + url, + params, + raw=True, + metric_action='user.retire', + metric_tags=self._metric_tags + ) def replace_username(self, new_username): - url = _url_for_username_replacement(self.id) - params = {"new_username": new_username} - - utils.perform_request( - 'post', - url, - params, - raw=True, - ) + course_key = utils.get_course_key(self.attributes.get("course_id")) + if is_forum_v2_enabled(course_key): + forum_api.update_username(user_id=self.id, new_username=new_username, course_id=str(course_key)) + else: + url = _url_for_username_replacement(self.id) + params = {"new_username": new_username} + + utils.perform_request( + 'post', + url, + params, + raw=True, + ) def _url_for_vote_comment(comment_id): diff --git a/openedx/core/djangoapps/django_comment_common/comment_client/utils.py b/openedx/core/djangoapps/django_comment_common/comment_client/utils.py index a67cdbdbc483..e77f39e6277d 100644 --- a/openedx/core/djangoapps/django_comment_common/comment_client/utils.py +++ b/openedx/core/djangoapps/django_comment_common/comment_client/utils.py @@ -7,6 +7,7 @@ import requests from django.utils.translation import get_language +from opaque_keys.edx.keys import CourseKey from .settings import SERVICE_HOST as COMMENTS_SERVICE @@ -167,3 +168,19 @@ def check_forum_heartbeat(): return 'forum', False, res.get('check', 'Forum heartbeat failed') except Exception as fail: return 'forum', False, str(fail) + + +def get_course_key(course_id: CourseKey | str | None) -> CourseKey | None: + """ + Returns a CourseKey if the provided course_id is a valid string representation of a CourseKey. + If course_id is None or already a CourseKey object, it returns the course_id as is. + Args: + course_id (CourseKey | str | None): The course ID to be converted. + Returns: + CourseKey | None: The corresponding CourseKey object or None if the input is None. + Raises: + KeyError: If course_id is not a valid string representation of a CourseKey. + """ + if course_id and isinstance(course_id, str): + course_id = CourseKey.from_string(course_id) + return course_id diff --git a/openedx/core/djangoapps/notifications/email/utils.py b/openedx/core/djangoapps/notifications/email/utils.py index 9fd761785e5a..34c245308785 100644 --- a/openedx/core/djangoapps/notifications/email/utils.py +++ b/openedx/core/djangoapps/notifications/email/utils.py @@ -9,7 +9,7 @@ from django.contrib.auth import get_user_model from django.shortcuts import get_object_or_404 from pytz import utc -from waffle import get_waffle_flag_model # pylint: disable=invalid-django-waffle-import +from waffle import get_waffle_flag_model # pylint: disable=invalid-django-waffle-import from common.djangoapps.student.models import CourseEnrollment from lms.djangoapps.branding.api import get_logo_url_for_email @@ -29,7 +29,6 @@ from .notification_icons import NotificationTypeIcons - User = get_user_model() @@ -370,14 +369,6 @@ def is_name_match(name, param_name): """ return True if param_name is None else name == param_name - def is_editable(app_name, notification_type, channel): - """ - Returns if notification type channel is editable - """ - if notification_type == 'core': - return channel not in COURSE_NOTIFICATION_APPS[app_name]['non_editable'] - return channel not in COURSE_NOTIFICATION_TYPES[notification_type]['non_editable'] - def get_default_cadence_value(app_name, notification_type): """ Returns default email cadence value @@ -417,9 +408,18 @@ def get_updated_preference(pref): for channel in ['web', 'email', 'push']: if not is_name_match(channel, channel_value): continue - if is_editable(app_name, noti_type, channel): + if is_notification_type_channel_editable(app_name, noti_type, channel): type_prefs[channel] = pref_value if channel == 'email' and pref_value and type_prefs.get('email_cadence') == EmailCadence.NEVER: type_prefs['email_cadence'] = get_default_cadence_value(app_name, noti_type) preference.save() notification_preference_unsubscribe_event(user) + + +def is_notification_type_channel_editable(app_name, notification_type, channel): + """ + Returns if notification type channel is editable + """ + if notification_type == 'core': + return channel not in COURSE_NOTIFICATION_APPS[app_name]['non_editable'] + return channel not in COURSE_NOTIFICATION_TYPES[notification_type]['non_editable'] diff --git a/openedx/core/djangoapps/notifications/grouping_notifications.py b/openedx/core/djangoapps/notifications/grouping_notifications.py index 3c4688b5ed53..0e84ea3f109a 100644 --- a/openedx/core/djangoapps/notifications/grouping_notifications.py +++ b/openedx/core/djangoapps/notifications/grouping_notifications.py @@ -132,7 +132,8 @@ def get_user_existing_notifications(user_ids, notification_type, group_by_id, co user__in=user_ids, notification_type=notification_type, group_by_id=group_by_id, - course_id=course_id + course_id=course_id, + last_seen__isnull=True, ) notifications_mapping = {user_id: [] for user_id in user_ids} for notification in notifications: diff --git a/openedx/core/djangoapps/notifications/serializers.py b/openedx/core/djangoapps/notifications/serializers.py index 79c8c4af9d13..80b1577b6355 100644 --- a/openedx/core/djangoapps/notifications/serializers.py +++ b/openedx/core/djangoapps/notifications/serializers.py @@ -1,6 +1,7 @@ """ Serializers for the notifications API. """ + from django.core.exceptions import ValidationError from rest_framework import serializers @@ -9,9 +10,12 @@ from openedx.core.djangoapps.notifications.models import ( CourseNotificationPreference, Notification, - get_notification_channels, get_additional_notification_channel_settings + get_additional_notification_channel_settings, + get_notification_channels ) + from .base_notification import COURSE_NOTIFICATION_APPS, COURSE_NOTIFICATION_TYPES, EmailCadence +from .email.utils import is_notification_type_channel_editable from .utils import remove_preferences_with_no_access @@ -202,3 +206,113 @@ class Meta: 'last_seen', 'created', ) + + +def validate_email_cadence(email_cadence: str) -> str: + """ + Validate email cadence value. + """ + if EmailCadence.get_email_cadence_value(email_cadence) is None: + raise ValidationError(f'{email_cadence} is not a valid email cadence.') + return email_cadence + + +def validate_notification_app(notification_app: str) -> str: + """ + Validate notification app value. + """ + if not COURSE_NOTIFICATION_APPS.get(notification_app): + raise ValidationError(f'{notification_app} is not a valid notification app.') + return notification_app + + +def validate_notification_app_enabled(notification_app: str) -> str: + """ + Validate notification app is enabled. + """ + + if COURSE_NOTIFICATION_APPS.get(notification_app) and COURSE_NOTIFICATION_APPS.get(notification_app)['enabled']: + return notification_app + raise ValidationError(f'{notification_app} is not a valid notification app.') + + +def validate_notification_type(notification_type: str) -> str: + """ + Validate notification type value. + """ + if not COURSE_NOTIFICATION_TYPES.get(notification_type): + raise ValidationError(f'{notification_type} is not a valid notification type.') + return notification_type + + +def validate_notification_channel(notification_channel: str) -> str: + """ + Validate notification channel value. + """ + valid_channels = set(get_notification_channels()) | set(get_additional_notification_channel_settings()) + if notification_channel not in valid_channels: + raise ValidationError(f'{notification_channel} is not a valid notification channel setting.') + return notification_channel + + +class UserNotificationPreferenceUpdateAllSerializer(serializers.Serializer): + """ + Serializer for user notification preferences update with custom field validators. + """ + notification_app = serializers.CharField( + required=True, + validators=[validate_notification_app, validate_notification_app_enabled] + ) + value = serializers.BooleanField(required=False) + notification_type = serializers.CharField( + required=True, + ) + notification_channel = serializers.CharField( + required=False, + validators=[validate_notification_channel] + ) + email_cadence = serializers.CharField( + required=False, + validators=[validate_email_cadence] + ) + + def validate(self, attrs): + """ + Cross-field validation for notification preference update. + """ + notification_app = attrs.get('notification_app') + notification_type = attrs.get('notification_type') + notification_channel = attrs.get('notification_channel') + email_cadence = attrs.get('email_cadence') + + # Validate email_cadence requirements + if email_cadence and not notification_type: + raise ValidationError({ + 'notification_type': 'notification_type is required for email_cadence.' + }) + + # Validate notification_channel requirements + if not email_cadence and notification_type and not notification_channel: + raise ValidationError({ + 'notification_channel': 'notification_channel is required for notification_type.' + }) + + # Validate notification type + if all([not COURSE_NOTIFICATION_TYPES.get(notification_type), notification_type != "core"]): + raise ValidationError(f'{notification_type} is not a valid notification type.') + + # Validate notification type and channel is editable + if notification_channel and notification_type: + if not is_notification_type_channel_editable( + notification_app, + notification_type, + notification_channel + ): + raise ValidationError({ + 'notification_channel': ( + f'{notification_channel} is not editable for notification type ' + f'{notification_type}.' + ) + }) + + return attrs diff --git a/openedx/core/djangoapps/notifications/tasks.py b/openedx/core/djangoapps/notifications/tasks.py index 75ad3f1ecd0b..a1792d27c3c9 100644 --- a/openedx/core/djangoapps/notifications/tasks.py +++ b/openedx/core/djangoapps/notifications/tasks.py @@ -185,9 +185,12 @@ def send_notifications(user_ids, course_key: str, app_name, notification_type, c ) if grouping_enabled and existing_notifications.get(user_id, None): group_user_notifications(new_notification, existing_notifications[user_id]) + if not notifications_generated: + notifications_generated = True + notification_content = new_notification.content else: notifications.append(new_notification) - generated_notification_audience.append(user_id) + generated_notification_audience.append(user_id) # send notification to users but use bulk_create notification_objects = Notification.objects.bulk_create(notifications) diff --git a/openedx/core/djangoapps/notifications/tests/test_notification_grouping.py b/openedx/core/djangoapps/notifications/tests/test_notification_grouping.py index fea954a0eabb..a3aace632c8b 100644 --- a/openedx/core/djangoapps/notifications/tests/test_notification_grouping.py +++ b/openedx/core/djangoapps/notifications/tests/test_notification_grouping.py @@ -2,11 +2,13 @@ Tests for notification grouping module """ +import ddt import unittest from unittest.mock import MagicMock, patch from datetime import datetime from pytz import utc +from common.djangoapps.student.tests.factories import UserFactory from openedx.core.djangoapps.notifications.grouping_notifications import ( BaseNotificationGrouper, NotificationRegistry, @@ -15,6 +17,8 @@ get_user_existing_notifications, NewPostGrouper ) from openedx.core.djangoapps.notifications.models import Notification +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory class TestNotificationRegistry(unittest.TestCase): @@ -143,7 +147,8 @@ def test_new_post_with_same_user(self): self.assertFalse(updated_context.get('grouped', False)) -class TestGroupUserNotifications(unittest.TestCase): +@ddt.ddt +class TestGroupUserNotifications(ModuleStoreTestCase): """ Tests for the group_user_notifications function """ @@ -179,6 +184,36 @@ def test_group_user_notifications_no_grouper(self): self.assertFalse(old_notification.save.called) + @ddt.data(datetime(2023, 1, 1, tzinfo=utc), None) + def test_not_grouped_when_notification_is_seen(self, last_seen): + """ + Notification is not grouped if the notification is marked as seen + """ + course = CourseFactory() + user = UserFactory() + notification_params = { + 'app_name': 'discussion', + 'notification_type': 'new_discussion_post', + 'course_id': course.id, + 'group_by_id': course.id, + 'content_url': 'http://example.com', + 'user': user, + 'last_seen': last_seen, + } + Notification.objects.create(content_context={ + 'username': 'User1', + 'post_title': ' Post title', + 'replier_name': 'User 1', + + }, **notification_params) + existing_notifications = get_user_existing_notifications( + [user.id], 'new_discussion_post', course.id, course.id + ) + if last_seen is None: + assert existing_notifications[user.id] is not None + else: + assert existing_notifications[user.id] is None + class TestGetUserExistingNotifications(unittest.TestCase): """ diff --git a/openedx/core/djangoapps/notifications/tests/test_utils.py b/openedx/core/djangoapps/notifications/tests/test_utils.py new file mode 100644 index 000000000000..e300c63c1d77 --- /dev/null +++ b/openedx/core/djangoapps/notifications/tests/test_utils.py @@ -0,0 +1,288 @@ +""" +Test cases for the notification utility functions. +""" +import unittest + +from openedx.core.djangoapps.notifications.utils import aggregate_notification_configs + + +class TestAggregateNotificationConfigs(unittest.TestCase): + """ + Test cases for the aggregate_notification_configs function. + """ + + def test_empty_configs_list_returns_default(self): + """ + If the configs list is empty, the default config should be returned. + """ + default_config = [{ + "grading": { + "enabled": False, + "non_editable": {}, + "notification_types": { + "core": { + "web": False, + "push": False, + "email": False, + "email_cadence": "Daily" + } + } + } + }] + + result = aggregate_notification_configs(default_config) + assert result == default_config[0] + + def test_enable_notification_type(self): + """ + If a config enables a notification type, it should be enabled in the result. + """ + + config_list = [ + { + "grading": { + "enabled": False, + "non_editable": {}, + "notification_types": { + "core": { + "web": False, + "push": False, + "email": False, + "email_cadence": "Weekly" + } + } + } + }, + { + "grading": { + "enabled": True, + "notification_types": { + "core": { + "web": True, + "push": True, + "email": True, + "email_cadence": "Weekly" + } + } + } + }] + + result = aggregate_notification_configs(config_list) + assert result["grading"]["enabled"] is True + assert result["grading"]["notification_types"]["core"]["web"] is True + assert result["grading"]["notification_types"]["core"]["push"] is True + assert result["grading"]["notification_types"]["core"]["email"] is True + # Use default email_cadence + assert result["grading"]["notification_types"]["core"]["email_cadence"] == "Weekly" + + def test_merge_core_notification_types(self): + """ + Core notification types should be merged across configs. + """ + + config_list = [ + { + "discussion": { + "enabled": True, + "core_notification_types": ["new_comment"], + "notification_types": {} + } + }, + { + "discussion": { + "core_notification_types": ["new_response", "new_comment"] + } + + }] + + result = aggregate_notification_configs(config_list) + assert set(result["discussion"]["core_notification_types"]) == { + "new_comment", "new_response" + } + + def test_multiple_configs_aggregate(self): + """ + Multiple configs should be aggregated together. + """ + + config_list = [ + { + "updates": { + "enabled": False, + "notification_types": { + "course_updates": { + "web": False, + "push": False, + "email": False, + "email_cadence": "Weekly" + } + } + } + }, + { + "updates": { + "enabled": True, + "notification_types": { + "course_updates": { + "web": True, + "email_cadence": "Weekly" + } + } + } + }, + { + "updates": { + "notification_types": { + "course_updates": { + "push": True, + "email_cadence": "Weekly" + } + } + } + } + ] + + result = aggregate_notification_configs(config_list) + assert result["updates"]["enabled"] is True + assert result["updates"]["notification_types"]["course_updates"]["web"] is True + assert result["updates"]["notification_types"]["course_updates"]["push"] is True + assert result["updates"]["notification_types"]["course_updates"]["email"] is False + # Use default email_cadence + assert result["updates"]["notification_types"]["course_updates"]["email_cadence"] == "Weekly" + + def test_ignore_unknown_notification_types(self): + """ + Unknown notification types should be ignored. + """ + config_list = [ + { + "grading": { + "enabled": False, + "notification_types": { + "core": { + "web": False, + "push": False, + "email": False, + "email_cadence": "Daily" + } + } + } + }, + { + "grading": { + "notification_types": { + "unknown_type": { + "web": True, + "push": True, + "email": True + } + } + } + }] + + result = aggregate_notification_configs(config_list) + assert "unknown_type" not in result["grading"]["notification_types"] + assert result["grading"]["notification_types"]["core"]["web"] is False + + def test_ignore_unknown_categories(self): + """ + Unknown categories should be ignored. + """ + + config_list = [ + { + "grading": { + "enabled": False, + "notification_types": {} + } + }, + { + "unknown_category": { + "enabled": True, + "notification_types": {} + } + }] + + result = aggregate_notification_configs(config_list) + assert "unknown_category" not in result + assert result["grading"]["enabled"] is False + + def test_preserves_default_structure(self): + """ + The resulting config should have the same structure as the default config. + """ + + config_list = [ + { + "discussion": { + "enabled": False, + "non_editable": {"core": ["web"]}, + "notification_types": { + "core": { + "web": False, + "push": False, + "email": False, + "email_cadence": "Weekly" + } + }, + "core_notification_types": [] + } + }, + { + "discussion": { + "enabled": True, + "extra_field": "should_not_appear" + } + } + ] + + result = aggregate_notification_configs(config_list) + assert set(result["discussion"].keys()) == { + "enabled", "non_editable", "notification_types", "core_notification_types" + } + assert "extra_field" not in result["discussion"] + + def test_if_email_cadence_has_diff_set_mix_as_value(self): + """ + If email_cadence is different in the configs, set it to "Mixed". + """ + config_list = [ + { + "grading": { + "enabled": False, + "notification_types": { + "core": { + "web": False, + "push": False, + "email": False, + "email_cadence": "Daily" + } + } + } + }, + { + "grading": { + "enabled": True, + "notification_types": { + "core": { + "web": True, + "push": True, + "email": True, + "email_cadence": "Weekly" + } + } + } + }, + { + "grading": { + "notification_types": { + "core": { + "email_cadence": "Monthly" + } + } + } + } + ] + + result = aggregate_notification_configs(config_list) + assert result["grading"]["notification_types"]["core"]["email_cadence"] == "Mixed" diff --git a/openedx/core/djangoapps/notifications/tests/test_views.py b/openedx/core/djangoapps/notifications/tests/test_views.py index 70e6fbc5739c..3e73c95be5be 100644 --- a/openedx/core/djangoapps/notifications/tests/test_views.py +++ b/openedx/core/djangoapps/notifications/tests/test_views.py @@ -2,11 +2,14 @@ Tests for the views in the notifications app. """ import json +from copy import deepcopy from datetime import datetime, timedelta from unittest import mock +from unittest.mock import patch import ddt from django.conf import settings +from django.contrib.auth import get_user_model from django.test.utils import override_settings from django.urls import reverse from edx_toggles.toggles.testutils import override_waffle_flag @@ -27,19 +30,21 @@ FORUM_ROLE_MODERATOR ) from openedx.core.djangoapps.notifications.config.waffle import ENABLE_NOTIFICATIONS +from openedx.core.djangoapps.notifications.email.utils import encrypt_object, encrypt_string from openedx.core.djangoapps.notifications.models import ( CourseNotificationPreference, Notification, get_course_notification_preference_config_version ) from openedx.core.djangoapps.notifications.serializers import NotificationCourseEnrollmentSerializer -from openedx.core.djangoapps.notifications.email.utils import encrypt_object, encrypt_string from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory from ..base_notification import COURSE_NOTIFICATION_APPS, COURSE_NOTIFICATION_TYPES, NotificationAppManager from ..utils import get_notification_types_with_visibility_settings +User = get_user_model() + @ddt.ddt class CourseEnrollmentListViewTest(ModuleStoreTestCase): @@ -903,6 +908,7 @@ class UpdatePreferenceFromEncryptedDataView(ModuleStoreTestCase): """ Tests if preference is updated when encrypted url is hit """ + def setUp(self): """ Setup test case @@ -968,3 +974,310 @@ def remove_notifications_with_visibility_settings(expected_response): notification_type ) return expected_response + + +class UpdateAllNotificationPreferencesViewTests(APITestCase): + """ + Tests for the UpdateAllNotificationPreferencesView. + """ + + def setUp(self): + # Create test user + self.user = User.objects.create_user( + username='testuser', + password='testpass123' + ) + self.client = APIClient() + self.client.force_authenticate(user=self.user) + self.url = reverse('update-all-notification-preferences') + + # Complex notification config structure + self.base_config = { + "grading": { + "enabled": True, + "non_editable": {}, + "notification_types": { + "core": { + "web": True, + "push": True, + "email": True, + "email_cadence": "Daily" + }, + "ora_staff_notification": { + "web": False, + "push": False, + "email": False, + "email_cadence": "Daily" + } + }, + "core_notification_types": [] + }, + "updates": { + "enabled": True, + "non_editable": {}, + "notification_types": { + "core": { + "web": True, + "push": True, + "email": True, + "email_cadence": "Daily" + }, + "course_updates": { + "web": True, + "push": True, + "email": False, + "email_cadence": "Daily" + } + }, + "core_notification_types": [] + }, + "discussion": { + "enabled": True, + "non_editable": { + "core": ["web"] + }, + "notification_types": { + "core": { + "web": True, + "push": True, + "email": True, + "email_cadence": "Daily" + }, + "content_reported": { + "web": True, + "push": True, + "email": True, + "email_cadence": "Daily" + }, + "new_question_post": { + "web": True, + "push": False, + "email": False, + "email_cadence": "Daily" + }, + "new_discussion_post": { + "web": True, + "push": False, + "email": False, + "email_cadence": "Daily" + } + }, + "core_notification_types": [ + "new_comment_on_response", + "new_comment", + "new_response", + "response_on_followed_post", + "comment_on_followed_post", + "response_endorsed_on_thread", + "response_endorsed" + ] + } + } + + # Create test notification preferences + self.preferences = [] + for i in range(3): + pref = CourseNotificationPreference.objects.create( + user=self.user, + course_id=f'course-v1:TestX+Test{i}+2024', + notification_preference_config=deepcopy(self.base_config), + is_active=True + ) + self.preferences.append(pref) + + # Create an inactive preference + self.inactive_pref = CourseNotificationPreference.objects.create( + user=self.user, + course_id='course-v1:TestX+Inactive+2024', + notification_preference_config=deepcopy(self.base_config), + is_active=False + ) + + def test_update_discussion_notification(self): + """ + Test updating discussion notification settings + """ + data = { + 'notification_app': 'discussion', + 'notification_type': 'content_reported', + 'notification_channel': 'push', + 'value': False + } + + response = self.client.post(self.url, data, format='json') + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['status'], 'success') + self.assertEqual(response.data['data']['total_updated'], 3) + + # Verify database updates + for pref in CourseNotificationPreference.objects.filter(is_active=True): + self.assertFalse( + pref.notification_preference_config['discussion']['notification_types']['content_reported']['push'] + ) + + def test_update_non_editable_field(self): + """ + Test attempting to update a non-editable field + """ + data = { + 'notification_app': 'discussion', + 'notification_type': 'core', + 'notification_channel': 'web', + 'value': False + } + + response = self.client.post(self.url, data, format='json') + + # Should fail because 'web' is non-editable for 'core' in discussion + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(response.data['status'], 'error') + + # Verify database remains unchanged + for pref in CourseNotificationPreference.objects.filter(is_active=True): + self.assertTrue( + pref.notification_preference_config['discussion']['notification_types']['core']['web'] + ) + + def test_update_email_cadence(self): + """ + Test updating email cadence setting + """ + data = { + 'notification_app': 'discussion', + 'notification_type': 'content_reported', + 'email_cadence': 'Weekly' + } + + response = self.client.post(self.url, data, format='json') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['status'], 'success') + + # Verify database updates + for pref in CourseNotificationPreference.objects.filter(is_active=True): + notification_type = pref.notification_preference_config['discussion']['notification_types'][ + 'content_reported'] + self.assertEqual( + notification_type['email_cadence'], + 'Weekly' + ) + + @patch.dict('openedx.core.djangoapps.notifications.serializers.COURSE_NOTIFICATION_APPS', { + **COURSE_NOTIFICATION_APPS, + 'grading': { + 'enabled': False, + 'core_info': 'Notifications for submission grading.', + 'core_web': True, + 'core_email': True, + 'core_push': True, + 'core_email_cadence': 'Daily', + 'non_editable': [] + } + }) + def test_update_disabled_app(self): + """ + Test updating notification for a disabled app + """ + # Disable the grading app in all preferences + for pref in self.preferences: + config = pref.notification_preference_config + config['grading']['enabled'] = False + pref.notification_preference_config = config + pref.save() + + data = { + 'notification_app': 'grading', + 'notification_type': 'core', + 'notification_channel': 'email', + 'value': False + } + response = self.client.post(self.url, data, format='json') + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(response.data['status'], 'error') + + def test_invalid_serializer_data(self): + """ + Test handling of invalid input data + """ + test_cases = [ + { + 'notification_app': 'invalid_app', + 'notification_type': 'core', + 'notification_channel': 'push', + 'value': False + }, + { + 'notification_app': 'discussion', + 'notification_type': 'invalid_type', + 'notification_channel': 'push', + 'value': False + }, + { + 'notification_app': 'discussion', + 'notification_type': 'core', + 'notification_channel': 'invalid_channel', + 'value': False + }, + { + 'notification_app': 'discussion', + 'notification_type': 'core', + 'notification_channel': 'email_cadence', + 'value': 'Invalid_Cadence' + } + ] + + for test_case in test_cases: + response = self.client.post(self.url, test_case, format='json') + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + +class GetAggregateNotificationPreferencesTest(APITestCase): + """ + Tests for the GetAggregateNotificationPreferences API view. + """ + + def setUp(self): + # Set up a user and API client + self.user = User.objects.create_user(username='testuser', password='testpass') + self.client = APIClient() + self.client.force_authenticate(user=self.user) + self.url = reverse('notification-preferences-aggregated') # Adjust with the actual name + + def test_no_active_notification_preferences(self): + """ + Test case: No active notification preferences found for the user + """ + response = self.client.get(self.url) + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + self.assertEqual(response.data['status'], 'error') + self.assertEqual(response.data['message'], 'No active notification preferences found') + + @patch('openedx.core.djangoapps.notifications.views.aggregate_notification_configs') + def test_with_active_notification_preferences(self, mock_aggregate): + """ + Test case: Active notification preferences found for the user + """ + # Mock aggregate_notification_configs for a controlled output + mock_aggregate.return_value = {'mocked': 'data'} + + # Create active notification preferences for the user + CourseNotificationPreference.objects.create( + user=self.user, + is_active=True, + notification_preference_config={'example': 'config'} + ) + + response = self.client.get(self.url) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['status'], 'success') + self.assertEqual(response.data['message'], 'Notification preferences retrieved') + self.assertEqual(response.data['data'], {'mocked': 'data'}) + + def test_unauthenticated_user(self): + """ + Test case: Request without authentication + """ + # Test case: Request without authentication + self.client.logout() # Remove authentication + response = self.client.get(self.url) + self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) diff --git a/openedx/core/djangoapps/notifications/urls.py b/openedx/core/djangoapps/notifications/urls.py index 7f611bc2c4ca..9892fa72de0d 100644 --- a/openedx/core/djangoapps/notifications/urls.py +++ b/openedx/core/djangoapps/notifications/urls.py @@ -11,13 +11,13 @@ NotificationCountView, NotificationListAPIView, NotificationReadAPIView, + UpdateAllNotificationPreferencesView, UserNotificationPreferenceView, - preference_update_from_encrypted_username_view, + preference_update_from_encrypted_username_view, AggregatedNotificationPreferences ) router = routers.DefaultRouter() - urlpatterns = [ path('enrollments/', CourseEnrollmentListView.as_view(), name='enrollment-list'), re_path( @@ -25,6 +25,11 @@ UserNotificationPreferenceView.as_view(), name='notification-preferences' ), + path( + 'configurations/', + AggregatedNotificationPreferences.as_view(), + name='notification-preferences-aggregated' + ), path('', NotificationListAPIView.as_view(), name='notifications-list'), path('count/', NotificationCountView.as_view(), name='notifications-count'), path( @@ -35,6 +40,11 @@ path('read/', NotificationReadAPIView.as_view(), name='notifications-read'), path('preferences/update///', preference_update_from_encrypted_username_view, name='preference_update_from_encrypted_username_view'), + path( + 'preferences/update-all/', + UpdateAllNotificationPreferencesView.as_view(), + name='update-all-notification-preferences' + ), ] urlpatterns += router.urls diff --git a/openedx/core/djangoapps/notifications/utils.py b/openedx/core/djangoapps/notifications/utils.py index fa948dcf425e..34f9d71cb04b 100644 --- a/openedx/core/djangoapps/notifications/utils.py +++ b/openedx/core/djangoapps/notifications/utils.py @@ -1,11 +1,12 @@ """ Utils function for notifications app """ -from typing import Dict, List +import copy +from typing import Dict, List, Set from common.djangoapps.student.models import CourseAccessRole, CourseEnrollment from openedx.core.djangoapps.django_comment_common.models import Role -from openedx.core.djangoapps.notifications.config.waffle import ENABLE_NOTIFICATIONS, ENABLE_NEW_NOTIFICATION_VIEW +from openedx.core.djangoapps.notifications.config.waffle import ENABLE_NEW_NOTIFICATION_VIEW, ENABLE_NOTIFICATIONS from openedx.core.lib.cache_utils import request_cached @@ -158,3 +159,113 @@ def clean_arguments(kwargs): if kwargs.get('created', {}): clean_kwargs.update(kwargs.get('created')) return clean_kwargs + + +def update_notification_types( + app_config: Dict, + user_app_config: Dict, +) -> None: + """ + Update notification types for a specific category configuration. + """ + if "notification_types" not in user_app_config: + return + + for type_key, type_config in user_app_config["notification_types"].items(): + if type_key not in app_config["notification_types"]: + continue + + update_notification_fields( + app_config["notification_types"][type_key], + type_config, + ) + + +def update_notification_fields( + target_config: Dict, + source_config: Dict, +) -> None: + """ + Update individual notification fields (web, push, email) and email_cadence. + """ + for field in ["web", "push", "email"]: + if field in source_config: + target_config[field] |= source_config[field] + if "email_cadence" in source_config: + if not target_config.get("email_cadence") or isinstance(target_config.get("email_cadence"), str): + target_config["email_cadence"] = set() + + target_config["email_cadence"].add(source_config["email_cadence"]) + + +def update_core_notification_types(app_config: Dict, user_config: Dict) -> None: + """ + Update core notification types by merging existing and new types. + """ + if "core_notification_types" not in user_config: + return + + existing_types: Set = set(app_config.get("core_notification_types", [])) + existing_types.update(user_config["core_notification_types"]) + app_config["core_notification_types"] = list(existing_types) + + +def process_app_config( + app_config: Dict, + user_config: Dict, + app: str, + default_config: Dict, +) -> None: + """ + Process a single category configuration against another config. + """ + if app not in user_config: + return + + user_app_config = user_config[app] + + # Update enabled status + app_config["enabled"] |= user_app_config.get("enabled", False) + + # Update core notification types + update_core_notification_types(app_config, user_app_config) + + # Update notification types + update_notification_types(app_config, user_app_config) + + +def aggregate_notification_configs(existing_user_configs: List[Dict]) -> Dict: + """ + Update default notification config with values from other configs. + Rules: + 1. Start with default config as base + 2. If any value is True in other configs, make it True + 3. Set email_cadence to "Mixed" if different cadences found, else use default + + Args: + existing_user_configs: List of notification config dictionaries to apply + + Returns: + Updated config following the same structure + """ + if not existing_user_configs: + return {} + + result_config = copy.deepcopy(existing_user_configs[0]) + apps = result_config.keys() + + for app in apps: + app_config = result_config[app] + + for user_config in existing_user_configs: + process_app_config(app_config, user_config, app, existing_user_configs[0]) + + # if email_cadence is mixed, set it to "Mixed" + for app in result_config: + for type_key, type_config in result_config[app]["notification_types"].items(): + if len(type_config.get("email_cadence", [])) > 1: + result_config[app]["notification_types"][type_key]["email_cadence"] = "Mixed" + else: + result_config[app]["notification_types"][type_key]["email_cadence"] = ( + result_config[app]["notification_types"][type_key]["email_cadence"].pop()) + return result_config diff --git a/openedx/core/djangoapps/notifications/views.py b/openedx/core/djangoapps/notifications/views.py index e87274088f84..8e41b11554c8 100644 --- a/openedx/core/djangoapps/notifications/views.py +++ b/openedx/core/djangoapps/notifications/views.py @@ -1,9 +1,11 @@ """ Views for the notifications API. """ +import copy from datetime import datetime, timedelta from django.conf import settings +from django.db import transaction from django.db.models import Count from django.shortcuts import get_object_or_404 from django.utils.translation import gettext as _ @@ -17,10 +19,7 @@ from common.djangoapps.student.models import CourseEnrollment from openedx.core.djangoapps.notifications.email.utils import update_user_preferences_from_patch -from openedx.core.djangoapps.notifications.models import ( - CourseNotificationPreference, - get_course_notification_preference_config_version -) +from openedx.core.djangoapps.notifications.models import get_course_notification_preference_config_version from openedx.core.djangoapps.notifications.permissions import allow_any_authenticated_user from .base_notification import COURSE_NOTIFICATION_APPS @@ -32,14 +31,15 @@ notification_tray_opened_event, notifications_app_all_read_event ) -from .models import Notification +from .models import CourseNotificationPreference, Notification from .serializers import ( NotificationCourseEnrollmentSerializer, NotificationSerializer, UserCourseNotificationPreferenceSerializer, - UserNotificationPreferenceUpdateSerializer, + UserNotificationPreferenceUpdateAllSerializer, + UserNotificationPreferenceUpdateSerializer ) -from .utils import get_show_notifications_tray, get_is_new_notification_view_enabled +from .utils import get_is_new_notification_view_enabled, get_show_notifications_tray, aggregate_notification_configs @allow_any_authenticated_user() @@ -444,3 +444,144 @@ def preference_update_from_encrypted_username_view(request, username, patch): """ update_user_preferences_from_patch(username, patch) return Response({"result": "success"}, status=status.HTTP_200_OK) + + +@allow_any_authenticated_user() +class UpdateAllNotificationPreferencesView(APIView): + """ + API view for updating all notification preferences for the current user. + """ + + def post(self, request): + """ + Update all notification preferences for the current user. + """ + # check if request have required params + serializer = UserNotificationPreferenceUpdateAllSerializer(data=request.data) + if not serializer.is_valid(): + return Response({ + 'status': 'error', + 'message': serializer.errors + }, status=status.HTTP_400_BAD_REQUEST) + # check if required config is not editable + try: + with transaction.atomic(): + # Get all active notification preferences for the current user + notification_preferences = ( + CourseNotificationPreference.objects + .select_for_update() + .filter( + user=request.user, + is_active=True + ) + ) + + if not notification_preferences.exists(): + return Response({ + 'status': 'error', + 'message': 'No active notification preferences found' + }, status=status.HTTP_404_NOT_FOUND) + + data = serializer.validated_data + app = data['notification_app'] + email_cadence = data.get('email_cadence', None) + channel = data.get('notification_channel', 'email_cadence' if email_cadence else None) + notification_type = data['notification_type'] + value = data.get('value', email_cadence if email_cadence else None) + + updated_courses = [] + errors = [] + + # Update each preference + for preference in notification_preferences: + try: + # Create a deep copy of the current config + updated_config = copy.deepcopy(preference.notification_preference_config) + + # Check if the path exists and update the value + if ( + updated_config.get(app, {}) + .get('notification_types', {}) + .get(notification_type, {}) + .get(channel) + ) is not None: + + # Update the specific setting in the config + updated_config[app]['notification_types'][notification_type][channel] = value + + # Update the notification preference + preference.notification_preference_config = updated_config + preference.save() + + updated_courses.append({ + 'course_id': str(preference.course_id), + 'current_setting': updated_config[app]['notification_types'][notification_type] + }) + else: + errors.append({ + 'course_id': str(preference.course_id), + 'error': f'Invalid path: {app}.notification_types.{notification_type}.{channel}' + }) + + except Exception as e: + errors.append({ + 'course_id': str(preference.course_id), + 'error': str(e) + }) + + response_data = { + 'status': 'success' if updated_courses else 'partial_success' if errors else 'error', + 'message': 'Notification preferences update completed', + 'data': { + 'updated_value': value, + 'notification_type': notification_type, + 'channel': channel, + 'app': app, + 'successfully_updated_courses': updated_courses, + 'total_updated': len(updated_courses), + 'total_courses': notification_preferences.count() + } + } + + if errors: + response_data['errors'] = errors + + return Response( + response_data, + status=status.HTTP_200_OK if updated_courses else status.HTTP_400_BAD_REQUEST + ) + + except Exception as e: + return Response({ + 'status': 'error', + 'message': str(e) + }, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + +@allow_any_authenticated_user() +class AggregatedNotificationPreferences(APIView): + """ + API view for getting the aggregate notification preferences for the current user. + """ + + def get(self, request): + """ + API view for getting the aggregate notification preferences for the current user. + """ + notification_preferences = CourseNotificationPreference.objects.filter(user=request.user, is_active=True) + + if not notification_preferences.exists(): + return Response({ + 'status': 'error', + 'message': 'No active notification preferences found' + }, status=status.HTTP_404_NOT_FOUND) + notification_configs = notification_preferences.values_list('notification_preference_config', flat=True) + notification_configs = aggregate_notification_configs( + notification_configs + ) + + return Response({ + 'status': 'success', + 'message': 'Notification preferences retrieved', + 'data': notification_configs + }, status=status.HTTP_200_OK) diff --git a/openedx/core/djangoapps/schedules/resolvers.py b/openedx/core/djangoapps/schedules/resolvers.py index 5f769e6af299..40d565962b0c 100644 --- a/openedx/core/djangoapps/schedules/resolvers.py +++ b/openedx/core/djangoapps/schedules/resolvers.py @@ -14,6 +14,7 @@ from edx_ace.recipient import Recipient from edx_ace.recipient_resolver import RecipientResolver from edx_django_utils.monitoring import function_trace, set_custom_attribute +from openedx_filters.learning.filters import ScheduleQuerySetRequested from lms.djangoapps.courseware.utils import verified_upgrade_deadline_link, can_show_verified_upgrade from lms.djangoapps.discussion.notification_prefs.views import UsernameCipher @@ -154,6 +155,10 @@ def get_schedules_with_target_date_by_bin_and_orgs( schedules = self.filter_by_org(schedules) + # .. filter_implemented_name: ScheduleQuerySetRequested + # .. filter_type: org.openedx.learning.schedule.queryset.requested.v1 + schedules = ScheduleQuerySetRequested.run_filter(schedules) + if "read_replica" in settings.DATABASES: schedules = schedules.using("read_replica") @@ -377,6 +382,13 @@ def send(self, msg_type): language, context, ) + LOG.info( + 'Sending email to user: {} for Instructor-paced course with course-key: {} and language: {}'.format( + user.username, + self.course_id, + language + ) + ) with function_trace('enqueue_send_task'): self.async_send_task.apply_async((self.site.id, str(msg)), retry=False) # pylint: disable=no-member @@ -464,6 +476,13 @@ def send(self): # lint-amnesty, pylint: disable=arguments-differ self.course_id ) ) + LOG.info( + 'Sending email to user: {} for Self-paced course with course-key: {} and language: {}'.format( + user.username, + self.course_id, + language + ) + ) with function_trace('enqueue_send_task'): self.async_send_task.apply_async((self.site.id, str(msg)), retry=False) diff --git a/openedx/core/djangoapps/schedules/tests/test_filters.py b/openedx/core/djangoapps/schedules/tests/test_filters.py new file mode 100644 index 000000000000..9f7efa050447 --- /dev/null +++ b/openedx/core/djangoapps/schedules/tests/test_filters.py @@ -0,0 +1,71 @@ +""" +Test cases for the Open edX Filters associated with the schedule app. +""" + +import datetime +from unittest.mock import Mock + +from django.db.models.query import QuerySet +from django.test import override_settings +from openedx_filters import PipelineStep + +from openedx.core.djangoapps.schedules.resolvers import BinnedSchedulesBaseResolver +from openedx.core.djangoapps.schedules.tests.test_resolvers import SchedulesResolverTestMixin +from openedx.core.djangolib.testing.utils import skip_unless_lms +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase + + +class TestScheduleQuerySetRequestedPipelineStep(PipelineStep): + """Pipeline step class to test a configured pipeline step""" + + filtered_schedules = Mock(spec=QuerySet, __len__=Mock(return_value=0)) + + def run_filter(self, schedules: QuerySet): # pylint: disable=arguments-differ + """Pipeline step to filter the schedules""" + return { + "schedules": self.filtered_schedules, + } + + +@skip_unless_lms +class ScheduleQuerySetRequestedFiltersTest(SchedulesResolverTestMixin, ModuleStoreTestCase): + """ + Tests for the Open edX Filters associated with the schedule queryset requested. + + The following filters are tested: + - ScheduleQuerySetRequested + """ + + def setUp(self): + super().setUp() + self.resolver = BinnedSchedulesBaseResolver( + async_send_task=Mock(name="async_send_task"), + site=self.site, + target_datetime=datetime.datetime.now(), + day_offset=3, + bin_num=2, + ) + self.resolver.schedule_date_field = "created" + + @override_settings( + OPEN_EDX_FILTERS_CONFIG={ + "org.openedx.learning.schedule.queryset.requested.v1": { + "pipeline": [ + "openedx.core.djangoapps.schedules.tests.test_filters.TestScheduleQuerySetRequestedPipelineStep", + ], + "fail_silently": False, + }, + }, + ) + def test_schedule_with_queryset_requested_filter_enabled(self) -> None: + """Test to verify the schedule queryset was modified by the pipeline step.""" + schedules = self.resolver.get_schedules_with_target_date_by_bin_and_orgs() + + self.assertEqual(TestScheduleQuerySetRequestedPipelineStep.filtered_schedules, schedules) + + @override_settings(OPEN_EDX_FILTERS_CONFIG={}) + def test_schedule_with_queryset_requested_filter_disabled(self) -> None: + """Test to verify the schedule queryset was not modified when the pipeline step is not configured.""" + schedules = self.resolver.get_schedules_with_target_date_by_bin_and_orgs() + + self.assertNotEqual(TestScheduleQuerySetRequestedPipelineStep.filtered_schedules, schedules) diff --git a/openedx/core/djangoapps/theming/management/commands/compile_sass.py b/openedx/core/djangoapps/theming/management/commands/compile_sass.py deleted file mode 100644 index fbfdd2f222a4..000000000000 --- a/openedx/core/djangoapps/theming/management/commands/compile_sass.py +++ /dev/null @@ -1,112 +0,0 @@ -""" -Management command for compiling sass. - -DEPRECATED in favor of `npm run compile-sass`. -""" -import shlex - -from django.core.management import BaseCommand -from django.conf import settings - -from pavelib.assets import run_deprecated_command_wrapper - - -class Command(BaseCommand): - """ - Compile theme sass and collect theme assets. - """ - - help = "DEPRECATED. Use 'npm run compile-sass' instead." - - # NOTE (CCB): This allows us to compile static assets in Docker containers without database access. - requires_system_checks = [] - - def add_arguments(self, parser): - """ - Add arguments for compile_sass command. - - Args: - parser (django.core.management.base.CommandParser): parsed for parsing command line arguments. - """ - parser.add_argument( - 'system', type=str, nargs='*', default=["lms", "cms"], - help="lms or studio", - ) - - # Named (optional) arguments - parser.add_argument( - '--theme-dirs', - dest='theme_dirs', - type=str, - nargs='+', - default=None, - help="List of dirs where given themes would be looked.", - ) - - parser.add_argument( - '--themes', - type=str, - nargs='+', - default=["all"], - help="List of themes whose sass need to compiled. Or 'no'/'all' to compile for no/all themes.", - ) - - # Named (optional) arguments - parser.add_argument( - '--force', - action='store_true', - default=False, - help="DEPRECATED. Full recompilation is now always forced.", - ) - parser.add_argument( - '--debug', - action='store_true', - default=False, - help="Disable Sass compression", - ) - - def handle(self, *args, **options): - """ - Handle compile_sass command. - """ - systems = set( - {"lms": "lms", "cms": "cms", "studio": "cms"}[sys] - for sys in options.get("system", ["lms", "cms"]) - ) - theme_dirs = options.get("theme_dirs") or settings.COMPREHENSIVE_THEME_DIRS or [] - themes_option = options.get("themes") or [] # '[]' means 'all' - if not settings.ENABLE_COMPREHENSIVE_THEMING: - compile_themes = False - themes = [] - elif "no" in themes_option: - compile_themes = False - themes = [] - elif "all" in themes_option: - compile_themes = True - themes = [] - else: - compile_themes = True - themes = themes_option - run_deprecated_command_wrapper( - old_command="./manage.py [lms|cms] compile_sass", - ignored_old_flags=list(set(["force"]) & set(options)), - new_command=shlex.join([ - "npm", - "run", - ("compile-sass-dev" if options.get("debug") else "compile-sass"), - "--", - *(["--skip-lms"] if "lms" not in systems else []), - *(["--skip-cms"] if "cms" not in systems else []), - *(["--skip-themes"] if not compile_themes else []), - *( - arg - for theme_dir in theme_dirs - for arg in ["--theme-dir", str(theme_dir)] - ), - *( - arg - for theme in themes - for arg in ["--theme", theme] - ), - ]), - ) diff --git a/openedx/core/djangoapps/util/management/commands/print_setting.py b/openedx/core/djangoapps/util/management/commands/print_setting.py index d90a17b9eb42..c53f49d23a19 100644 --- a/openedx/core/djangoapps/util/management/commands/print_setting.py +++ b/openedx/core/djangoapps/util/management/commands/print_setting.py @@ -3,7 +3,11 @@ ============= Django command to output a single Django setting. -Useful when paver or a shell script needs such a value. +Originally used by "paver" scripts before we removed them. +Still useful when a shell script needs such a value. +Keep in mind that the LMS/CMS startup time is slow, so if you invoke this +Django management multiple times in a command that gets run often, you are +going to be sad. This handles the one specific use case of the "print_settings" command from django-extensions that we were actually using. diff --git a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py index dde2084e54cd..44dedcf42874 100644 --- a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py +++ b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py @@ -6,6 +6,7 @@ import logging from collections import defaultdict from datetime import datetime, timezone +from urllib.parse import unquote from django.core.exceptions import ObjectDoesNotExist, ValidationError from django.db.transaction import atomic @@ -446,9 +447,20 @@ def _lookup_asset_url(self, block: XBlock, asset_path: str) -> str | None: .get(key=f"static/{asset_path}") ) except ObjectDoesNotExist: - # This means we see a path that _looks_ like it should be a static - # asset for this Component, but that static asset doesn't really - # exist. - return None + try: + # Retry with unquoted path. We don't always unquote because it would not + # be backwards-compatible, but we need to try both. + asset_path = unquote(asset_path) + content = ( + component_version + .componentversioncontent_set + .filter(content__has_file=True) + .get(key=f"static/{asset_path}") + ) + except ObjectDoesNotExist: + # This means we see a path that _looks_ like it should be a static + # asset for this Component, but that static asset doesn't really + # exist. + return None return self._absolute_url_for_asset(component_version, asset_path) diff --git a/openedx/core/types/http.py b/openedx/core/types/http.py new file mode 100644 index 000000000000..2896256a107a --- /dev/null +++ b/openedx/core/types/http.py @@ -0,0 +1,45 @@ +""" +Typing utilities for the HTTP requests, responses, etc. + +Includes utilties to work with both vanilla django as well as djangorestframework. +""" +from __future__ import annotations + +import django.contrib.auth.models # pylint: disable=imported-auth-user +import django.http +import rest_framework.request + +import openedx.core.types.user +from openedx.core.types.meta import type_annotation_only + + +@type_annotation_only +class HttpRequest(django.http.HttpRequest): + """ + A request which either has a concrete User (from django.contrib.auth) or is anonymous. + """ + user: openedx.core.types.User + + +@type_annotation_only +class AuthenticatedHttpRequest(HttpRequest): + """ + A request which is guaranteed to have a concrete User (from django.contrib.auth). + """ + user: django.contrib.auth.models.User + + +@type_annotation_only +class RestRequest(rest_framework.request.Request): + """ + Same as HttpRequest, but extended for rest_framework views. + """ + user: openedx.core.types.User + + +@type_annotation_only +class AuthenticatedRestRequest(RestRequest): + """ + Same as AuthenticatedHttpRequest, but extended for rest_framework views. + """ + user: django.contrib.auth.models.User diff --git a/openedx/core/types/meta.py b/openedx/core/types/meta.py new file mode 100644 index 000000000000..39162b05b879 --- /dev/null +++ b/openedx/core/types/meta.py @@ -0,0 +1,37 @@ +""" +Typing utilities for use on other typing utilities. +""" +from __future__ import annotations + +import typing as t + + +def type_annotation_only(cls: type) -> type: + """ + Decorates class which should only be used in type annotations. + + This is useful when you want to enhance an existing 3rd-party concrete class with + type annotations for its members, but don't want the enhanced class to ever actually + be instantiated. For examples, see openedx.core.types.http. + """ + if t.TYPE_CHECKING: + return cls + return _forbid_init(cls) + + +def _forbid_init(forbidden: type) -> type: + """ + Return a class which refuses to be instantiated. + """ + class _ForbidInit: + """ + The resulting class. + """ + def __init__(self, *args, **kwargs): + raise NotImplementedError( + f"Class {forbidden.__module__}:{forbidden.__name__} " + "cannot be instantiated. You may use it as a type annotation, but objects " + "can only be created from its concrete superclasses." + ) + + return _ForbidInit diff --git a/openedx/core/types/user.py b/openedx/core/types/user.py index 95b1fec607fc..9eb63edba358 100644 --- a/openedx/core/types/user.py +++ b/openedx/core/types/user.py @@ -1,8 +1,10 @@ """ Typing utilities for the User models. """ -from typing import Union +from __future__ import annotations + +import typing as t import django.contrib.auth.models -User = Union[django.contrib.auth.models.User, django.contrib.auth.models.AnonymousUser] +User: t.TypeAlias = django.contrib.auth.models.User | django.contrib.auth.models.AnonymousUser diff --git a/openedx/features/course_bookmarks/static/course_bookmarks/js/models/bookmark.js b/openedx/features/course_bookmarks/static/course_bookmarks/js/models/bookmark.js index 13671bddf56a..9ad45a5422ce 100644 --- a/openedx/features/course_bookmarks/static/course_bookmarks/js/models/bookmark.js +++ b/openedx/features/course_bookmarks/static/course_bookmarks/js/models/bookmark.js @@ -1,20 +1,37 @@ (function(define) { - 'use strict'; + 'use strict'; - define(['backbone'], function(Backbone) { - return Backbone.Model.extend({ - idAttribute: 'id', - defaults: { - course_id: '', - usage_id: '', - display_name: '', - path: [], - created: '' - }, + define(['backbone'], function(Backbone) { + return Backbone.Model.extend({ + idAttribute: 'id', + defaults: { + course_id: '', + usage_id: '', + display_name: '', + path: [], + created: '' + }, - blockUrl: function() { - return '/courses/' + this.get('course_id') + '/jump_to/' + this.get('usage_id'); - } - }); + blockUrl: function() { + var path = this.get('path'); + var url = '/courses/' + this.get('course_id') + '/jump_to/' + this.get('usage_id'); + var params = new URLSearchParams(); + var usage_id = this.get('usage_id'); + // Confirm that current usage_id does not correspond to current unit + // path contains an array of parent blocks to the bookmarked block. + // Units only have two parents i.e. section and subsections. + // Below condition is only satisfied if a block lower than unit is bookmarked. + if (path.length > 2 && usage_id !== path[path.length - 1]) { + params.append('jumpToId', usage_id); + } + if (params.size > 0) { + // Pass nested block details via query parameters for it to be passed to learning mfe + // The learning mfe should pass it back to unit xblock via iframe url params. + // This would allow us to scroll to the child xblock. + url = url + '?' + params.toString(); + } + return url; + } }); + }); }(define || RequireJS.define)); diff --git a/openedx/features/course_bookmarks/static/course_bookmarks/js/views/bookmark_button.js b/openedx/features/course_bookmarks/static/course_bookmarks/js/views/bookmark_button.js index 838f631868dc..3612038842c5 100644 --- a/openedx/features/course_bookmarks/static/course_bookmarks/js/views/bookmark_button.js +++ b/openedx/features/course_bookmarks/static/course_bookmarks/js/views/bookmark_button.js @@ -20,6 +20,12 @@ this.bookmarkId = options.bookmarkId; this.bookmarked = options.bookmarked; this.usageId = options.usageId; + if (options.bookmarkedText) { + this.bookmarkedText = options.bookmarkedText; + } + if (options.bookmarkText) { + this.bookmarkText = options.bookmarkText; + } this.setBookmarkState(this.bookmarked); }, diff --git a/openedx/features/course_bookmarks/static/course_bookmarks/js/views/bookmarks_list.js b/openedx/features/course_bookmarks/static/course_bookmarks/js/views/bookmarks_list.js index 52f5fbd74c1e..55dd1bd58ae5 100644 --- a/openedx/features/course_bookmarks/static/course_bookmarks/js/views/bookmarks_list.js +++ b/openedx/features/course_bookmarks/static/course_bookmarks/js/views/bookmarks_list.js @@ -78,9 +78,7 @@ component_type: componentType, component_usage_id: componentUsageId } - ).always(function() { - window.location.href = event.currentTarget.pathname; - }); + ); }, /** diff --git a/package.json b/package.json index 92f7de9124dc..8ed93a322633 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,27 @@ "compile-sass-dev": "scripts/compile_sass.py --env=development", "watch": "{ npm run watch-webpack& npm run watch-sass& } && sleep infinity", "watch-webpack": "npm run webpack-dev -- --watch", - "watch-sass": "scripts/watch_sass.sh" + "watch-sass": "scripts/watch_sass.sh", + "lint": "python scripts/eslint.py", + "test": "npm run test-jest && npm run test-karma", + "test-jest": "jest", + "test-karma": "npm run test-karma-vanilla && npm run test-karma-require && echo 'WARNING: Skipped broken webpack tests. For details, see: https://github.com/openedx/edx-platform/issues/35956'", + "test-karma-vanilla": "npm run test-cms-vanilla && npm run test-xmodule-vanilla && npm run test-common-vanilla", + "test-karma-require": "npm run test-cms-require && npm run test-common-require", + "test-karma-webpack": "npm run test-cms-webpack && npm run test-lms-webpack && npm run test-xmodule-webpack", + "test-karma-conf": "${NODE_WRAPPER:-xvfb-run --auto-servernum} node --max_old_space_size=4096 node_modules/.bin/karma start --single-run=true --capture-timeout=60000 --browsers=FirefoxNoUpdates", + "test-cms": "npm run test-cms-vanilla && npm run test-cms-require && npm run test-cms-webpack", + "test-cms-vanilla": "npm run test-karma-conf -- cms/static/karma_cms.conf.js", + "test-cms-require": "npm run test-karma-conf -- cms/static/karma_cms_squire.conf.js", + "test-cms-webpack": "npm run test-karma-conf -- cms/static/karma_cms_webpack.conf.js", + "test-lms": "npm run test-jest && npm run test-lms-webpack", + "test-lms-webpack": "npm run test-karma-conf -- lms/static/karma_lms.conf.js", + "test-xmodule": "npm run test-xmodule-vanilla && npm run test-xmodule-webpack", + "test-xmodule-vanilla": "npm run test-karma-conf -- xmodule/js/karma_xmodule.conf.js", + "test-xmodule-webpack": "npm run test-karma-conf -- xmodule/js/karma_xmodule_webpack.conf.js", + "test-common": "npm run test-common-vanilla && npm run test-common-require", + "test-common-vanilla": "npm run test-karma-conf -- common/static/karma_common.conf.js", + "test-common-require": "npm run test-karma-conf -- common/static/karma_common_requirejs.conf.js" }, "dependencies": { "@babel/core": "7.26.0", diff --git a/pavelib/__init__.py b/pavelib/__init__.py deleted file mode 100644 index 875068166ff5..000000000000 --- a/pavelib/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -""" # lint-amnesty, pylint: disable=django-not-configured -paver commands -""" - - -from . import assets, js_test, prereqs, quality diff --git a/pavelib/assets.py b/pavelib/assets.py deleted file mode 100644 index f437b6427f93..000000000000 --- a/pavelib/assets.py +++ /dev/null @@ -1,506 +0,0 @@ -""" -Asset compilation and collection. - -This entire module is DEPRECATED. In Redwood, it exists just as a collection of temporary compatibility wrappers. -In Sumac, this module will be deleted. To migrate, follow the advice in the printed warnings and/or read the -instructions on the DEPR ticket: https://github.com/openedx/edx-platform/issues/31895 -""" - -import argparse -import glob -import json -import shlex -import traceback -from functools import wraps -from threading import Timer - -from paver import tasks -from paver.easy import call_task, cmdopts, consume_args, needs, no_help, sh, task -from watchdog.events import PatternMatchingEventHandler -from watchdog.observers import Observer # pylint: disable=unused-import # Used by Tutor. Remove after Sumac cut. - -from .utils.cmd import django_cmd -from .utils.envs import Env -from .utils.timer import timed - - -SYSTEMS = { - 'lms': 'lms', - 'cms': 'cms', - 'studio': 'cms', -} - -WARNING_SYMBOLS = "⚠️ " * 50 # A row of 'warning' emoji to catch CLI users' attention - - -def run_deprecated_command_wrapper(*, old_command, ignored_old_flags, new_command): - """ - Run the new version of shell command, plus a warning that the old version is deprecated. - """ - depr_warning = ( - "\n" + - f"{WARNING_SYMBOLS}\n" + - "\n" + - f"WARNING: '{old_command}' is DEPRECATED! It will be removed before Sumac.\n" + - "The command you ran is now just a temporary wrapper around a new,\n" + - "supported command, which you should use instead:\n" + - "\n" + - f"\t{new_command}\n" + - "\n" + - "Details: https://github.com/openedx/edx-platform/issues/31895\n" + - "".join( - f" WARNING: ignored deprecated paver flag '{flag}'\n" - for flag in ignored_old_flags - ) + - f"{WARNING_SYMBOLS}\n" + - "\n" - ) - # Print deprecation warning twice so that it's more likely to be seen in the logs. - print(depr_warning) - sh(new_command) - print(depr_warning) - - -def debounce(seconds=1): - """ - Prevents the decorated function from being called more than every `seconds` - seconds. Waits until calls stop coming in before calling the decorated - function. - - This is DEPRECATED. It exists in Redwood just to ease the transition for Tutor. - """ - def decorator(func): - func.timer = None - - @wraps(func) - def wrapper(*args, **kwargs): - def call(): - func(*args, **kwargs) - func.timer = None - if func.timer: - func.timer.cancel() - func.timer = Timer(seconds, call) - func.timer.start() - - return wrapper - return decorator - - -class SassWatcher(PatternMatchingEventHandler): - """ - Watches for sass file changes - - This is DEPRECATED. It exists in Redwood just to ease the transition for Tutor. - """ - ignore_directories = True - patterns = ['*.scss'] - - def register(self, observer, directories): - """ - register files with observer - Arguments: - observer (watchdog.observers.Observer): sass file observer - directories (list): list of directories to be register for sass watcher. - """ - for dirname in directories: - paths = [] - if '*' in dirname: - paths.extend(glob.glob(dirname)) - else: - paths.append(dirname) - - for obs_dirname in paths: - observer.schedule(self, obs_dirname, recursive=True) - - @debounce() - def on_any_event(self, event): - print('\tCHANGED:', event.src_path) - try: - compile_sass() # pylint: disable=no-value-for-parameter - except Exception: # pylint: disable=broad-except - traceback.print_exc() - - -@task -@no_help -@cmdopts([ - ('system=', 's', 'The system to compile sass for (defaults to all)'), - ('theme-dirs=', '-td', 'Theme dirs containing all themes (defaults to None)'), - ('themes=', '-t', 'The theme to compile sass for (defaults to None)'), - ('debug', 'd', 'Whether to use development settings'), - ('force', '', 'DEPRECATED. Full recompilation is now always forced.'), -]) -@timed -def compile_sass(options): - """ - Compile Sass to CSS. If command is called without any arguments, it will - only compile lms, cms sass for the open source theme. And none of the comprehensive theme's sass would be compiled. - - If you want to compile sass for all comprehensive themes you will have to run compile_sass - specifying all the themes that need to be compiled.. - - The following is a list of some possible ways to use this command. - - Command: - paver compile_sass - Description: - compile sass files for both lms and cms. If command is called like above (i.e. without any arguments) it will - only compile lms, cms sass for the open source theme. None of the theme's sass will be compiled. - - Command: - paver compile_sass --theme-dirs /edx/app/edxapp/edx-platform/themes --themes=red-theme - Description: - compile sass files for both lms and cms for 'red-theme' present in '/edx/app/edxapp/edx-platform/themes' - - Command: - paver compile_sass --theme-dirs=/edx/app/edxapp/edx-platform/themes --themes red-theme stanford-style - Description: - compile sass files for both lms and cms for 'red-theme' and 'stanford-style' present in - '/edx/app/edxapp/edx-platform/themes'. - - Command: - paver compile_sass --system=cms - --theme-dirs /edx/app/edxapp/edx-platform/themes /edx/app/edxapp/edx-platform/common/test/ - --themes red-theme stanford-style test-theme - Description: - compile sass files for cms only for 'red-theme', 'stanford-style' and 'test-theme' present in - '/edx/app/edxapp/edx-platform/themes' and '/edx/app/edxapp/edx-platform/common/test/'. - - This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run compile-sass` instead. - """ - systems = [SYSTEMS[sys] for sys in get_parsed_option(options, 'system', ['lms', 'cms'])] # normalize studio->cms - run_deprecated_command_wrapper( - old_command="paver compile_sass", - ignored_old_flags=(set(["--force"]) & set(options)), - new_command=shlex.join([ - "npm", - "run", - ("compile-sass-dev" if options.get("debug") else "compile-sass"), - "--", - *(["--dry"] if tasks.environment.dry_run else []), - *(["--skip-lms"] if "lms" not in systems else []), - *(["--skip-cms"] if "cms" not in systems else []), - *( - arg - for theme_dir in get_parsed_option(options, 'theme_dirs', []) - for arg in ["--theme-dir", str(theme_dir)] - ), - *( - arg - for theme in get_parsed_option(options, "themes", []) - for arg in ["--theme", theme] - ), - ]), - ) - - -def _compile_sass(system, theme, debug, force, _timing_info): - """ - This is a DEPRECATED COMPATIBILITY WRAPPER - - It exists to ease the transition for Tutor in Redwood, which directly imported and used this function. - """ - run_deprecated_command_wrapper( - old_command="pavelib.assets:_compile_sass", - ignored_old_flags=(set(["--force"]) if force else set()), - new_command=[ - "npm", - "run", - ("compile-sass-dev" if debug else "compile-sass"), - "--", - *(["--dry"] if tasks.environment.dry_run else []), - *( - ["--skip-default", "--theme-dir", str(theme.parent), "--theme", str(theme.name)] - if theme - else [] - ), - ("--skip-cms" if system == "lms" else "--skip-lms"), - ] - ) - - -def process_npm_assets(): - """ - Process vendor libraries installed via NPM. - - This is a DEPRECATED COMPATIBILITY WRAPPER. It is now handled as part of `npm clean-install`. - If you need to invoke it explicitly, you can run `npm run postinstall`. - """ - run_deprecated_command_wrapper( - old_command="pavelib.assets:process_npm_assets", - ignored_old_flags=[], - new_command=shlex.join(["npm", "run", "postinstall"]), - ) - - -@task -@no_help -def process_xmodule_assets(): - """ - Process XModule static assets. - - This is a DEPRECATED COMPATIBILITY STUB. Refrences to it should be deleted. - """ - print( - "\n" + - f"{WARNING_SYMBOLS}", - "\n" + - "WARNING: 'paver process_xmodule_assets' is DEPRECATED! It will be removed before Sumac.\n" + - "\n" + - "Starting with Quince, it is no longer necessary to post-process XModule assets, so \n" + - "'paver process_xmodule_assets' is a no-op. Please simply remove it from your build scripts.\n" + - "\n" + - "Details: https://github.com/openedx/edx-platform/issues/31895\n" + - f"{WARNING_SYMBOLS}", - ) - - -def collect_assets(systems, settings, **kwargs): - """ - Collect static assets, including Django pipeline processing. - `systems` is a list of systems (e.g. 'lms' or 'studio' or both) - `settings` is the Django settings module to use. - `**kwargs` include arguments for using a log directory for collectstatic output. Defaults to /dev/null. - - This is a DEPRECATED COMPATIBILITY WRAPPER - - It exists to ease the transition for Tutor in Redwood, which directly imported and used this function. - """ - run_deprecated_command_wrapper( - old_command="pavelib.asset:collect_assets", - ignored_old_flags=[], - new_command=" && ".join( - "( " + - shlex.join( - ["./manage.py", SYSTEMS[sys], f"--settings={settings}", "collectstatic", "--noinput"] - ) + ( - "" - if "collect_log_dir" not in kwargs else - " > /dev/null" - if kwargs["collect_log_dir"] is None else - f"> {kwargs['collect_log_dir']}/{SYSTEMS[sys]}-collectstatic.out" - ) + - " )" - for sys in systems - ), - ) - - -def execute_compile_sass(args): - """ - Construct django management command compile_sass (defined in theming app) and execute it. - Args: - args: command line argument passed via update_assets command - - This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run compile-sass` instead. - """ - for sys in args.system: - options = "" - options += " --theme-dirs " + " ".join(args.theme_dirs) if args.theme_dirs else "" - options += " --themes " + " ".join(args.themes) if args.themes else "" - options += " --debug" if args.debug else "" - - sh( - django_cmd( - sys, - args.settings, - "compile_sass {system} {options}".format( - system='cms' if sys == 'studio' else sys, - options=options, - ), - ), - ) - - -@task -@cmdopts([ - ('settings=', 's', "Django settings (defaults to devstack)"), - ('watch', 'w', "DEPRECATED. This flag never did anything anyway."), -]) -@timed -def webpack(options): - """ - Run a Webpack build. - - This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run webpack` instead. - """ - settings = getattr(options, 'settings', Env.DEVSTACK_SETTINGS) - result = Env.get_django_settings(['STATIC_ROOT', 'WEBPACK_CONFIG_PATH'], "lms", settings=settings) - static_root_lms, config_path = result - static_root_cms, = Env.get_django_settings(["STATIC_ROOT"], "cms", settings=settings) - js_env_extra_config_setting, = Env.get_django_json_settings(["JS_ENV_EXTRA_CONFIG"], "cms", settings=settings) - js_env_extra_config = json.dumps(js_env_extra_config_setting or "{}") - node_env = "development" if config_path == 'webpack.dev.config.js' else "production" - run_deprecated_command_wrapper( - old_command="paver webpack", - ignored_old_flags=(set(["watch"]) & set(options)), - new_command=' '.join([ - f"WEBPACK_CONFIG_PATH={config_path}", - f"NODE_ENV={node_env}", - f"STATIC_ROOT_LMS={static_root_lms}", - f"STATIC_ROOT_CMS={static_root_cms}", - f"JS_ENV_EXTRA_CONFIG={js_env_extra_config}", - "npm", - "run", - "webpack", - ]), - ) - - -def get_parsed_option(command_opts, opt_key, default=None): - """ - Extract user command option and parse it. - Arguments: - command_opts: Command line arguments passed via paver command. - opt_key: name of option to get and parse - default: if `command_opt_value` not in `command_opts`, `command_opt_value` will be set to default. - Returns: - list or None - """ - command_opt_value = getattr(command_opts, opt_key, default) - if command_opt_value: - command_opt_value = listfy(command_opt_value) - - return command_opt_value - - -def listfy(data): - """ - Check and convert data to list. - Arguments: - data: data structure to be converted. - """ - - if isinstance(data, str): - data = data.split(',') - elif not isinstance(data, list): - data = [data] - - return data - - -@task -@cmdopts([ - ('background', 'b', 'DEPRECATED. Use shell tools like & to run in background if needed.'), - ('settings=', 's', "DEPRECATED. Django is not longer invoked to compile JS/Sass."), - ('theme-dirs=', '-td', 'The themes dir containing all themes (defaults to None)'), - ('themes=', '-t', 'DEPRECATED. All themes in --theme-dirs are now watched.'), - ('wait=', '-w', 'DEPRECATED. Watchdog\'s default wait time is now used.'), -]) -@timed -def watch_assets(options): - """ - Watch for changes to asset files, and regenerate js/css - - This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run watch` instead. - """ - # Don't watch assets when performing a dry run - if tasks.environment.dry_run: - return - - theme_dirs = ':'.join(get_parsed_option(options, 'theme_dirs', [])) - run_deprecated_command_wrapper( - old_command="paver watch_assets", - ignored_old_flags=(set(["debug", "themes", "settings", "background"]) & set(options)), - new_command=shlex.join([ - *( - ["env", f"COMPREHENSIVE_THEME_DIRS={theme_dirs}"] - if theme_dirs else [] - ), - "npm", - "run", - "watch", - ]), - ) - - -@task -@needs( - 'pavelib.prereqs.install_node_prereqs', - 'pavelib.prereqs.install_python_prereqs', -) -@consume_args -@timed -def update_assets(args): - """ - Compile Sass, then collect static assets. - - This is a DEPRECATED COMPATIBILITY WRAPPER around other DEPRECATED COMPATIBILITY WRAPPERS. - The aggregate affect of this command can be achieved with this sequence of commands instead: - - * pip install -r requirements/edx/assets.txt # replaces install_python_prereqs - * npm clean-install # replaces install_node_prereqs - * npm run build # replaces execute_compile_sass and webpack - * ./manage.py lms collectstatic --noinput # replaces collect_assets (for LMS) - * ./manage.py cms collectstatic --noinput # replaces collect_assets (for CMS) - """ - parser = argparse.ArgumentParser(prog='paver update_assets') - parser.add_argument( - 'system', type=str, nargs='*', default=["lms", "studio"], - help="lms or studio", - ) - parser.add_argument( - '--settings', type=str, default=Env.DEVSTACK_SETTINGS, - help="Django settings module", - ) - parser.add_argument( - '--debug', action='store_true', default=False, - help="Enable all debugging", - ) - parser.add_argument( - '--debug-collect', action='store_true', default=False, - help="Disable collect static", - ) - parser.add_argument( - '--skip-collect', dest='collect', action='store_false', default=True, - help="Skip collection of static assets", - ) - parser.add_argument( - '--watch', action='store_true', default=False, - help="Watch files for changes", - ) - parser.add_argument( - '--theme-dirs', dest='theme_dirs', type=str, nargs='+', default=None, - help="base directories where themes are placed", - ) - parser.add_argument( - '--themes', type=str, nargs='+', default=None, - help="list of themes to compile sass for. ignored when --watch is used; all themes are watched.", - ) - parser.add_argument( - '--collect-log', dest="collect_log_dir", default=None, - help="When running collectstatic, direct output to specified log directory", - ) - parser.add_argument( - '--wait', type=float, default=0.0, - help="DEPRECATED. Watchdog's default wait time is now used.", - ) - args = parser.parse_args(args) - - # Build Webpack - call_task('pavelib.assets.webpack', options={'settings': args.settings}) - - # Compile sass for themes and system - execute_compile_sass(args) - - if args.collect: - if args.collect_log_dir: - collect_log_args = {"collect_log_dir": args.collect_log_dir} - elif args.debug or args.debug_collect: - collect_log_args = {"collect_log_dir": None} - else: - collect_log_args = {} - - collect_assets(args.system, args.settings, **collect_log_args) - - if args.watch: - call_task( - 'pavelib.assets.watch_assets', - options={ - 'background': not args.debug, - 'settings': args.settings, - 'theme_dirs': args.theme_dirs, - 'themes': args.themes, - 'wait': [float(args.wait)] - }, - ) diff --git a/pavelib/js_test.py b/pavelib/js_test.py deleted file mode 100644 index fb9c213499ac..000000000000 --- a/pavelib/js_test.py +++ /dev/null @@ -1,143 +0,0 @@ -""" -Javascript test tasks -""" - - -import os -import re -import sys - -from paver.easy import cmdopts, needs, sh, task - -from pavelib.utils.envs import Env -from pavelib.utils.test.suites import JestSnapshotTestSuite, JsTestSuite -from pavelib.utils.timer import timed - -try: - from pygments.console import colorize -except ImportError: - colorize = lambda color, text: text - -__test__ = False # do not collect - - -@task -@needs( - 'pavelib.prereqs.install_node_prereqs', - 'pavelib.utils.test.utils.clean_reports_dir', -) -@cmdopts([ - ("suite=", "s", "Test suite to run"), - ("mode=", "m", "dev or run"), - ("coverage", "c", "Run test under coverage"), - ("port=", "p", "Port to run test server on (dev mode only)"), - ('skip-clean', 'C', 'skip cleaning repository before running tests'), - ('skip_clean', None, 'deprecated in favor of skip-clean'), -], share_with=["pavelib.utils.tests.utils.clean_reports_dir"]) -@timed -def test_js(options): - """ - Run the JavaScript tests - """ - mode = getattr(options, 'mode', 'run') - port = None - skip_clean = getattr(options, 'skip_clean', False) - - if mode == 'run': - suite = getattr(options, 'suite', 'all') - coverage = getattr(options, 'coverage', False) - elif mode == 'dev': - suite = getattr(options, 'suite', None) - coverage = False - port = getattr(options, 'port', None) - else: - sys.stderr.write("Invalid mode. Please choose 'dev' or 'run'.") - return - - if (suite != 'all') and (suite not in Env.JS_TEST_ID_KEYS): - sys.stderr.write( - "Unknown test suite. Please choose from ({suites})\n".format( - suites=", ".join(Env.JS_TEST_ID_KEYS) - ) - ) - return - - if suite != 'jest-snapshot': - test_suite = JsTestSuite(suite, mode=mode, with_coverage=coverage, port=port, skip_clean=skip_clean) - test_suite.run() - - if (suite == 'jest-snapshot') or (suite == 'all'): # lint-amnesty, pylint: disable=consider-using-in - test_suite = JestSnapshotTestSuite('jest') - test_suite.run() - - -@task -@cmdopts([ - ("suite=", "s", "Test suite to run"), - ("coverage", "c", "Run test under coverage"), -]) -@timed -def test_js_run(options): - """ - Run the JavaScript tests and print results to the console - """ - options.mode = 'run' - test_js(options) - - -@task -@cmdopts([ - ("suite=", "s", "Test suite to run"), - ("port=", "p", "Port to run test server on"), -]) -@timed -def test_js_dev(options): - """ - Run the JavaScript tests in your default browsers - """ - options.mode = 'dev' - test_js(options) - - -@task -@needs('pavelib.prereqs.install_coverage_prereqs') -@cmdopts([ - ("compare-branch=", "b", "Branch to compare against, defaults to origin/master"), -], share_with=['coverage']) -@timed -def diff_coverage(options): - """ - Build the diff coverage reports - """ - compare_branch = options.get('compare_branch', 'origin/master') - - # Find all coverage XML files (both Python and JavaScript) - xml_reports = [] - - for filepath in Env.REPORT_DIR.walk(): - if bool(re.match(r'^coverage.*\.xml$', filepath.basename())): - xml_reports.append(filepath) - - if not xml_reports: - err_msg = colorize( - 'red', - "No coverage info found. Run `paver test` before running " - "`paver coverage`.\n" - ) - sys.stderr.write(err_msg) - else: - xml_report_str = ' '.join(xml_reports) - diff_html_path = os.path.join(Env.REPORT_DIR, 'diff_coverage_combined.html') - - # Generate the diff coverage reports (HTML and console) - # The --diff-range-notation parameter is a workaround for https://github.com/Bachmann1234/diff_cover/issues/153 - sh( - "diff-cover {xml_report_str} --diff-range-notation '..' --compare-branch={compare_branch} " - "--html-report {diff_html_path}".format( - xml_report_str=xml_report_str, - compare_branch=compare_branch, - diff_html_path=diff_html_path, - ) - ) - - print("\n") diff --git a/pavelib/paver_tests/__init__.py b/pavelib/paver_tests/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/pavelib/paver_tests/conftest.py b/pavelib/paver_tests/conftest.py deleted file mode 100644 index 214a35e3fe85..000000000000 --- a/pavelib/paver_tests/conftest.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -Pytest fixtures for the pavelib unit tests. -""" - - -import os -from shutil import rmtree - -import pytest - -from pavelib.utils.envs import Env - - -@pytest.fixture(autouse=True, scope='session') -def delete_quality_junit_xml(): - """ - Delete the JUnit XML results files for quality check tasks run during the - unit tests. - """ - yield - if os.path.exists(Env.QUALITY_DIR): - rmtree(Env.QUALITY_DIR, ignore_errors=True) diff --git a/pavelib/paver_tests/pylint_test_list.json b/pavelib/paver_tests/pylint_test_list.json deleted file mode 100644 index d0e8b43aa93d..000000000000 --- a/pavelib/paver_tests/pylint_test_list.json +++ /dev/null @@ -1,6 +0,0 @@ -[ - "foo/bar.py:192: [C0111(missing-docstring), Bliptv] Missing docstring", - "foo/bar/test.py:74: [C0322(no-space-before-operator)] Operator not preceded by a space", - "ugly/string/test.py:16: [C0103(invalid-name)] Invalid name \"whats up\" for type constant (should match (([A-Z_][A-Z0-9_]*)|(__.*__)|log|urlpatterns)$)", - "multiple/lines/test.py:72: [C0322(no-space-before-operator)] Operator not preceded by a space\nFOO_BAR='pipeline.storage.NonPackagingPipelineStorage'\n ^" -] diff --git a/pavelib/paver_tests/test_assets.py b/pavelib/paver_tests/test_assets.py deleted file mode 100644 index f7100a7f03c3..000000000000 --- a/pavelib/paver_tests/test_assets.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Unit tests for the Paver asset tasks.""" - -import json -import os -from pathlib import Path -from unittest import TestCase -from unittest.mock import patch - -import ddt -import paver.easy -from paver import tasks - -import pavelib.assets -from pavelib.assets import Env - - -REPO_ROOT = Path(__file__).parent.parent.parent - -LMS_SETTINGS = { - "WEBPACK_CONFIG_PATH": "webpack.fake.config.js", - "STATIC_ROOT": "/fake/lms/staticfiles", - -} -CMS_SETTINGS = { - "WEBPACK_CONFIG_PATH": "webpack.fake.config", - "STATIC_ROOT": "/fake/cms/staticfiles", - "JS_ENV_EXTRA_CONFIG": json.dumps({"key1": [True, False], "key2": {"key2.1": 1369, "key2.2": "1369"}}), -} - - -def _mock_get_django_settings(django_settings, system, settings=None): # pylint: disable=unused-argument - return [(LMS_SETTINGS if system == "lms" else CMS_SETTINGS)[s] for s in django_settings] - - -@ddt.ddt -@patch.object(Env, 'get_django_settings', _mock_get_django_settings) -@patch.object(Env, 'get_django_json_settings', _mock_get_django_settings) -class TestDeprecatedPaverAssets(TestCase): - """ - Simple test to ensure that the soon-to-be-removed Paver commands are correctly translated into the new npm-run - commands. - """ - def setUp(self): - super().setUp() - self.maxDiff = None - os.environ['NO_PREREQ_INSTALL'] = 'true' - tasks.environment = tasks.Environment() - - def tearDown(self): - super().tearDown() - del os.environ['NO_PREREQ_INSTALL'] - - @ddt.data( - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={}, - expected=["npm run compile-sass --"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"system": "lms,studio"}, - expected=["npm run compile-sass --"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"debug": True}, - expected=["npm run compile-sass-dev --"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"system": "lms"}, - expected=["npm run compile-sass -- --skip-cms"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"system": "studio"}, - expected=["npm run compile-sass -- --skip-lms"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"system": "cms", "theme_dirs": f"{REPO_ROOT}/common/test,{REPO_ROOT}/themes"}, - expected=[ - "npm run compile-sass -- --skip-lms " + - f"--theme-dir {REPO_ROOT}/common/test --theme-dir {REPO_ROOT}/themes" - ], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"theme_dirs": f"{REPO_ROOT}/common/test,{REPO_ROOT}/themes", "themes": "red-theme,test-theme"}, - expected=[ - "npm run compile-sass -- " + - f"--theme-dir {REPO_ROOT}/common/test --theme-dir {REPO_ROOT}/themes " + - "--theme red-theme --theme test-theme" - ], - ), - dict( - task_name='pavelib.assets.update_assets', - args=["lms", "studio", "--settings=fake.settings"], - kwargs={}, - expected=[ - ( - "WEBPACK_CONFIG_PATH=webpack.fake.config.js " + - "NODE_ENV=production " + - "STATIC_ROOT_LMS=/fake/lms/staticfiles " + - "STATIC_ROOT_CMS=/fake/cms/staticfiles " + - 'JS_ENV_EXTRA_CONFIG=' + - '"{\\"key1\\": [true, false], \\"key2\\": {\\"key2.1\\": 1369, \\"key2.2\\": \\"1369\\"}}" ' + - "npm run webpack" - ), - "python manage.py lms --settings=fake.settings compile_sass lms ", - "python manage.py cms --settings=fake.settings compile_sass cms ", - ( - "( ./manage.py lms --settings=fake.settings collectstatic --noinput ) && " + - "( ./manage.py cms --settings=fake.settings collectstatic --noinput )" - ), - ], - ), - ) - @ddt.unpack - @patch.object(pavelib.assets, 'sh') - def test_paver_assets_wrapper_invokes_new_commands(self, mock_sh, task_name, args, kwargs, expected): - paver.easy.call_task(task_name, args=args, options=kwargs) - assert [call_args[0] for (call_args, call_kwargs) in mock_sh.call_args_list] == expected diff --git a/pavelib/paver_tests/test_eslint.py b/pavelib/paver_tests/test_eslint.py deleted file mode 100644 index 5802d7d0d21b..000000000000 --- a/pavelib/paver_tests/test_eslint.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -Tests for Paver's Stylelint tasks. -""" - - -import unittest -from unittest.mock import patch - -import pytest -from paver.easy import BuildFailure, call_task - -import pavelib.quality - - -class TestPaverESLint(unittest.TestCase): - """ - For testing run_eslint - """ - - def setUp(self): - super().setUp() - - # Mock the paver @needs decorator - self._mock_paver_needs = patch.object(pavelib.quality.run_eslint, 'needs').start() - self._mock_paver_needs.return_value = 0 - - # Mock shell commands - patcher = patch('pavelib.quality.sh') - self._mock_paver_sh = patcher.start() - - # Cleanup mocks - self.addCleanup(patcher.stop) - self.addCleanup(self._mock_paver_needs.stop) - - @patch.object(pavelib.quality, '_write_metric') - @patch.object(pavelib.quality, '_prepare_report_dir') - @patch.object(pavelib.quality, '_get_count_from_last_line') - def test_eslint_violation_number_not_found(self, mock_count, mock_report_dir, mock_write_metric): # pylint: disable=unused-argument - """ - run_eslint encounters an error parsing the eslint output log - """ - mock_count.return_value = None - with pytest.raises(BuildFailure): - call_task('pavelib.quality.run_eslint', args=['']) - - @patch.object(pavelib.quality, '_write_metric') - @patch.object(pavelib.quality, '_prepare_report_dir') - @patch.object(pavelib.quality, '_get_count_from_last_line') - def test_eslint_vanilla(self, mock_count, mock_report_dir, mock_write_metric): # pylint: disable=unused-argument - """ - eslint finds violations, but a limit was not set - """ - mock_count.return_value = 1 - pavelib.quality.run_eslint("") diff --git a/pavelib/paver_tests/test_js_test.py b/pavelib/paver_tests/test_js_test.py deleted file mode 100644 index 4b165a156674..000000000000 --- a/pavelib/paver_tests/test_js_test.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Unit tests for the Paver JavaScript testing tasks.""" - -from unittest.mock import patch - -import ddt -from paver.easy import call_task - -import pavelib.js_test -from pavelib.utils.envs import Env - -from .utils import PaverTestCase - - -@ddt.ddt -class TestPaverJavaScriptTestTasks(PaverTestCase): - """ - Test the Paver JavaScript testing tasks. - """ - - EXPECTED_DELETE_JAVASCRIPT_REPORT_COMMAND = 'find {platform_root}/reports/javascript -type f -delete' - EXPECTED_KARMA_OPTIONS = ( - "{config_file} " - "--single-run={single_run} " - "--capture-timeout=60000 " - "--junitreportpath=" - "{platform_root}/reports/javascript/javascript_xunit-{suite}.xml " - "--browsers={browser}" - ) - EXPECTED_COVERAGE_OPTIONS = ( - ' --coverage --coveragereportpath={platform_root}/reports/javascript/coverage-{suite}.xml' - ) - - EXPECTED_COMMANDS = [ - "make report_dir", - 'git clean -fqdx test_root/logs test_root/data test_root/staticfiles test_root/uploads', - "find . -name '.git' -prune -o -name '*.pyc' -exec rm {} \\;", - 'rm -rf test_root/log/auto_screenshots/*', - "rm -rf /tmp/mako_[cl]ms", - ] - - def setUp(self): - super().setUp() - - # Mock the paver @needs decorator - self._mock_paver_needs = patch.object(pavelib.js_test.test_js, 'needs').start() - self._mock_paver_needs.return_value = 0 - - # Cleanup mocks - self.addCleanup(self._mock_paver_needs.stop) - - @ddt.data( - [""], - ["--coverage"], - ["--suite=lms"], - ["--suite=lms --coverage"], - ) - @ddt.unpack - def test_test_js_run(self, options_string): - """ - Test the "test_js_run" task. - """ - options = self.parse_options_string(options_string) - self.reset_task_messages() - call_task("pavelib.js_test.test_js_run", options=options) - self.verify_messages(options=options, dev_mode=False) - - @ddt.data( - [""], - ["--port=9999"], - ["--suite=lms"], - ["--suite=lms --port=9999"], - ) - @ddt.unpack - def test_test_js_dev(self, options_string): - """ - Test the "test_js_run" task. - """ - options = self.parse_options_string(options_string) - self.reset_task_messages() - call_task("pavelib.js_test.test_js_dev", options=options) - self.verify_messages(options=options, dev_mode=True) - - def parse_options_string(self, options_string): - """ - Parse a string containing the options for a test run - """ - parameters = options_string.split(" ") - suite = "all" - if "--system=lms" in parameters: - suite = "lms" - elif "--system=common" in parameters: - suite = "common" - coverage = "--coverage" in parameters - port = None - if "--port=9999" in parameters: - port = 9999 - return { - "suite": suite, - "coverage": coverage, - "port": port, - } - - def verify_messages(self, options, dev_mode): - """ - Verify that the messages generated when running tests are as expected - for the specified options and dev_mode. - """ - is_coverage = options['coverage'] - port = options['port'] - expected_messages = [] - suites = Env.JS_TEST_ID_KEYS if options['suite'] == 'all' else [options['suite']] - - expected_messages.extend(self.EXPECTED_COMMANDS) - if not dev_mode and not is_coverage: - expected_messages.append(self.EXPECTED_DELETE_JAVASCRIPT_REPORT_COMMAND.format( - platform_root=self.platform_root - )) - - command_template = ( - 'node --max_old_space_size=4096 node_modules/.bin/karma start {options}' - ) - - for suite in suites: - # Karma test command - if suite != 'jest-snapshot': - karma_config_file = Env.KARMA_CONFIG_FILES[Env.JS_TEST_ID_KEYS.index(suite)] - expected_test_tool_command = command_template.format( - options=self.EXPECTED_KARMA_OPTIONS.format( - config_file=karma_config_file, - single_run='false' if dev_mode else 'true', - suite=suite, - platform_root=self.platform_root, - browser=Env.KARMA_BROWSER, - ), - ) - if is_coverage: - expected_test_tool_command += self.EXPECTED_COVERAGE_OPTIONS.format( - platform_root=self.platform_root, - suite=suite - ) - if port: - expected_test_tool_command += f" --port={port}" - else: - expected_test_tool_command = 'jest' - - expected_messages.append(expected_test_tool_command) - - assert self.task_messages == expected_messages diff --git a/pavelib/paver_tests/test_paver_quality.py b/pavelib/paver_tests/test_paver_quality.py deleted file mode 100644 index 36d6dd59e172..000000000000 --- a/pavelib/paver_tests/test_paver_quality.py +++ /dev/null @@ -1,156 +0,0 @@ -""" # lint-amnesty, pylint: disable=django-not-configured -Tests for paver quality tasks -""" - - -import os -import shutil # lint-amnesty, pylint: disable=unused-import -import tempfile -import textwrap -import unittest -from unittest.mock import MagicMock, mock_open, patch # lint-amnesty, pylint: disable=unused-import - -import pytest # lint-amnesty, pylint: disable=unused-import -from ddt import data, ddt, file_data, unpack # lint-amnesty, pylint: disable=unused-import -from path import Path as path -from paver.easy import BuildFailure # lint-amnesty, pylint: disable=unused-import - -import pavelib.quality -from pavelib.paver_tests.utils import PaverTestCase, fail_on_eslint # lint-amnesty, pylint: disable=unused-import - -OPEN_BUILTIN = 'builtins.open' - - -@ddt -class TestPaverQualityViolations(unittest.TestCase): - """ - For testing the paver violations-counting tasks - """ - def setUp(self): - super().setUp() - self.f = tempfile.NamedTemporaryFile(delete=False) # lint-amnesty, pylint: disable=consider-using-with - self.f.close() - self.addCleanup(os.remove, self.f.name) - - def test_pep8_parser(self): - with open(self.f.name, 'w') as f: - f.write("hello\nhithere") - num = len(pavelib.quality._pep8_violations(f.name)) # pylint: disable=protected-access - assert num == 2 - - -class TestPaverReportViolationsCounts(unittest.TestCase): - """ - For testing utility functions for getting counts from reports for - run_eslint and run_xsslint. - """ - - def setUp(self): - super().setUp() - - # Temporary file infrastructure - self.f = tempfile.NamedTemporaryFile(delete=False) # lint-amnesty, pylint: disable=consider-using-with - self.f.close() - - # Cleanup various mocks and tempfiles - self.addCleanup(os.remove, self.f.name) - - def test_get_eslint_violations_count(self): - with open(self.f.name, 'w') as f: - f.write("3000 violations found") - actual_count = pavelib.quality._get_count_from_last_line(self.f.name, "eslint") # pylint: disable=protected-access - assert actual_count == 3000 - - def test_get_eslint_violations_no_number_found(self): - with open(self.f.name, 'w') as f: - f.write("Not expected string regex") - actual_count = pavelib.quality._get_count_from_last_line(self.f.name, "eslint") # pylint: disable=protected-access - assert actual_count is None - - def test_get_eslint_violations_count_truncated_report(self): - """ - A truncated report (i.e. last line is just a violation) - """ - with open(self.f.name, 'w') as f: - f.write("foo/bar/js/fizzbuzz.js: line 45, col 59, Missing semicolon.") - actual_count = pavelib.quality._get_count_from_last_line(self.f.name, "eslint") # pylint: disable=protected-access - assert actual_count is None - - def test_generic_value(self): - """ - Default behavior is to look for an integer appearing at head of line - """ - with open(self.f.name, 'w') as f: - f.write("5.777 good to see you") - actual_count = pavelib.quality._get_count_from_last_line(self.f.name, "foo") # pylint: disable=protected-access - assert actual_count == 5 - - def test_generic_value_none_found(self): - """ - Default behavior is to look for an integer appearing at head of line - """ - with open(self.f.name, 'w') as f: - f.write("hello 5.777 good to see you") - actual_count = pavelib.quality._get_count_from_last_line(self.f.name, "foo") # pylint: disable=protected-access - assert actual_count is None - - def test_get_xsslint_counts_happy(self): - """ - Test happy path getting violation counts from xsslint report. - """ - report = textwrap.dedent(""" - test.html: 30:53: javascript-jquery-append: $('#test').append(print_tos); - - javascript-concat-html: 310 violations - javascript-escape: 7 violations - - 2608 violations total - """) - with open(self.f.name, 'w') as f: - f.write(report) - counts = pavelib.quality._get_xsslint_counts(self.f.name) # pylint: disable=protected-access - self.assertDictEqual(counts, { - 'rules': { - 'javascript-concat-html': 310, - 'javascript-escape': 7, - }, - 'total': 2608, - }) - - def test_get_xsslint_counts_bad_counts(self): - """ - Test getting violation counts from truncated and malformed xsslint - report. - """ - report = textwrap.dedent(""" - javascript-concat-html: violations - """) - with open(self.f.name, 'w') as f: - f.write(report) - counts = pavelib.quality._get_xsslint_counts(self.f.name) # pylint: disable=protected-access - self.assertDictEqual(counts, { - 'rules': {}, - 'total': None, - }) - - -class TestPrepareReportDir(unittest.TestCase): - """ - Tests the report directory preparation - """ - - def setUp(self): - super().setUp() - self.test_dir = tempfile.mkdtemp() - self.test_file = tempfile.NamedTemporaryFile(delete=False, dir=self.test_dir) # lint-amnesty, pylint: disable=consider-using-with - self.addCleanup(os.removedirs, self.test_dir) - - def test_report_dir_with_files(self): - assert os.path.exists(self.test_file.name) - pavelib.quality._prepare_report_dir(path(self.test_dir)) # pylint: disable=protected-access - assert not os.path.exists(self.test_file.name) - - def test_report_dir_without_files(self): - os.remove(self.test_file.name) - pavelib.quality._prepare_report_dir(path(self.test_dir)) # pylint: disable=protected-access - assert os.listdir(path(self.test_dir)) == [] diff --git a/pavelib/paver_tests/test_pii_check.py b/pavelib/paver_tests/test_pii_check.py deleted file mode 100644 index d034360acde0..000000000000 --- a/pavelib/paver_tests/test_pii_check.py +++ /dev/null @@ -1,79 +0,0 @@ -""" -Tests for Paver's PII checker task. -""" - -import shutil -import tempfile -import unittest -from unittest.mock import patch - -from path import Path as path -from paver.easy import call_task, BuildFailure - -import pavelib.quality -from pavelib.utils.envs import Env - - -class TestPaverPIICheck(unittest.TestCase): - """ - For testing the paver run_pii_check task - """ - def setUp(self): - super().setUp() - self.report_dir = path(tempfile.mkdtemp()) - self.addCleanup(shutil.rmtree, self.report_dir) - - @patch.object(pavelib.quality.run_pii_check, 'needs') - @patch('pavelib.quality.sh') - def test_pii_check_report_dir_override(self, mock_paver_sh, mock_needs): - """ - run_pii_check succeeds with proper report dir - """ - # Make the expected stdout files. - cms_stdout_report = self.report_dir / 'pii_check_cms.report' - cms_stdout_report.write_lines(['Coverage found 33 uncovered models:\n']) - lms_stdout_report = self.report_dir / 'pii_check_lms.report' - lms_stdout_report.write_lines(['Coverage found 66 uncovered models:\n']) - - mock_needs.return_value = 0 - call_task('pavelib.quality.run_pii_check', options={"report_dir": str(self.report_dir)}) - mock_calls = [str(call) for call in mock_paver_sh.mock_calls] - assert len(mock_calls) == 2 - assert any('lms.envs.test' in call for call in mock_calls) - assert any('cms.envs.test' in call for call in mock_calls) - assert all(str(self.report_dir) in call for call in mock_calls) - metrics_file = Env.METRICS_DIR / 'pii' - assert open(metrics_file).read() == 'Number of PII Annotation violations: 66\n' - - @patch.object(pavelib.quality.run_pii_check, 'needs') - @patch('pavelib.quality.sh') - def test_pii_check_failed(self, mock_paver_sh, mock_needs): - """ - run_pii_check fails due to crossing the threshold. - """ - # Make the expected stdout files. - cms_stdout_report = self.report_dir / 'pii_check_cms.report' - cms_stdout_report.write_lines(['Coverage found 33 uncovered models:\n']) - lms_stdout_report = self.report_dir / 'pii_check_lms.report' - lms_stdout_report.write_lines([ - 'Coverage found 66 uncovered models:', - 'Coverage threshold not met! Needed 100.0, actually 95.0!', - ]) - - mock_needs.return_value = 0 - try: - with self.assertRaises(BuildFailure): - call_task('pavelib.quality.run_pii_check', options={"report_dir": str(self.report_dir)}) - except SystemExit: - # Sometimes the BuildFailure raises a SystemExit, sometimes it doesn't, not sure why. - # As a hack, we just wrap it in try-except. - # This is not good, but these tests weren't even running for years, and we're removing this whole test - # suite soon anyway. - pass - mock_calls = [str(call) for call in mock_paver_sh.mock_calls] - assert len(mock_calls) == 2 - assert any('lms.envs.test' in call for call in mock_calls) - assert any('cms.envs.test' in call for call in mock_calls) - assert all(str(self.report_dir) in call for call in mock_calls) - metrics_file = Env.METRICS_DIR / 'pii' - assert open(metrics_file).read() == 'Number of PII Annotation violations: 66\n' diff --git a/pavelib/paver_tests/test_stylelint.py b/pavelib/paver_tests/test_stylelint.py deleted file mode 100644 index 3e1c79c93f28..000000000000 --- a/pavelib/paver_tests/test_stylelint.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -Tests for Paver's Stylelint tasks. -""" - -from unittest.mock import MagicMock, patch - -import pytest -import ddt -from paver.easy import call_task - -from .utils import PaverTestCase - - -@ddt.ddt -class TestPaverStylelint(PaverTestCase): - """ - Tests for Paver's Stylelint tasks. - """ - @ddt.data( - [False], - [True], - ) - @ddt.unpack - def test_run_stylelint(self, should_pass): - """ - Verify that the quality task fails with Stylelint violations. - """ - if should_pass: - _mock_stylelint_violations = MagicMock(return_value=0) - with patch('pavelib.quality._get_stylelint_violations', _mock_stylelint_violations): - call_task('pavelib.quality.run_stylelint') - else: - _mock_stylelint_violations = MagicMock(return_value=100) - with patch('pavelib.quality._get_stylelint_violations', _mock_stylelint_violations): - with pytest.raises(SystemExit): - call_task('pavelib.quality.run_stylelint') diff --git a/pavelib/paver_tests/test_timer.py b/pavelib/paver_tests/test_timer.py deleted file mode 100644 index bc9817668347..000000000000 --- a/pavelib/paver_tests/test_timer.py +++ /dev/null @@ -1,168 +0,0 @@ -""" -Tests of the pavelib.utils.timer module. -""" - - -from datetime import datetime, timedelta -from unittest import TestCase - -from unittest.mock import MagicMock, patch - -from pavelib.utils import timer - - -@timer.timed -def identity(*args, **kwargs): - """ - An identity function used as a default task to test the timing of. - """ - return args, kwargs - - -MOCK_OPEN = MagicMock(spec=open) - - -@patch.dict('pavelib.utils.timer.__builtins__', open=MOCK_OPEN) -class TimedDecoratorTests(TestCase): - """ - Tests of the pavelib.utils.timer:timed decorator. - """ - def setUp(self): - super().setUp() - - patch_dumps = patch.object(timer.json, 'dump', autospec=True) - self.mock_dump = patch_dumps.start() - self.addCleanup(patch_dumps.stop) - - patch_makedirs = patch.object(timer.os, 'makedirs', autospec=True) - self.mock_makedirs = patch_makedirs.start() - self.addCleanup(patch_makedirs.stop) - - patch_datetime = patch.object(timer, 'datetime', autospec=True) - self.mock_datetime = patch_datetime.start() - self.addCleanup(patch_datetime.stop) - - patch_exists = patch.object(timer, 'exists', autospec=True) - self.mock_exists = patch_exists.start() - self.addCleanup(patch_exists.stop) - - MOCK_OPEN.reset_mock() - - def get_log_messages(self, task=identity, args=None, kwargs=None, raises=None): - """ - Return all timing messages recorded during the execution of ``task``. - """ - if args is None: - args = [] - if kwargs is None: - kwargs = {} - - if raises is None: - task(*args, **kwargs) - else: - self.assertRaises(raises, task, *args, **kwargs) - - return [ - call[0][0] # log_message - for call in self.mock_dump.call_args_list - ] - - @patch.object(timer, 'PAVER_TIMER_LOG', '/tmp/some-log') - def test_times(self): - start = datetime(2016, 7, 20, 10, 56, 19) - end = start + timedelta(seconds=35.6) - - self.mock_datetime.utcnow.side_effect = [start, end] - - messages = self.get_log_messages() - assert len(messages) == 1 - - assert 'duration' in messages[0] and messages[0]['duration'] == 35.6 - assert 'started_at' in messages[0] and messages[0]['started_at'] == start.isoformat(' ') - assert 'ended_at' in messages[0] and messages[0]['ended_at'] == end.isoformat(' ') - - @patch.object(timer, 'PAVER_TIMER_LOG', None) - def test_no_logs(self): - messages = self.get_log_messages() - assert len(messages) == 0 - - @patch.object(timer, 'PAVER_TIMER_LOG', '/tmp/some-log') - def test_arguments(self): - messages = self.get_log_messages(args=(1, 'foo'), kwargs=dict(bar='baz')) - assert len(messages) == 1 - - assert 'args' in messages[0] and messages[0]['args'] == [repr(1), repr('foo')] - assert 'kwargs' in messages[0] and messages[0]['kwargs'] == {'bar': repr('baz')} - - @patch.object(timer, 'PAVER_TIMER_LOG', '/tmp/some-log') - def test_task_name(self): - messages = self.get_log_messages() - assert len(messages) == 1 - - assert 'task' in messages[0] and messages[0]['task'] == 'pavelib.paver_tests.test_timer.identity' - - @patch.object(timer, 'PAVER_TIMER_LOG', '/tmp/some-log') - def test_exceptions(self): - @timer.timed - def raises(): - """ - A task used for testing exception handling of the timed decorator. - """ - raise Exception('The Message!') - - messages = self.get_log_messages(task=raises, raises=Exception) - assert len(messages) == 1 - - assert 'exception' in messages[0] and messages[0]['exception'] == 'Exception: The Message!' - - @patch.object(timer, 'PAVER_TIMER_LOG', '/tmp/some-log-%Y-%m-%d-%H-%M-%S.log') - def test_date_formatting(self): - start = datetime(2016, 7, 20, 10, 56, 19) - end = start + timedelta(seconds=35.6) - - self.mock_datetime.utcnow.side_effect = [start, end] - - messages = self.get_log_messages() - assert len(messages) == 1 - - MOCK_OPEN.assert_called_once_with('/tmp/some-log-2016-07-20-10-56-19.log', 'a') - - @patch.object(timer, 'PAVER_TIMER_LOG', '/tmp/some-log') - def test_nested_tasks(self): - - @timer.timed - def parent(): - """ - A timed task that calls another task - """ - identity() - - parent_start = datetime(2016, 7, 20, 10, 56, 19) - parent_end = parent_start + timedelta(seconds=60) - child_start = parent_start + timedelta(seconds=10) - child_end = parent_end - timedelta(seconds=10) - - self.mock_datetime.utcnow.side_effect = [parent_start, child_start, child_end, parent_end] - - messages = self.get_log_messages(task=parent) - assert len(messages) == 2 - - # Child messages first - assert 'duration' in messages[0] - assert 40 == messages[0]['duration'] - - assert 'started_at' in messages[0] - assert child_start.isoformat(' ') == messages[0]['started_at'] - - assert 'ended_at' in messages[0] - assert child_end.isoformat(' ') == messages[0]['ended_at'] - - # Parent messages after - assert 'duration' in messages[1] - assert 60 == messages[1]['duration'] - - assert 'started_at' in messages[1] - assert parent_start.isoformat(' ') == messages[1]['started_at'] - - assert 'ended_at' in messages[1] - assert parent_end.isoformat(' ') == messages[1]['ended_at'] diff --git a/pavelib/paver_tests/test_xsslint.py b/pavelib/paver_tests/test_xsslint.py deleted file mode 100644 index a9b4a41e1600..000000000000 --- a/pavelib/paver_tests/test_xsslint.py +++ /dev/null @@ -1,120 +0,0 @@ -""" -Tests for paver xsslint quality tasks -""" -from unittest.mock import patch - -import pytest -from paver.easy import call_task - -import pavelib.quality - -from .utils import PaverTestCase - - -class PaverXSSLintTest(PaverTestCase): - """ - Test run_xsslint with a mocked environment in order to pass in opts - """ - - def setUp(self): - super().setUp() - self.reset_task_messages() - - @patch.object(pavelib.quality, '_write_metric') - @patch.object(pavelib.quality, '_prepare_report_dir') - @patch.object(pavelib.quality, '_get_xsslint_counts') - def test_xsslint_violation_number_not_found(self, _mock_counts, _mock_report_dir, _mock_write_metric): - """ - run_xsslint encounters an error parsing the xsslint output log - """ - _mock_counts.return_value = {} - with pytest.raises(SystemExit): - call_task('pavelib.quality.run_xsslint') - - @patch.object(pavelib.quality, '_write_metric') - @patch.object(pavelib.quality, '_prepare_report_dir') - @patch.object(pavelib.quality, '_get_xsslint_counts') - def test_xsslint_vanilla(self, _mock_counts, _mock_report_dir, _mock_write_metric): - """ - run_xsslint finds violations, but a limit was not set - """ - _mock_counts.return_value = {'total': 0} - call_task('pavelib.quality.run_xsslint') - - @patch.object(pavelib.quality, '_write_metric') - @patch.object(pavelib.quality, '_prepare_report_dir') - @patch.object(pavelib.quality, '_get_xsslint_counts') - def test_xsslint_invalid_thresholds_option(self, _mock_counts, _mock_report_dir, _mock_write_metric): - """ - run_xsslint fails when thresholds option is poorly formatted - """ - _mock_counts.return_value = {'total': 0} - with pytest.raises(SystemExit): - call_task('pavelib.quality.run_xsslint', options={"thresholds": "invalid"}) - - @patch.object(pavelib.quality, '_write_metric') - @patch.object(pavelib.quality, '_prepare_report_dir') - @patch.object(pavelib.quality, '_get_xsslint_counts') - def test_xsslint_invalid_thresholds_option_key(self, _mock_counts, _mock_report_dir, _mock_write_metric): - """ - run_xsslint fails when thresholds option is poorly formatted - """ - _mock_counts.return_value = {'total': 0} - with pytest.raises(SystemExit): - call_task('pavelib.quality.run_xsslint', options={"thresholds": '{"invalid": 3}'}) - - @patch.object(pavelib.quality, '_write_metric') - @patch.object(pavelib.quality, '_prepare_report_dir') - @patch.object(pavelib.quality, '_get_xsslint_counts') - def test_xsslint_too_many_violations(self, _mock_counts, _mock_report_dir, _mock_write_metric): - """ - run_xsslint finds more violations than are allowed - """ - _mock_counts.return_value = {'total': 4} - with pytest.raises(SystemExit): - call_task('pavelib.quality.run_xsslint', options={"thresholds": '{"total": 3}'}) - - @patch.object(pavelib.quality, '_write_metric') - @patch.object(pavelib.quality, '_prepare_report_dir') - @patch.object(pavelib.quality, '_get_xsslint_counts') - def test_xsslint_under_limit(self, _mock_counts, _mock_report_dir, _mock_write_metric): - """ - run_xsslint finds fewer violations than are allowed - """ - _mock_counts.return_value = {'total': 4} - # No System Exit is expected - call_task('pavelib.quality.run_xsslint', options={"thresholds": '{"total": 5}'}) - - @patch.object(pavelib.quality, '_write_metric') - @patch.object(pavelib.quality, '_prepare_report_dir') - @patch.object(pavelib.quality, '_get_xsslint_counts') - def test_xsslint_rule_violation_number_not_found(self, _mock_counts, _mock_report_dir, _mock_write_metric): - """ - run_xsslint encounters an error parsing the xsslint output log for a - given rule threshold that was set. - """ - _mock_counts.return_value = {'total': 4} - with pytest.raises(SystemExit): - call_task('pavelib.quality.run_xsslint', options={"thresholds": '{"rules": {"javascript-escape": 3}}'}) - - @patch.object(pavelib.quality, '_write_metric') - @patch.object(pavelib.quality, '_prepare_report_dir') - @patch.object(pavelib.quality, '_get_xsslint_counts') - def test_xsslint_too_many_rule_violations(self, _mock_counts, _mock_report_dir, _mock_write_metric): - """ - run_xsslint finds more rule violations than are allowed - """ - _mock_counts.return_value = {'total': 4, 'rules': {'javascript-escape': 4}} - with pytest.raises(SystemExit): - call_task('pavelib.quality.run_xsslint', options={"thresholds": '{"rules": {"javascript-escape": 3}}'}) - - @patch.object(pavelib.quality, '_write_metric') - @patch.object(pavelib.quality, '_prepare_report_dir') - @patch.object(pavelib.quality, '_get_xsslint_counts') - def test_xsslint_under_rule_limit(self, _mock_counts, _mock_report_dir, _mock_write_metric): - """ - run_xsslint finds fewer rule violations than are allowed - """ - _mock_counts.return_value = {'total': 4, 'rules': {'javascript-escape': 4}} - # No System Exit is expected - call_task('pavelib.quality.run_xsslint', options={"thresholds": '{"rules": {"javascript-escape": 5}}'}) diff --git a/pavelib/paver_tests/utils.py b/pavelib/paver_tests/utils.py deleted file mode 100644 index 1db26cf76a4c..000000000000 --- a/pavelib/paver_tests/utils.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Unit tests for the Paver server tasks.""" - - -import os -from unittest import TestCase -from uuid import uuid4 - -from paver import tasks -from paver.easy import BuildFailure - - -class PaverTestCase(TestCase): - """ - Base class for Paver test cases. - """ - def setUp(self): - super().setUp() - - # Show full length diffs upon test failure - self.maxDiff = None # pylint: disable=invalid-name - - # Create a mock Paver environment - tasks.environment = MockEnvironment() - - # Don't run pre-reqs - os.environ['NO_PREREQ_INSTALL'] = 'true' - - def tearDown(self): - super().tearDown() - tasks.environment = tasks.Environment() - del os.environ['NO_PREREQ_INSTALL'] - - @property - def task_messages(self): - """Returns the messages output by the Paver task.""" - return tasks.environment.messages - - @property - def platform_root(self): - """Returns the current platform's root directory.""" - return os.getcwd() - - def reset_task_messages(self): - """Clear the recorded message""" - tasks.environment.messages = [] - - -class MockEnvironment(tasks.Environment): - """ - Mock environment that collects information about Paver commands. - """ - def __init__(self): - super().__init__() - self.dry_run = True - self.messages = [] - - def info(self, message, *args): - """Capture any messages that have been recorded""" - if args: - output = message % args - else: - output = message - if not output.startswith("--->"): - self.messages.append(str(output)) - - -def fail_on_eslint(*args, **kwargs): - """ - For our tests, we need the call for diff-quality running eslint reports - to fail, since that is what is going to fail when we pass in a - percentage ("p") requirement. - """ - if "eslint" in args[0]: # lint-amnesty, pylint: disable=no-else-raise - raise BuildFailure('Subprocess return code: 1') - else: - if kwargs.get('capture', False): - return uuid4().hex - else: - return - - -def fail_on_npm_install(): - """ - Used to simulate an error when executing "npm install" - """ - return 1 - - -def unexpected_fail_on_npm_install(*args, **kwargs): # pylint: disable=unused-argument - """ - For our tests, we need the call for diff-quality running pycodestyle reports to fail, since that is what - is going to fail when we pass in a percentage ("p") requirement. - """ - if ["npm", "install", "--verbose"] == args[0]: # lint-amnesty, pylint: disable=no-else-raise - raise BuildFailure('Subprocess return code: 50') - else: - return diff --git a/pavelib/prereqs.py b/pavelib/prereqs.py deleted file mode 100644 index 4453176c94da..000000000000 --- a/pavelib/prereqs.py +++ /dev/null @@ -1,351 +0,0 @@ -""" -Install Python and Node prerequisites. -""" - - -import hashlib -import os -import re -import subprocess -import sys -from distutils import sysconfig # pylint: disable=deprecated-module - -from paver.easy import sh, task # lint-amnesty, pylint: disable=unused-import - -from .utils.envs import Env -from .utils.timer import timed - -PREREQS_STATE_DIR = os.getenv('PREREQ_CACHE_DIR', Env.REPO_ROOT / '.prereqs_cache') -NO_PREREQ_MESSAGE = "NO_PREREQ_INSTALL is set, not installing prereqs" -NO_PYTHON_UNINSTALL_MESSAGE = 'NO_PYTHON_UNINSTALL is set. No attempts will be made to uninstall old Python libs.' -COVERAGE_REQ_FILE = 'requirements/edx/coverage.txt' - -# If you make any changes to this list you also need to make -# a corresponding change to circle.yml, which is how the python -# prerequisites are installed for builds on circleci.com -toxenv = os.environ.get('TOXENV') -if toxenv and toxenv != 'quality': - PYTHON_REQ_FILES = ['requirements/edx/testing.txt'] -else: - PYTHON_REQ_FILES = ['requirements/edx/development.txt'] - -# Developers can have private requirements, for local copies of github repos, -# or favorite debugging tools, etc. -PRIVATE_REQS = 'requirements/edx/private.txt' -if os.path.exists(PRIVATE_REQS): - PYTHON_REQ_FILES.append(PRIVATE_REQS) - - -def str2bool(s): - s = str(s) - return s.lower() in ('yes', 'true', 't', '1') - - -def no_prereq_install(): - """ - Determine if NO_PREREQ_INSTALL should be truthy or falsy. - """ - return str2bool(os.environ.get('NO_PREREQ_INSTALL', 'False')) - - -def no_python_uninstall(): - """ Determine if we should run the uninstall_python_packages task. """ - return str2bool(os.environ.get('NO_PYTHON_UNINSTALL', 'False')) - - -def create_prereqs_cache_dir(): - """Create the directory for storing the hashes, if it doesn't exist already.""" - try: - os.makedirs(PREREQS_STATE_DIR) - except OSError: - if not os.path.isdir(PREREQS_STATE_DIR): - raise - - -def compute_fingerprint(path_list): - """ - Hash the contents of all the files and directories in `path_list`. - Returns the hex digest. - """ - - hasher = hashlib.sha1() - - for path_item in path_list: - - # For directories, create a hash based on the modification times - # of first-level subdirectories - if os.path.isdir(path_item): - for dirname in sorted(os.listdir(path_item)): - path_name = os.path.join(path_item, dirname) - if os.path.isdir(path_name): - hasher.update(str(os.stat(path_name).st_mtime).encode('utf-8')) - - # For files, hash the contents of the file - if os.path.isfile(path_item): - with open(path_item, "rb") as file_handle: - hasher.update(file_handle.read()) - - return hasher.hexdigest() - - -def prereq_cache(cache_name, paths, install_func): - """ - Conditionally execute `install_func()` only if the files/directories - specified by `paths` have changed. - - If the code executes successfully (no exceptions are thrown), the cache - is updated with the new hash. - """ - # Retrieve the old hash - cache_filename = cache_name.replace(" ", "_") - cache_file_path = os.path.join(PREREQS_STATE_DIR, f"{cache_filename}.sha1") - old_hash = None - if os.path.isfile(cache_file_path): - with open(cache_file_path) as cache_file: - old_hash = cache_file.read() - - # Compare the old hash to the new hash - # If they do not match (either the cache hasn't been created, or the files have changed), - # then execute the code within the block. - new_hash = compute_fingerprint(paths) - if new_hash != old_hash: - install_func() - - # Update the cache with the new hash - # If the code executed within the context fails (throws an exception), - # then this step won't get executed. - create_prereqs_cache_dir() - with open(cache_file_path, "wb") as cache_file: - # Since the pip requirement files are modified during the install - # process, we need to store the hash generated AFTER the installation - post_install_hash = compute_fingerprint(paths) - cache_file.write(post_install_hash.encode('utf-8')) - else: - print(f'{cache_name} unchanged, skipping...') - - -def node_prereqs_installation(): - """ - Configures npm and installs Node prerequisites - """ - # Before July 2023, these directories were created and written to - # as root. Afterwards, they are created as being owned by the - # `app` user -- but also need to be deleted by that user (due to - # how npm runs post-install scripts.) Developers with an older - # devstack installation who are reprovisioning will see errors - # here if the files are still owned by root. Deleting the files in - # advance prevents this error. - # - # This hack should probably be left in place for at least a year. - # See ADR 17 for more background on the transition. - sh("rm -rf common/static/common/js/vendor/ common/static/common/css/vendor/") - # At the time of this writing, the js dir has git-versioned files - # but the css dir does not, so the latter would have been created - # as root-owned (in the process of creating the vendor - # subdirectory). Delete it only if empty, just in case - # git-versioned files are added later. - sh("rmdir common/static/common/css || true") - - # NPM installs hang sporadically. Log the installation process so that we - # determine if any packages are chronic offenders. - npm_log_file_path = f'{Env.GEN_LOG_DIR}/npm-install.log' - npm_log_file = open(npm_log_file_path, 'wb') # lint-amnesty, pylint: disable=consider-using-with - npm_command = 'npm ci --verbose'.split() - - # The implementation of Paver's `sh` function returns before the forked - # actually returns. Using a Popen object so that we can ensure that - # the forked process has returned - proc = subprocess.Popen(npm_command, stderr=npm_log_file) # lint-amnesty, pylint: disable=consider-using-with - retcode = proc.wait() - if retcode == 1: - raise Exception(f"npm install failed: See {npm_log_file_path}") - print("Successfully clean-installed NPM packages. Log found at {}".format( - npm_log_file_path - )) - - -def python_prereqs_installation(): - """ - Installs Python prerequisites - """ - # edx-platform installs some Python projects from within the edx-platform repo itself. - sh("pip install -e .") - for req_file in PYTHON_REQ_FILES: - pip_install_req_file(req_file) - - -def pip_install_req_file(req_file): - """Pip install the requirements file.""" - pip_cmd = 'pip install -q --disable-pip-version-check --exists-action w' - sh(f"{pip_cmd} -r {req_file}") - - -@task -@timed -def install_node_prereqs(): - """ - Installs Node prerequisites - """ - if no_prereq_install(): - print(NO_PREREQ_MESSAGE) - return - - prereq_cache("Node prereqs", ["package.json", "package-lock.json"], node_prereqs_installation) - - -# To add a package to the uninstall list, just add it to this list! No need -# to touch any other part of this file. -PACKAGES_TO_UNINSTALL = [ - "MySQL-python", # Because mysqlclient shares the same directory name - "South", # Because it interferes with Django 1.8 migrations. - "edxval", # Because it was bork-installed somehow. - "django-storages", - "django-oauth2-provider", # Because now it's called edx-django-oauth2-provider. - "edx-oauth2-provider", # Because it moved from github to pypi - "enum34", # Because enum34 is not needed in python>3.4 - "i18n-tools", # Because now it's called edx-i18n-tools - "moto", # Because we no longer use it and it conflicts with recent jsondiff versions - "python-saml", # Because python3-saml shares the same directory name - "pytest-faulthandler", # Because it was bundled into pytest - "djangorestframework-jwt", # Because now its called drf-jwt. -] - - -@task -@timed -def uninstall_python_packages(): - """ - Uninstall Python packages that need explicit uninstallation. - - Some Python packages that we no longer want need to be explicitly - uninstalled, notably, South. Some other packages were once installed in - ways that were resistant to being upgraded, like edxval. Also uninstall - them. - """ - - if no_python_uninstall(): - print(NO_PYTHON_UNINSTALL_MESSAGE) - return - - # So that we don't constantly uninstall things, use a hash of the packages - # to be uninstalled. Check it, and skip this if we're up to date. - hasher = hashlib.sha1() - hasher.update(repr(PACKAGES_TO_UNINSTALL).encode('utf-8')) - expected_version = hasher.hexdigest() - state_file_path = os.path.join(PREREQS_STATE_DIR, "Python_uninstall.sha1") - create_prereqs_cache_dir() - - if os.path.isfile(state_file_path): - with open(state_file_path) as state_file: - version = state_file.read() - if version == expected_version: - print('Python uninstalls unchanged, skipping...') - return - - # Run pip to find the packages we need to get rid of. Believe it or not, - # edx-val is installed in a way that it is present twice, so we have a loop - # to really really get rid of it. - for _ in range(3): - uninstalled = False - frozen = sh("pip freeze", capture=True) - - for package_name in PACKAGES_TO_UNINSTALL: - if package_in_frozen(package_name, frozen): - # Uninstall the pacakge - sh(f"pip uninstall --disable-pip-version-check -y {package_name}") - uninstalled = True - if not uninstalled: - break - else: - # We tried three times and didn't manage to get rid of the pests. - print("Couldn't uninstall unwanted Python packages!") - return - - # Write our version. - with open(state_file_path, "wb") as state_file: - state_file.write(expected_version.encode('utf-8')) - - -def package_in_frozen(package_name, frozen_output): - """Is this package in the output of 'pip freeze'?""" - # Look for either: - # - # PACKAGE-NAME== - # - # or: - # - # blah_blah#egg=package_name-version - # - pattern = r"(?mi)^{pkg}==|#egg={pkg_under}-".format( - pkg=re.escape(package_name), - pkg_under=re.escape(package_name.replace("-", "_")), - ) - return bool(re.search(pattern, frozen_output)) - - -@task -@timed -def install_coverage_prereqs(): - """ Install python prereqs for measuring coverage. """ - if no_prereq_install(): - print(NO_PREREQ_MESSAGE) - return - pip_install_req_file(COVERAGE_REQ_FILE) - - -@task -@timed -def install_python_prereqs(): - """ - Installs Python prerequisites. - """ - if no_prereq_install(): - print(NO_PREREQ_MESSAGE) - return - - uninstall_python_packages() - - # Include all of the requirements files in the fingerprint. - files_to_fingerprint = list(PYTHON_REQ_FILES) - - # Also fingerprint the directories where packages get installed: - # ("/edx/app/edxapp/venvs/edxapp/lib/python2.7/site-packages") - files_to_fingerprint.append(sysconfig.get_python_lib()) - - # In a virtualenv, "-e installs" get put in a src directory. - if Env.PIP_SRC: - src_dir = Env.PIP_SRC - else: - src_dir = os.path.join(sys.prefix, "src") - if os.path.isdir(src_dir): - files_to_fingerprint.append(src_dir) - - # Also fingerprint this source file, so that if the logic for installations - # changes, we will redo the installation. - this_file = __file__ - if this_file.endswith(".pyc"): - this_file = this_file[:-1] # use the .py file instead of the .pyc - files_to_fingerprint.append(this_file) - - prereq_cache("Python prereqs", files_to_fingerprint, python_prereqs_installation) - - -@task -@timed -def install_prereqs(): - """ - Installs Node and Python prerequisites - """ - if no_prereq_install(): - print(NO_PREREQ_MESSAGE) - return - - if not str2bool(os.environ.get('SKIP_NPM_INSTALL', 'False')): - install_node_prereqs() - install_python_prereqs() - log_installed_python_prereqs() - - -def log_installed_python_prereqs(): - """ Logs output of pip freeze for debugging. """ - sh("pip freeze > {}".format(Env.GEN_LOG_DIR + "/pip_freeze.log")) diff --git a/pavelib/quality.py b/pavelib/quality.py deleted file mode 100644 index 774179f45048..000000000000 --- a/pavelib/quality.py +++ /dev/null @@ -1,602 +0,0 @@ -""" # lint-amnesty, pylint: disable=django-not-configured -Check code quality using pycodestyle, pylint, and diff_quality. -""" - -import json -import os -import re -from datetime import datetime -from xml.sax.saxutils import quoteattr - -from paver.easy import BuildFailure, cmdopts, needs, sh, task - -from .utils.envs import Env -from .utils.timer import timed - -ALL_SYSTEMS = 'lms,cms,common,openedx,pavelib,scripts' -JUNIT_XML_TEMPLATE = """ - -{failure_element} - -""" -JUNIT_XML_FAILURE_TEMPLATE = '' -START_TIME = datetime.utcnow() - - -def write_junit_xml(name, message=None): - """ - Write a JUnit results XML file describing the outcome of a quality check. - """ - if message: - failure_element = JUNIT_XML_FAILURE_TEMPLATE.format(message=quoteattr(message)) - else: - failure_element = '' - data = { - 'failure_count': 1 if message else 0, - 'failure_element': failure_element, - 'name': name, - 'seconds': (datetime.utcnow() - START_TIME).total_seconds(), - } - Env.QUALITY_DIR.makedirs_p() - filename = Env.QUALITY_DIR / f'{name}.xml' - with open(filename, 'w') as f: - f.write(JUNIT_XML_TEMPLATE.format(**data)) - - -def fail_quality(name, message): - """ - Fail the specified quality check by generating the JUnit XML results file - and raising a ``BuildFailure``. - """ - write_junit_xml(name, message) - raise BuildFailure(message) - - -def top_python_dirs(dirname): - """ - Find the directories to start from in order to find all the Python files in `dirname`. - """ - top_dirs = [] - - dir_init = os.path.join(dirname, "__init__.py") - if os.path.exists(dir_init): - top_dirs.append(dirname) - - for directory in ['djangoapps', 'lib']: - subdir = os.path.join(dirname, directory) - subdir_init = os.path.join(subdir, "__init__.py") - if os.path.exists(subdir) and not os.path.exists(subdir_init): - dirs = os.listdir(subdir) - top_dirs.extend(d for d in dirs if os.path.isdir(os.path.join(subdir, d))) - - modules_to_remove = ['__pycache__'] - for module in modules_to_remove: - if module in top_dirs: - top_dirs.remove(module) - - return top_dirs - - -def _get_pep8_violations(clean=True): - """ - Runs pycodestyle. Returns a tuple of (number_of_violations, violations_string) - where violations_string is a string of all PEP 8 violations found, separated - by new lines. - """ - report_dir = (Env.REPORT_DIR / 'pep8') - if clean: - report_dir.rmtree(ignore_errors=True) - report_dir.makedirs_p() - report = report_dir / 'pep8.report' - - # Make sure the metrics subdirectory exists - Env.METRICS_DIR.makedirs_p() - - if not report.exists(): - sh(f'pycodestyle . | tee {report} -a') - - violations_list = _pep8_violations(report) - - return len(violations_list), violations_list - - -def _pep8_violations(report_file): - """ - Returns the list of all PEP 8 violations in the given report_file. - """ - with open(report_file) as f: - return f.readlines() - - -@task -@cmdopts([ - ("system=", "s", "System to act on"), -]) -@timed -def run_pep8(options): # pylint: disable=unused-argument - """ - Run pycodestyle on system code. - Fail the task if any violations are found. - """ - (count, violations_list) = _get_pep8_violations() - violations_list = ''.join(violations_list) - - # Print number of violations to log - violations_count_str = f"Number of PEP 8 violations: {count}" - print(violations_count_str) - print(violations_list) - - # Also write the number of violations to a file - with open(Env.METRICS_DIR / "pep8", "w") as f: - f.write(violations_count_str + '\n\n') - f.write(violations_list) - - # Fail if any violations are found - if count: - failure_string = "FAILURE: Too many PEP 8 violations. " + violations_count_str - failure_string += f"\n\nViolations:\n{violations_list}" - fail_quality('pep8', failure_string) - else: - write_junit_xml('pep8') - - -@task -@needs( - 'pavelib.prereqs.install_node_prereqs', - 'pavelib.utils.test.utils.ensure_clean_package_lock', -) -@cmdopts([ - ("limit=", "l", "limit for number of acceptable violations"), -]) -@timed -def run_eslint(options): - """ - Runs eslint on static asset directories. - If limit option is passed, fails build if more violations than the limit are found. - """ - - eslint_report_dir = (Env.REPORT_DIR / "eslint") - eslint_report = eslint_report_dir / "eslint.report" - _prepare_report_dir(eslint_report_dir) - violations_limit = int(getattr(options, 'limit', -1)) - - sh( - "node --max_old_space_size=4096 node_modules/.bin/eslint " - "--ext .js --ext .jsx --format=compact . | tee {eslint_report}".format( - eslint_report=eslint_report - ), - ignore_error=True - ) - - try: - num_violations = int(_get_count_from_last_line(eslint_report, "eslint")) - except TypeError: - fail_quality( - 'eslint', - "FAILURE: Number of eslint violations could not be found in {eslint_report}".format( - eslint_report=eslint_report - ) - ) - - # Record the metric - _write_metric(num_violations, (Env.METRICS_DIR / "eslint")) - - # Fail if number of violations is greater than the limit - if num_violations > violations_limit > -1: - fail_quality( - 'eslint', - "FAILURE: Too many eslint violations ({count}).\nThe limit is {violations_limit}.".format( - count=num_violations, violations_limit=violations_limit - ) - ) - else: - write_junit_xml('eslint') - - -def _get_stylelint_violations(): - """ - Returns the number of Stylelint violations. - """ - stylelint_report_dir = (Env.REPORT_DIR / "stylelint") - stylelint_report = stylelint_report_dir / "stylelint.report" - _prepare_report_dir(stylelint_report_dir) - formatter = 'node_modules/stylelint-formatter-pretty' - - sh( - "stylelint **/*.scss --custom-formatter={formatter} | tee {stylelint_report}".format( - formatter=formatter, - stylelint_report=stylelint_report, - ), - ignore_error=True - ) - - try: - return int(_get_count_from_last_line(stylelint_report, "stylelint")) - except TypeError: - fail_quality( - 'stylelint', - "FAILURE: Number of stylelint violations could not be found in {stylelint_report}".format( - stylelint_report=stylelint_report - ) - ) - - -@task -@needs('pavelib.prereqs.install_node_prereqs') -@cmdopts([ - ("limit=", "l", "limit for number of acceptable violations"), -]) -@timed -def run_stylelint(options): - """ - Runs stylelint on Sass files. - If limit option is passed, fails build if more violations than the limit are found. - """ - violations_limit = 0 - num_violations = _get_stylelint_violations() - - # Record the metric - _write_metric(num_violations, (Env.METRICS_DIR / "stylelint")) - - # Fail if number of violations is greater than the limit - if num_violations > violations_limit: - fail_quality( - 'stylelint', - "FAILURE: Stylelint failed with too many violations: ({count}).\nThe limit is {violations_limit}.".format( - count=num_violations, - violations_limit=violations_limit, - ) - ) - else: - write_junit_xml('stylelint') - - -@task -@needs('pavelib.prereqs.install_python_prereqs') -@cmdopts([ - ("thresholds=", "t", "json containing limit for number of acceptable violations per rule"), -]) -@timed -def run_xsslint(options): - """ - Runs xsslint/xss_linter.py on the codebase - """ - - thresholds_option = getattr(options, 'thresholds', '{}') - try: - violation_thresholds = json.loads(thresholds_option) - except ValueError: - violation_thresholds = None - if isinstance(violation_thresholds, dict) is False or \ - any(key not in ("total", "rules") for key in violation_thresholds.keys()): - - fail_quality( - 'xsslint', - """FAILURE: Thresholds option "{thresholds_option}" was not supplied using proper format.\n""" - """Here is a properly formatted example, '{{"total":100,"rules":{{"javascript-escape":0}}}}' """ - """with property names in double-quotes.""".format( - thresholds_option=thresholds_option - ) - ) - - xsslint_script = "xss_linter.py" - xsslint_report_dir = (Env.REPORT_DIR / "xsslint") - xsslint_report = xsslint_report_dir / "xsslint.report" - _prepare_report_dir(xsslint_report_dir) - - sh( - "{repo_root}/scripts/xsslint/{xsslint_script} --rule-totals --config={cfg_module} >> {xsslint_report}".format( - repo_root=Env.REPO_ROOT, - xsslint_script=xsslint_script, - xsslint_report=xsslint_report, - cfg_module='scripts.xsslint_config' - ), - ignore_error=True - ) - - xsslint_counts = _get_xsslint_counts(xsslint_report) - - try: - metrics_str = "Number of {xsslint_script} violations: {num_violations}\n".format( - xsslint_script=xsslint_script, num_violations=int(xsslint_counts['total']) - ) - if 'rules' in xsslint_counts and any(xsslint_counts['rules']): - metrics_str += "\n" - rule_keys = sorted(xsslint_counts['rules'].keys()) - for rule in rule_keys: - metrics_str += "{rule} violations: {count}\n".format( - rule=rule, - count=int(xsslint_counts['rules'][rule]) - ) - except TypeError: - fail_quality( - 'xsslint', - "FAILURE: Number of {xsslint_script} violations could not be found in {xsslint_report}".format( - xsslint_script=xsslint_script, xsslint_report=xsslint_report - ) - ) - - metrics_report = (Env.METRICS_DIR / "xsslint") - # Record the metric - _write_metric(metrics_str, metrics_report) - # Print number of violations to log. - sh(f"cat {metrics_report}", ignore_error=True) - - error_message = "" - - # Test total violations against threshold. - if 'total' in list(violation_thresholds.keys()): - if violation_thresholds['total'] < xsslint_counts['total']: - error_message = "Too many violations total ({count}).\nThe limit is {violations_limit}.".format( - count=xsslint_counts['total'], violations_limit=violation_thresholds['total'] - ) - - # Test rule violations against thresholds. - if 'rules' in violation_thresholds: - threshold_keys = sorted(violation_thresholds['rules'].keys()) - for threshold_key in threshold_keys: - if threshold_key not in xsslint_counts['rules']: - error_message += ( - "\nNumber of {xsslint_script} violations for {rule} could not be found in " - "{xsslint_report}." - ).format( - xsslint_script=xsslint_script, rule=threshold_key, xsslint_report=xsslint_report - ) - elif violation_thresholds['rules'][threshold_key] < xsslint_counts['rules'][threshold_key]: - error_message += \ - "\nToo many {rule} violations ({count}).\nThe {rule} limit is {violations_limit}.".format( - rule=threshold_key, count=xsslint_counts['rules'][threshold_key], - violations_limit=violation_thresholds['rules'][threshold_key], - ) - - if error_message: - fail_quality( - 'xsslint', - "FAILURE: XSSLinter Failed.\n{error_message}\n" - "See {xsslint_report} or run the following command to hone in on the problem:\n" - " ./scripts/xss-commit-linter.sh -h".format( - error_message=error_message, xsslint_report=xsslint_report - ) - ) - else: - write_junit_xml('xsslint') - - -def _write_metric(metric, filename): - """ - Write a given metric to a given file - Used for things like reports/metrics/eslint, which will simply tell you the number of - eslint violations found - """ - Env.METRICS_DIR.makedirs_p() - - with open(filename, "w") as metric_file: - metric_file.write(str(metric)) - - -def _prepare_report_dir(dir_name): - """ - Sets a given directory to a created, but empty state - """ - dir_name.rmtree_p() - dir_name.mkdir_p() - - -def _get_report_contents(filename, report_name, last_line_only=False): - """ - Returns the contents of the given file. Use last_line_only to only return - the last line, which can be used for getting output from quality output - files. - - Arguments: - last_line_only: True to return the last line only, False to return a - string with full contents. - - Returns: - String containing full contents of the report, or the last line. - - """ - if os.path.isfile(filename): - with open(filename) as report_file: - if last_line_only: - lines = report_file.readlines() - for line in reversed(lines): - if line != '\n': - return line - return None - else: - return report_file.read() - else: - file_not_found_message = f"FAILURE: The following log file could not be found: {filename}" - fail_quality(report_name, file_not_found_message) - - -def _get_count_from_last_line(filename, file_type): - """ - This will return the number in the last line of a file. - It is returning only the value (as a floating number). - """ - report_contents = _get_report_contents(filename, file_type, last_line_only=True) - - if report_contents is None: - return 0 - - last_line = report_contents.strip() - # Example of the last line of a compact-formatted eslint report (for example): "62829 problems" - regex = r'^\d+' - - try: - return float(re.search(regex, last_line).group(0)) - # An AttributeError will occur if the regex finds no matches. - # A ValueError will occur if the returned regex cannot be cast as a float. - except (AttributeError, ValueError): - return None - - -def _get_xsslint_counts(filename): - """ - This returns a dict of violations from the xsslint report. - - Arguments: - filename: The name of the xsslint report. - - Returns: - A dict containing the following: - rules: A dict containing the count for each rule as follows: - violation-rule-id: N, where N is the number of violations - total: M, where M is the number of total violations - - """ - report_contents = _get_report_contents(filename, 'xsslint') - rule_count_regex = re.compile(r"^(?P[a-z-]+):\s+(?P\d+) violations", re.MULTILINE) - total_count_regex = re.compile(r"^(?P\d+) violations total", re.MULTILINE) - violations = {'rules': {}} - for violation_match in rule_count_regex.finditer(report_contents): - try: - violations['rules'][violation_match.group('rule_id')] = int(violation_match.group('count')) - except ValueError: - violations['rules'][violation_match.group('rule_id')] = None - try: - violations['total'] = int(total_count_regex.search(report_contents).group('count')) - # An AttributeError will occur if the regex finds no matches. - # A ValueError will occur if the returned regex cannot be cast as a float. - except (AttributeError, ValueError): - violations['total'] = None - return violations - - -def _extract_missing_pii_annotations(filename): - """ - Returns the number of uncovered models from the stdout report of django_find_annotations. - - Arguments: - filename: Filename where stdout of django_find_annotations was captured. - - Returns: - three-tuple containing: - 1. The number of uncovered models, - 2. A bool indicating whether the coverage is still below the threshold, and - 3. The full report as a string. - """ - uncovered_models = 0 - pii_check_passed = True - if os.path.isfile(filename): - with open(filename) as report_file: - lines = report_file.readlines() - - # Find the count of uncovered models. - uncovered_regex = re.compile(r'^Coverage found ([\d]+) uncovered') - for line in lines: - uncovered_match = uncovered_regex.match(line) - if uncovered_match: - uncovered_models = int(uncovered_match.groups()[0]) - break - - # Find a message which suggests the check failed. - failure_regex = re.compile(r'^Coverage threshold not met!') - for line in lines: - failure_match = failure_regex.match(line) - if failure_match: - pii_check_passed = False - break - - # Each line in lines already contains a newline. - full_log = ''.join(lines) - else: - fail_quality('pii', f'FAILURE: Log file could not be found: {filename}') - - return (uncovered_models, pii_check_passed, full_log) - - -@task -@needs('pavelib.prereqs.install_python_prereqs') -@cmdopts([ - ("report-dir=", "r", "Directory in which to put PII reports"), -]) -@timed -def run_pii_check(options): - """ - Guarantee that all Django models are PII-annotated. - """ - pii_report_name = 'pii' - default_report_dir = (Env.REPORT_DIR / pii_report_name) - report_dir = getattr(options, 'report_dir', default_report_dir) - output_file = os.path.join(report_dir, 'pii_check_{}.report') - env_report = [] - pii_check_passed = True - for env_name, env_settings_file in (("CMS", "cms.envs.test"), ("LMS", "lms.envs.test")): - try: - print() - print(f"Running {env_name} PII Annotation check and report") - print("-" * 45) - run_output_file = str(output_file).format(env_name.lower()) - sh( - "mkdir -p {} && " # lint-amnesty, pylint: disable=duplicate-string-formatting-argument - "export DJANGO_SETTINGS_MODULE={}; " - "code_annotations django_find_annotations " - "--config_file .pii_annotations.yml --report_path {} --app_name {} " - "--lint --report --coverage | tee {}".format( - report_dir, env_settings_file, report_dir, env_name.lower(), run_output_file - ) - ) - uncovered_model_count, pii_check_passed_env, full_log = _extract_missing_pii_annotations(run_output_file) - env_report.append(( - uncovered_model_count, - full_log, - )) - - except BuildFailure as error_message: - fail_quality(pii_report_name, f'FAILURE: {error_message}') - - if not pii_check_passed_env: - pii_check_passed = False - - # Determine which suite is the worst offender by obtaining the max() keying off uncovered_count. - uncovered_count, full_log = max(env_report, key=lambda r: r[0]) - - # Write metric file. - if uncovered_count is None: - uncovered_count = 0 - metrics_str = f"Number of PII Annotation violations: {uncovered_count}\n" - _write_metric(metrics_str, (Env.METRICS_DIR / pii_report_name)) - - # Finally, fail the paver task if code_annotations suggests that the check failed. - if not pii_check_passed: - fail_quality('pii', full_log) - - -@task -@needs('pavelib.prereqs.install_python_prereqs') -@timed -def check_keywords(): - """ - Check Django model fields for names that conflict with a list of reserved keywords - """ - report_path = os.path.join(Env.REPORT_DIR, 'reserved_keywords') - sh(f"mkdir -p {report_path}") - - overall_status = True - for env, env_settings_file in [('lms', 'lms.envs.test'), ('cms', 'cms.envs.test')]: - report_file = f"{env}_reserved_keyword_report.csv" - override_file = os.path.join(Env.REPO_ROOT, "db_keyword_overrides.yml") - try: - sh( - "export DJANGO_SETTINGS_MODULE={settings_file}; " - "python manage.py {app} check_reserved_keywords " - "--override_file {override_file} " - "--report_path {report_path} " - "--report_file {report_file}".format( - settings_file=env_settings_file, app=env, override_file=override_file, - report_path=report_path, report_file=report_file - ) - ) - except BuildFailure: - overall_status = False - - if not overall_status: - fail_quality( - 'keywords', - 'Failure: reserved keyword checker failed. Reports can be found here: {}'.format( - report_path - ) - ) diff --git a/pavelib/utils/__init__.py b/pavelib/utils/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/pavelib/utils/cmd.py b/pavelib/utils/cmd.py deleted file mode 100644 index a350c90a6a96..000000000000 --- a/pavelib/utils/cmd.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -Helper functions for constructing shell commands. -""" - - -def cmd(*args): - """ - Concatenate the arguments into a space-separated shell command. - """ - return " ".join(str(arg) for arg in args if arg) - - -def django_cmd(sys, settings, *args): - """ - Construct a Django management command. - - `sys` is either 'lms' or 'studio'. - `settings` is the Django settings module (such as "dev" or "test") - `args` are concatenated to form the rest of the command. - """ - # Maintain backwards compatibility with manage.py, - # which calls "studio" "cms" - sys = 'cms' if sys == 'studio' else sys - return cmd("python manage.py", sys, f"--settings={settings}", *args) diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py deleted file mode 100644 index 7dc9870fbdea..000000000000 --- a/pavelib/utils/envs.py +++ /dev/null @@ -1,271 +0,0 @@ -""" -Helper functions for loading environment settings. -""" -import configparser -import json -import os -import sys -from time import sleep - -from lazy import lazy -from path import Path as path -from paver.easy import BuildFailure, sh - -from pavelib.utils.cmd import django_cmd - - -def repo_root(): - """ - Get the root of the git repository (edx-platform). - - This sometimes fails on Docker Devstack, so it's been broken - down with some additional error handling. It usually starts - working within 30 seconds or so; for more details, see - https://openedx.atlassian.net/browse/PLAT-1629 and - https://github.com/docker/for-mac/issues/1509 - """ - file_path = path(__file__) - attempt = 1 - while True: - try: - absolute_path = file_path.abspath() - break - except OSError: - print(f'Attempt {attempt}/180 to get an absolute path failed') - if attempt < 180: - attempt += 1 - sleep(1) - else: - print('Unable to determine the absolute path of the edx-platform repo, aborting') - raise - return absolute_path.parent.parent.parent - - -class Env: - """ - Load information about the execution environment. - """ - - # Root of the git repository (edx-platform) - REPO_ROOT = repo_root() - - # Reports Directory - REPORT_DIR = REPO_ROOT / 'reports' - METRICS_DIR = REPORT_DIR / 'metrics' - QUALITY_DIR = REPORT_DIR / 'quality_junitxml' - - # Generic log dir - GEN_LOG_DIR = REPO_ROOT / "test_root" / "log" - - # Python unittest dirs - PYTHON_COVERAGERC = REPO_ROOT / ".coveragerc" - - # Which Python version should be used in xdist workers? - PYTHON_VERSION = os.environ.get("PYTHON_VERSION", "2.7") - - # Directory that videos are served from - VIDEO_SOURCE_DIR = REPO_ROOT / "test_root" / "data" / "video" - - PRINT_SETTINGS_LOG_FILE = GEN_LOG_DIR / "print_settings.log" - - # Detect if in a Docker container, and if so which one - FRONTEND_TEST_SERVER_HOST = os.environ.get('FRONTEND_TEST_SERVER_HOSTNAME', '0.0.0.0') - USING_DOCKER = FRONTEND_TEST_SERVER_HOST != '0.0.0.0' - DEVSTACK_SETTINGS = 'devstack_docker' if USING_DOCKER else 'devstack' - TEST_SETTINGS = 'test' - - # Mongo databases that will be dropped before/after the tests run - MONGO_HOST = 'localhost' - - # Test Ids Directory - TEST_DIR = REPO_ROOT / ".testids" - - # Configured browser to use for the js test suites - SELENIUM_BROWSER = os.environ.get('SELENIUM_BROWSER', 'firefox') - if USING_DOCKER: - KARMA_BROWSER = 'ChromeDocker' if SELENIUM_BROWSER == 'chrome' else 'FirefoxDocker' - else: - KARMA_BROWSER = 'FirefoxNoUpdates' - - # Files used to run each of the js test suites - # TODO: We have [temporarily disabled] the three Webpack-based tests suites. They have been silently - # broken for a long time; after noticing they were broken, we added the DieHardPlugin to - # webpack.common.config.js to prevent future silent breakage, but have not yet been able to - # fix and re-enable the suites. Note that the LMS suite is all Webpack-based even though it's - # not in the name. - # Issue: https://github.com/openedx/edx-platform/issues/35956 - KARMA_CONFIG_FILES = [ - REPO_ROOT / 'cms/static/karma_cms.conf.js', - REPO_ROOT / 'cms/static/karma_cms_squire.conf.js', - ## [temporarily disabled] REPO_ROOT / 'cms/static/karma_cms_webpack.conf.js', - ## [temporarily disabled] REPO_ROOT / 'lms/static/karma_lms.conf.js', - REPO_ROOT / 'xmodule/js/karma_xmodule.conf.js', - ## [temporarily disabled] REPO_ROOT / 'xmodule/js/karma_xmodule_webpack.conf.js', - REPO_ROOT / 'common/static/karma_common.conf.js', - REPO_ROOT / 'common/static/karma_common_requirejs.conf.js', - ] - - JS_TEST_ID_KEYS = [ - 'cms', - 'cms-squire', - ## [temporarily-disabled] 'cms-webpack', - ## [temporarily-disabled] 'lms', - 'xmodule', - ## [temporarily-disabled] 'xmodule-webpack', - 'common', - 'common-requirejs', - 'jest-snapshot' - ] - - JS_REPORT_DIR = REPORT_DIR / 'javascript' - - # Directories used for pavelib/ tests - IGNORED_TEST_DIRS = ('__pycache__', '.cache', '.pytest_cache') - LIB_TEST_DIRS = [path("pavelib/paver_tests"), path("scripts/xsslint/tests")] - - # Directory for i18n test reports - I18N_REPORT_DIR = REPORT_DIR / 'i18n' - - # Directory for keeping src folder that comes with pip installation. - # Setting this is equivalent to passing `--src ` to pip directly. - PIP_SRC = os.environ.get("PIP_SRC") - - # Service variant (lms, cms, etc.) configured with an environment variable - # We use this to determine which envs.json file to load. - SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None) - - # If service variant not configured in env, then pass the correct - # environment for lms / cms - if not SERVICE_VARIANT: # this will intentionally catch ""; - if any(i in sys.argv[1:] for i in ('cms', 'studio')): - SERVICE_VARIANT = 'cms' - else: - SERVICE_VARIANT = 'lms' - - @classmethod - def get_django_settings(cls, django_settings, system, settings=None, print_setting_args=None): - """ - Interrogate Django environment for specific settings values - :param django_settings: list of django settings values to get - :param system: the django app to use when asking for the setting (lms | cms) - :param settings: the settings file to use when asking for the value - :param print_setting_args: the additional arguments to send to print_settings - :return: unicode value of the django setting - """ - if not settings: - settings = os.environ.get("EDX_PLATFORM_SETTINGS", "aws") - log_dir = os.path.dirname(cls.PRINT_SETTINGS_LOG_FILE) - if not os.path.exists(log_dir): - os.makedirs(log_dir) - settings_length = len(django_settings) - django_settings = ' '.join(django_settings) # parse_known_args makes a list again - print_setting_args = ' '.join(print_setting_args or []) - try: - value = sh( - django_cmd( - system, - settings, - "print_setting {django_settings} 2>{log_file} {print_setting_args}".format( - django_settings=django_settings, - print_setting_args=print_setting_args, - log_file=cls.PRINT_SETTINGS_LOG_FILE - ).strip() - ), - capture=True - ) - # else for cases where values are not found & sh returns one None value - return tuple(str(value).splitlines()) if value else tuple(None for _ in range(settings_length)) - except BuildFailure: - print(f"Unable to print the value of the {django_settings} setting:") - with open(cls.PRINT_SETTINGS_LOG_FILE) as f: - print(f.read()) - sys.exit(1) - - @classmethod - def get_django_json_settings(cls, django_settings, system, settings=None): - """ - Interrogate Django environment for specific settings value - :param django_settings: list of django settings values to get - :param system: the django app to use when asking for the setting (lms | cms) - :param settings: the settings file to use when asking for the value - :return: json string value of the django setting - """ - return cls.get_django_settings( - django_settings, - system, - settings=settings, - print_setting_args=["--json"], - ) - - @classmethod - def covered_modules(cls): - """ - List the source modules listed in .coveragerc for which coverage - will be measured. - """ - coveragerc = configparser.RawConfigParser() - coveragerc.read(cls.PYTHON_COVERAGERC) - modules = coveragerc.get('run', 'source') - result = [] - for module in modules.split('\n'): - module = module.strip() - if module: - result.append(module) - return result - - @lazy - def env_tokens(self): - """ - Return a dict of environment settings. - If we couldn't find the JSON file, issue a warning and return an empty dict. - """ - - # Find the env JSON file - if self.SERVICE_VARIANT: - env_path = self.REPO_ROOT.parent / f"{self.SERVICE_VARIANT}.env.json" - else: - env_path = path("env.json").abspath() - - # If the file does not exist, here or one level up, - # issue a warning and return an empty dict - if not env_path.isfile(): - env_path = env_path.parent.parent / env_path.basename() - if not env_path.isfile(): - print( - "Warning: could not find environment JSON file " - "at '{path}'".format(path=env_path), - file=sys.stderr, - ) - return {} - - # Otherwise, load the file as JSON and return the resulting dict - try: - with open(env_path) as env_file: - return json.load(env_file) - - except ValueError: - print( - "Error: Could not parse JSON " - "in {path}".format(path=env_path), - file=sys.stderr, - ) - sys.exit(1) - - @lazy - def feature_flags(self): - """ - Return a dictionary of feature flags configured by the environment. - """ - return self.env_tokens.get('FEATURES', {}) - - @classmethod - def rsync_dirs(cls): - """ - List the directories that should be synced during pytest-xdist - execution. Needs to include all modules for which coverage is - measured, not just the tests being run. - """ - result = set() - for module in cls.covered_modules(): - result.add(module.split('/')[0]) - return result diff --git a/pavelib/utils/process.py b/pavelib/utils/process.py deleted file mode 100644 index da2dafa8803d..000000000000 --- a/pavelib/utils/process.py +++ /dev/null @@ -1,121 +0,0 @@ -""" -Helper functions for managing processes. -""" - - -import atexit -import os -import signal -import subprocess -import sys - -import psutil -from paver import tasks - - -def kill_process(proc): - """ - Kill the process `proc` created with `subprocess`. - """ - p1_group = psutil.Process(proc.pid) - child_pids = p1_group.children(recursive=True) - - for child_pid in child_pids: - os.kill(child_pid.pid, signal.SIGKILL) - - -def run_multi_processes(cmd_list, out_log=None, err_log=None): - """ - Run each shell command in `cmd_list` in a separate process, - piping stdout to `out_log` (a path) and stderr to `err_log` (also a path). - - Terminates the processes on CTRL-C and ensures the processes are killed - if an error occurs. - """ - kwargs = {'shell': True, 'cwd': None} - pids = [] - - if out_log: - out_log_file = open(out_log, 'w') # lint-amnesty, pylint: disable=consider-using-with - kwargs['stdout'] = out_log_file - - if err_log: - err_log_file = open(err_log, 'w') # lint-amnesty, pylint: disable=consider-using-with - kwargs['stderr'] = err_log_file - - # If the user is performing a dry run of a task, then just log - # the command strings and return so that no destructive operations - # are performed. - if tasks.environment.dry_run: - for cmd in cmd_list: - tasks.environment.info(cmd) - return - - try: - for cmd in cmd_list: - pids.extend([subprocess.Popen(cmd, **kwargs)]) - - # pylint: disable=unused-argument - def _signal_handler(*args): - """ - What to do when process is ended - """ - print("\nEnding...") - - signal.signal(signal.SIGINT, _signal_handler) - print("Enter CTL-C to end") - signal.pause() - print("Processes ending") - - # pylint: disable=broad-except - except Exception as err: - print(f"Error running process {err}", file=sys.stderr) - - finally: - for pid in pids: - kill_process(pid) - - -def run_process(cmd, out_log=None, err_log=None): - """ - Run the shell command `cmd` in a separate process, - piping stdout to `out_log` (a path) and stderr to `err_log` (also a path). - - Terminates the process on CTRL-C or if an error occurs. - """ - return run_multi_processes([cmd], out_log=out_log, err_log=err_log) - - -def run_background_process(cmd, out_log=None, err_log=None, cwd=None): - """ - Runs a command as a background process. Sends SIGINT at exit. - """ - - kwargs = {'shell': True, 'cwd': cwd} - if out_log: - out_log_file = open(out_log, 'w') # lint-amnesty, pylint: disable=consider-using-with - kwargs['stdout'] = out_log_file - - if err_log: - err_log_file = open(err_log, 'w') # lint-amnesty, pylint: disable=consider-using-with - kwargs['stderr'] = err_log_file - - proc = subprocess.Popen(cmd, **kwargs) # lint-amnesty, pylint: disable=consider-using-with - - def exit_handler(): - """ - Send SIGINT to the process's children. This is important - for running commands under coverage, as coverage will not - produce the correct artifacts if the child process isn't - killed properly. - """ - p1_group = psutil.Process(proc.pid) - child_pids = p1_group.children(recursive=True) - - for child_pid in child_pids: - os.kill(child_pid.pid, signal.SIGINT) - - # Wait for process to actually finish - proc.wait() - - atexit.register(exit_handler) diff --git a/pavelib/utils/test/__init__.py b/pavelib/utils/test/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/pavelib/utils/test/suites/__init__.py b/pavelib/utils/test/suites/__init__.py deleted file mode 100644 index 34ecd49c1c74..000000000000 --- a/pavelib/utils/test/suites/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -""" -TestSuite class and subclasses -""" -from .js_suite import JestSnapshotTestSuite, JsTestSuite -from .suite import TestSuite diff --git a/pavelib/utils/test/suites/js_suite.py b/pavelib/utils/test/suites/js_suite.py deleted file mode 100644 index 4e53d454fee5..000000000000 --- a/pavelib/utils/test/suites/js_suite.py +++ /dev/null @@ -1,109 +0,0 @@ -""" -Javascript test tasks -""" - - -from paver import tasks - -from pavelib.utils.envs import Env -from pavelib.utils.test import utils as test_utils -from pavelib.utils.test.suites.suite import TestSuite - -__test__ = False # do not collect - - -class JsTestSuite(TestSuite): - """ - A class for running JavaScript tests. - """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.run_under_coverage = kwargs.get('with_coverage', True) - self.mode = kwargs.get('mode', 'run') - self.report_dir = Env.JS_REPORT_DIR - self.opts = kwargs - - suite = args[0] - self.subsuites = self._default_subsuites if suite == 'all' else [JsTestSubSuite(*args, **kwargs)] - - def __enter__(self): - super().__enter__() - if tasks.environment.dry_run: - tasks.environment.info("make report_dir") - else: - self.report_dir.makedirs_p() - if not self.skip_clean: - test_utils.clean_test_files() - - if self.mode == 'run' and not self.run_under_coverage: - test_utils.clean_dir(self.report_dir) - - @property - def _default_subsuites(self): - """ - Returns all JS test suites - """ - return [JsTestSubSuite(test_id, **self.opts) for test_id in Env.JS_TEST_ID_KEYS if test_id != 'jest-snapshot'] - - -class JsTestSubSuite(TestSuite): - """ - Class for JS suites like cms, cms-squire, lms, common, - common-requirejs and xmodule - """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.test_id = args[0] - self.run_under_coverage = kwargs.get('with_coverage', True) - self.mode = kwargs.get('mode', 'run') - self.port = kwargs.get('port') - self.root = self.root + ' javascript' - self.report_dir = Env.JS_REPORT_DIR - - try: - self.test_conf_file = Env.KARMA_CONFIG_FILES[Env.JS_TEST_ID_KEYS.index(self.test_id)] - except ValueError: - self.test_conf_file = Env.KARMA_CONFIG_FILES[0] - - self.coverage_report = self.report_dir / f'coverage-{self.test_id}.xml' - self.xunit_report = self.report_dir / f'javascript_xunit-{self.test_id}.xml' - - @property - def cmd(self): - """ - Run the tests using karma runner. - """ - cmd = [ - "node", - "--max_old_space_size=4096", - "node_modules/.bin/karma", - "start", - self.test_conf_file, - "--single-run={}".format('false' if self.mode == 'dev' else 'true'), - "--capture-timeout=60000", - f"--junitreportpath={self.xunit_report}", - f"--browsers={Env.KARMA_BROWSER}", - ] - - if self.port: - cmd.append(f"--port={self.port}") - - if self.run_under_coverage: - cmd.extend([ - "--coverage", - f"--coveragereportpath={self.coverage_report}", - ]) - - return cmd - - -class JestSnapshotTestSuite(TestSuite): - """ - A class for running Jest Snapshot tests. - """ - @property - def cmd(self): - """ - Run the tests using Jest. - """ - return ["jest"] diff --git a/pavelib/utils/test/suites/suite.py b/pavelib/utils/test/suites/suite.py deleted file mode 100644 index 5a423c827c21..000000000000 --- a/pavelib/utils/test/suites/suite.py +++ /dev/null @@ -1,149 +0,0 @@ -""" -A class used for defining and running test suites -""" - - -import os -import subprocess -import sys - -from paver import tasks - -from pavelib.utils.process import kill_process - -try: - from pygments.console import colorize -except ImportError: - colorize = lambda color, text: text - -__test__ = False # do not collect - - -class TestSuite: - """ - TestSuite is a class that defines how groups of tests run. - """ - def __init__(self, *args, **kwargs): - self.root = args[0] - self.subsuites = kwargs.get('subsuites', []) - self.failed_suites = [] - self.verbosity = int(kwargs.get('verbosity', 1)) - self.skip_clean = kwargs.get('skip_clean', False) - self.passthrough_options = kwargs.get('passthrough_options', []) - - def __enter__(self): - """ - This will run before the test suite is run with the run_suite_tests method. - If self.run_test is called directly, it should be run in a 'with' block to - ensure that the proper context is created. - - Specific setup tasks should be defined in each subsuite. - - i.e. Checking for and defining required directories. - """ - print(f"\nSetting up for {self.root}") - self.failed_suites = [] - - def __exit__(self, exc_type, exc_value, traceback): - """ - This is run after the tests run with the run_suite_tests method finish. - Specific clean up tasks should be defined in each subsuite. - - If self.run_test is called directly, it should be run in a 'with' block - to ensure that clean up happens properly. - - i.e. Cleaning mongo after the lms tests run. - """ - print(f"\nCleaning up after {self.root}") - - @property - def cmd(self): - """ - The command to run tests (as a string). For this base class there is none. - """ - return None - - @staticmethod - def is_success(exit_code): - """ - Determine if the given exit code represents a success of the test - suite. By default, only a zero counts as a success. - """ - return exit_code == 0 - - def run_test(self): - """ - Runs a self.cmd in a subprocess and waits for it to finish. - It returns False if errors or failures occur. Otherwise, it - returns True. - """ - cmd = " ".join(self.cmd) - - if tasks.environment.dry_run: - tasks.environment.info(cmd) - return - - sys.stdout.write(cmd) - - msg = colorize( - 'green', - '\n{bar}\n Running tests for {suite_name} \n{bar}\n'.format(suite_name=self.root, bar='=' * 40), - ) - - sys.stdout.write(msg) - sys.stdout.flush() - - if 'TEST_SUITE' not in os.environ: - os.environ['TEST_SUITE'] = self.root.replace("/", "_") - kwargs = {'shell': True, 'cwd': None} - process = None - - try: - process = subprocess.Popen(cmd, **kwargs) # lint-amnesty, pylint: disable=consider-using-with - return self.is_success(process.wait()) - except KeyboardInterrupt: - kill_process(process) - sys.exit(1) - - def run_suite_tests(self): - """ - Runs each of the suites in self.subsuites while tracking failures - """ - # Uses __enter__ and __exit__ for context - with self: - # run the tests for this class, and for all subsuites - if self.cmd: - passed = self.run_test() - if not passed: - self.failed_suites.append(self) - - for suite in self.subsuites: - suite.run_suite_tests() - if suite.failed_suites: - self.failed_suites.extend(suite.failed_suites) - - def report_test_results(self): - """ - Writes a list of failed_suites to sys.stderr - """ - if self.failed_suites: - msg = colorize('red', "\n\n{bar}\nTests failed in the following suites:\n* ".format(bar="=" * 48)) - msg += colorize('red', '\n* '.join([s.root for s in self.failed_suites]) + '\n\n') - else: - msg = colorize('green', "\n\n{bar}\nNo test failures ".format(bar="=" * 48)) - - print(msg) - - def run(self): - """ - Runs the tests in the suite while tracking and reporting failures. - """ - self.run_suite_tests() - - if tasks.environment.dry_run: - return - - self.report_test_results() - - if self.failed_suites: - sys.exit(1) diff --git a/pavelib/utils/test/utils.py b/pavelib/utils/test/utils.py deleted file mode 100644 index 0851251e2222..000000000000 --- a/pavelib/utils/test/utils.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -Helper functions for test tasks -""" - - -import os - -from paver.easy import cmdopts, sh, task - -from pavelib.utils.envs import Env -from pavelib.utils.timer import timed - - -MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017')) - -COVERAGE_CACHE_BUCKET = "edx-tools-coverage-caches" -COVERAGE_CACHE_BASEPATH = "test_root/who_tests_what" -COVERAGE_CACHE_BASELINE = "who_tests_what.{}.baseline".format(os.environ.get('WTW_CONTEXT', 'all')) -WHO_TESTS_WHAT_DIFF = "who_tests_what.diff" - - -__test__ = False # do not collect - - -@task -@timed -def clean_test_files(): - """ - Clean fixture files used by tests and .pyc files - """ - sh("git clean -fqdx test_root/logs test_root/data test_root/staticfiles test_root/uploads") - # This find command removes all the *.pyc files that aren't in the .git - # directory. See this blog post for more details: - # http://nedbatchelder.com/blog/201505/be_careful_deleting_files_around_git.html - sh(r"find . -name '.git' -prune -o -name '*.pyc' -exec rm {} \;") - sh("rm -rf test_root/log/auto_screenshots/*") - sh("rm -rf /tmp/mako_[cl]ms") - - -@task -@timed -def ensure_clean_package_lock(): - """ - Ensure no untracked changes have been made in the current git context. - """ - sh(""" - git diff --name-only --exit-code package-lock.json || - (echo \"Dirty package-lock.json, run 'npm install' and commit the generated changes\" && exit 1) - """) - - -def clean_dir(directory): - """ - Delete all the files from the specified directory. - """ - # We delete the files but preserve the directory structure - # so that coverage.py has a place to put the reports. - sh(f'find {directory} -type f -delete') - - -@task -@cmdopts([ - ('skip-clean', 'C', 'skip cleaning repository before running tests'), - ('skip_clean', None, 'deprecated in favor of skip-clean'), -]) -@timed -def clean_reports_dir(options): - """ - Clean coverage files, to ensure that we don't use stale data to generate reports. - """ - if getattr(options, 'skip_clean', False): - print('--skip-clean is set, skipping...') - return - - # We delete the files but preserve the directory structure - # so that coverage.py has a place to put the reports. - reports_dir = Env.REPORT_DIR.makedirs_p() - clean_dir(reports_dir) - - -@task -@timed -def clean_mongo(): - """ - Clean mongo test databases - """ - sh("mongo {host}:{port} {repo_root}/scripts/delete-mongo-test-dbs.js".format( - host=Env.MONGO_HOST, - port=MONGO_PORT_NUM, - repo_root=Env.REPO_ROOT, - )) diff --git a/pavelib/utils/timer.py b/pavelib/utils/timer.py deleted file mode 100644 index fc6f3003736a..000000000000 --- a/pavelib/utils/timer.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -Tools for timing paver tasks -""" - - -import json -import logging -import os -import sys -import traceback -from datetime import datetime -from os.path import dirname, exists - -import wrapt - -LOGGER = logging.getLogger(__file__) -PAVER_TIMER_LOG = os.environ.get('PAVER_TIMER_LOG') - - -@wrapt.decorator -def timed(wrapped, instance, args, kwargs): # pylint: disable=unused-argument - """ - Log execution time for a function to a log file. - - Logging is only actually executed if the PAVER_TIMER_LOG environment variable - is set. That variable is expanded for the current user and current - environment variables. It also can have :meth:`~Datetime.strftime` format - identifiers which are substituted using the time when the task started. - - For example, ``PAVER_TIMER_LOG='~/.paver.logs/%Y-%d-%m.log'`` will create a new - log file every day containing reconds for paver tasks run that day, and - will put those log files in the ``.paver.logs`` directory inside the users - home. - - Must be earlier in the decorator stack than the paver task declaration. - """ - start = datetime.utcnow() - exception_info = {} - try: - return wrapped(*args, **kwargs) - except Exception as exc: - exception_info = { - 'exception': "".join(traceback.format_exception_only(type(exc), exc)).strip() - } - raise - finally: - end = datetime.utcnow() - - # N.B. This is intended to provide a consistent interface and message format - # across all of Open edX tooling, so it deliberately eschews standard - # python logging infrastructure. - if PAVER_TIMER_LOG is not None: - - log_path = start.strftime(PAVER_TIMER_LOG) - - log_message = { - 'python_version': sys.version, - 'task': f"{wrapped.__module__}.{wrapped.__name__}", - 'args': [repr(arg) for arg in args], - 'kwargs': {key: repr(value) for key, value in kwargs.items()}, - 'started_at': start.isoformat(' '), - 'ended_at': end.isoformat(' '), - 'duration': (end - start).total_seconds(), - } - log_message.update(exception_info) - - try: - log_dir = dirname(log_path) - if log_dir and not exists(log_dir): - os.makedirs(log_dir) - - with open(log_path, 'a') as outfile: - json.dump( - log_message, - outfile, - separators=(',', ':'), - sort_keys=True, - ) - outfile.write('\n') - except OSError: - # Squelch OSErrors, because we expect them and they shouldn't - # interrupt the rest of the process. - LOGGER.exception("Unable to write timing logs") diff --git a/pavement.py b/pavement.py deleted file mode 100644 index 41a6227dbfb5..000000000000 --- a/pavement.py +++ /dev/null @@ -1,12 +0,0 @@ -import sys # lint-amnesty, pylint: disable=django-not-configured, missing-module-docstring -import os - -# Ensure that we can import pavelib, and that our copy of pavelib -# takes precedence over anything else installed in the virtualenv. -# In local dev, we usually don't need to do this, because Python -# automatically puts the current working directory on the system path. -# Until we re-run pip install, the other copies of edx-platform could -# take precedence, leading to some strange results. -sys.path.insert(0, os.path.dirname(__file__)) - -from pavelib import * # lint-amnesty, pylint: disable=wildcard-import, wrong-import-position diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt index b8166ba67540..f3cc8fc9c9e8 100644 --- a/requirements/common_constraints.txt +++ b/requirements/common_constraints.txt @@ -28,3 +28,7 @@ elasticsearch<7.14.0 # Cause: https://github.com/openedx/edx-lint/issues/458 # This can be unpinned once https://github.com/openedx/edx-lint/issues/459 has been resolved. pip<24.3 + +# Cause: https://github.com/openedx/edx-lint/issues/475 +# This can be unpinned once https://github.com/openedx/edx-lint/issues/476 has been resolved. +urllib3<2.3.0 diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 52978f1bae1d..fcea298aedae 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -60,12 +60,14 @@ django-webpack-loader==0.7.0 # Adding pin to avoid any major upgrade djangorestframework<3.15.0 -# Date: 2023-07-19 -# The version of django-stubs we can use depends on which Django release we're using -# 1.16.0 works with Django 3.2 through 4.1 -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35275 -django-stubs==1.16.0 -djangorestframework-stubs==3.14.0 # Pinned to match django-stubs. Remove this when we can remove the above pin. +# Date: 2024-07-19 +# Generally speaking, the major version of django-stubs must either match the major version +# of django, or exceed it by 1. So, we will need to perpetually constrain django-stubs and +# update it as we perform django upgrades. For more details, see: +# https://github.com/typeddjango/django-stubs?tab=readme-ov-file#version-compatibility +# including the note on "Partial Support". +# Issue: https://github.com/openedx/edx-platform/issues/35275 +django-stubs<6 # Date: 2024-07-23 # django-storages==1.14.4 breaks course imports @@ -78,7 +80,7 @@ django-storages<1.14.4 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==5.4.1 +edx-enterprise==5.5.2 # Date: 2024-05-09 # This has to be constrained as well because newer versions of edx-i18n-tools need the @@ -112,12 +114,6 @@ markdown<3.4.0 # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35270 moto<5.0 -# Date: 2024-10-16 -# MyPY 1.12.0 fails on all PRs with the following error: -# openedx/core/djangoapps/content_libraries/api.py:732: error: INTERNAL ERROR -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35667 -mypy<1.12.0 - # Date: 2024-07-16 # We need to upgrade the version of elasticsearch to atleast 7.15 before we can upgrade to Numpy 2.0.0 # Otherwise we see a failure while running the following command: @@ -181,3 +177,10 @@ social-auth-app-django<=5.4.1 # # Date: 2024-10-14 # # The edx-enterprise is currently using edx-rest-api-client==5.7.1, which needs to be updated first. # edx-rest-api-client==5.7.1 + +# Date 2025-01-08 +# elasticsearch==7.13.x is downgrading urllib3 from 2.2.3 to 1.26.20 +# https://github.com/elastic/elasticsearch-py/blob/v7.13.4/setup.py#L42 +# We are pinning this until we can upgrade to a version of elasticsearch that uses a more recent version of urllib3. +# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35126 +elasticsearch==7.9.1 diff --git a/requirements/edx-sandbox/base.txt b/requirements/edx-sandbox/base.txt index 60a5eff8a62d..1bd3d149f487 100644 --- a/requirements/edx-sandbox/base.txt +++ b/requirements/edx-sandbox/base.txt @@ -20,11 +20,11 @@ cryptography==44.0.0 # via -r requirements/edx-sandbox/base.in cycler==0.12.1 # via matplotlib -fonttools==4.55.2 +fonttools==4.55.3 # via matplotlib joblib==1.4.2 # via nltk -kiwisolver==1.4.7 +kiwisolver==1.4.8 # via matplotlib lxml[html-clean,html_clean]==5.3.0 # via @@ -37,7 +37,7 @@ markupsafe==3.0.2 # via # chem # openedx-calc -matplotlib==3.9.3 +matplotlib==3.10.0 # via -r requirements/edx-sandbox/base.in mpmath==1.3.0 # via sympy @@ -59,11 +59,11 @@ openedx-calc==4.0.1 # via -r requirements/edx-sandbox/base.in packaging==24.2 # via matplotlib -pillow==11.0.0 +pillow==11.1.0 # via matplotlib pycparser==2.22 # via cffi -pyparsing==3.2.0 +pyparsing==3.2.1 # via # -r requirements/edx-sandbox/base.in # chem @@ -75,7 +75,7 @@ random2==1.0.2 # via -r requirements/edx-sandbox/base.in regex==2024.11.6 # via nltk -scipy==1.14.1 +scipy==1.15.0 # via # -r requirements/edx-sandbox/base.in # chem diff --git a/requirements/edx-sandbox/py38.txt b/requirements/edx-sandbox/py38.txt deleted file mode 100644 index 5164c3975a12..000000000000 --- a/requirements/edx-sandbox/py38.txt +++ /dev/null @@ -1,4 +0,0 @@ -# This file is a temporary compatibility wrapper around quince.txt. -# It will be removed before Sumac. - --r releases/quince.txt diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 5e44a3acba34..92a33194504b 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -10,11 +10,11 @@ acid-xblock==0.4.1 # via -r requirements/edx/kernel.in aiohappyeyeballs==2.4.4 # via aiohttp -aiohttp==3.11.9 +aiohttp==3.11.11 # via # geoip2 # openai -aiosignal==1.3.1 +aiosignal==1.3.2 # via aiohttp algoliasearch==3.0.0 # via @@ -37,7 +37,7 @@ asgiref==3.8.1 # django-countries asn1crypto==1.5.1 # via snowflake-connector-python -attrs==24.2.0 +attrs==24.3.0 # via # -r requirements/edx/kernel.in # aiohttp @@ -57,7 +57,9 @@ backoff==1.10.0 bcrypt==4.2.1 # via paramiko beautifulsoup4==4.12.3 - # via pynliner + # via + # openedx-forum + # pynliner billiard==4.2.1 # via celery bleach[css]==6.2.0 @@ -70,13 +72,13 @@ bleach[css]==6.2.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/kernel.in -boto3==1.35.76 +boto3==1.35.93 # via # -r requirements/edx/kernel.in # django-ses # fs-s3fs # ora2 -botocore==1.35.76 +botocore==1.35.93 # via # -r requirements/edx/kernel.in # boto3 @@ -99,9 +101,8 @@ celery==5.4.0 # edx-enterprise # event-tracking # openedx-learning -certifi==2024.8.30 +certifi==2024.12.14 # via - # -r requirements/edx/paver.txt # elasticsearch # py2neo # requests @@ -116,7 +117,6 @@ chardet==5.2.0 charset-normalizer==2.0.12 # via # -c requirements/edx/../constraints.txt - # -r requirements/edx/paver.txt # requests # snowflake-connector-python chem==1.3.0 @@ -138,7 +138,7 @@ click-plugins==1.1.1 # via celery click-repl==0.3.0 # via celery -code-annotations==2.0.0 +code-annotations==2.1.0 # via # edx-enterprise # edx-toggles @@ -234,6 +234,7 @@ django==4.2.17 # openedx-django-wiki # openedx-events # openedx-filters + # openedx-forum # openedx-learning # ora2 # social-auth-app-django @@ -242,7 +243,7 @@ django==4.2.17 # xss-utils django-appconf==1.0.6 # via django-statici18n -django-cache-memoize==0.2.0 +django-cache-memoize==0.2.1 # via edx-enterprise django-celery-results==2.5.1 # via -r requirements/edx/kernel.in @@ -281,7 +282,7 @@ django-ipware==7.0.1 # -r requirements/edx/kernel.in # edx-enterprise # edx-proctoring -django-js-asset==2.2.0 +django-js-asset==3.0.1 # via django-mptt django-method-override==1.0.4 # via -r requirements/edx/kernel.in @@ -318,7 +319,7 @@ django-oauth-toolkit==1.7.1 # edx-enterprise django-object-actions==4.3.0 # via edx-enterprise -django-pipeline==3.1.0 +django-pipeline==4.0.0 # via -r requirements/edx/kernel.in django-push-notifications==3.1.0 # via edx-ace @@ -328,7 +329,7 @@ django-sekizai==4.1.0 # via # -r requirements/edx/kernel.in # openedx-django-wiki -django-ses==4.3.0 +django-ses==4.3.1 # via -r requirements/edx/bundled.in django-simple-history==3.4.0 # via @@ -383,15 +384,14 @@ djangorestframework==3.14.0 # edx-organizations # edx-proctoring # edx-submissions + # openedx-forum # openedx-learning # ora2 # super-csv djangorestframework-xml==2.0.0 # via edx-enterprise dnspython==2.7.0 - # via - # -r requirements/edx/paver.txt - # pymongo + # via pymongo done-xblock==2.4.0 # via -r requirements/edx/bundled.in drf-jwt==1.19.2 @@ -430,7 +430,7 @@ edx-celeryutils==1.3.0 # super-csv edx-codejail==3.5.2 # via -r requirements/edx/kernel.in -edx-completion==4.7.6 +edx-completion==4.7.8 # via -r requirements/edx/kernel.in edx-django-release-util==1.4.0 # via @@ -468,7 +468,7 @@ edx-drf-extensions==10.5.0 # edx-when # edxval # openedx-learning -edx-enterprise==5.4.1 +edx-enterprise==5.5.2 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in @@ -489,7 +489,6 @@ edx-name-affirmation==3.0.1 edx-opaque-keys[django]==2.11.0 # via # -r requirements/edx/kernel.in - # -r requirements/edx/paver.txt # edx-bulk-grades # edx-ccx-keys # edx-completion @@ -516,10 +515,12 @@ edx-rest-api-client==6.0.0 # edx-enterprise # edx-proctoring edx-search==4.1.1 - # via -r requirements/edx/kernel.in + # via + # -r requirements/edx/kernel.in + # openedx-forum edx-sga==0.25.0 # via -r requirements/edx/bundled.in -edx-submissions==3.8.3 +edx-submissions==3.8.4 # via # -r requirements/edx/kernel.in # ora2 @@ -543,12 +544,14 @@ edx-when==2.5.0 # via # -r requirements/edx/kernel.in # edx-proctoring -edxval==2.6.1 +edxval==2.8.0 # via -r requirements/edx/kernel.in elasticsearch==7.9.1 # via # -c requirements/edx/../common_constraints.txt + # -c requirements/edx/../constraints.txt # edx-search + # openedx-forum enmerkar==0.7.1 # via enmerkar-underscore enmerkar-underscore==2.3.1 @@ -560,7 +563,7 @@ event-tracking==3.0.0 # edx-completion # edx-proctoring # edx-search -fastavro==1.9.7 +fastavro==1.10.0 # via openedx-events filelock==3.16.1 # via snowflake-connector-python @@ -586,16 +589,16 @@ geoip2==4.8.1 # via -r requirements/edx/kernel.in glob2==0.7 # via -r requirements/edx/kernel.in -google-api-core[grpc]==2.23.0 +google-api-core[grpc]==2.24.0 # via # firebase-admin # google-api-python-client # google-cloud-core # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.154.0 +google-api-python-client==2.157.0 # via firebase-admin -google-auth==2.36.0 +google-auth==2.37.0 # via # google-api-core # google-api-python-client @@ -623,11 +626,11 @@ googleapis-common-protos==1.66.0 # via # google-api-core # grpcio-status -grpcio==1.68.1 +grpcio==1.69.0 # via # google-api-core # grpcio-status -grpcio-status==1.68.1 +grpcio-status==1.69.0 # via google-api-core gunicorn==23.0.0 # via -r requirements/edx/kernel.in @@ -645,7 +648,6 @@ icalendar==6.1.0 # via -r requirements/edx/kernel.in idna==3.10 # via - # -r requirements/edx/paver.txt # optimizely-sdk # requests # snowflake-connector-python @@ -662,7 +664,7 @@ ipaddress==1.0.23 # via -r requirements/edx/kernel.in isodate==0.7.2 # via python3-saml -jinja2==3.1.4 +jinja2==3.1.5 # via code-annotations jmespath==1.0.1 # via @@ -697,18 +699,13 @@ laboratory==1.0.2 # via -r requirements/edx/kernel.in lazy==1.6 # via - # -r requirements/edx/paver.txt # acid-xblock # lti-consumer-xblock # ora2 # xblock -libsass==0.10.0 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/paver.txt loremipsum==1.0.5 # via ora2 -lti-consumer-xblock==9.12.0 +lti-consumer-xblock==9.12.1 # via -r requirements/edx/kernel.in lxml[html-clean,html_clean]==5.3.0 # via @@ -727,7 +724,7 @@ lxml-html-clean==0.4.1 # via lxml mailsnake==1.6.4 # via -r requirements/edx/bundled.in -mako==1.3.7 +mako==1.3.8 # via # -r requirements/edx/kernel.in # acid-xblock @@ -743,7 +740,6 @@ markdown==3.3.7 # xblock-poll markupsafe==3.0.2 # via - # -r requirements/edx/paver.txt # chem # jinja2 # mako @@ -755,8 +751,6 @@ meilisearch==0.33.0 # via # -r requirements/edx/kernel.in # edx-search -mock==5.1.0 - # via -r requirements/edx/paver.txt mongoengine==0.29.1 # via -r requirements/edx/kernel.in monotonic==1.6 @@ -774,10 +768,12 @@ multidict==6.1.0 # aiohttp # yarl mysqlclient==2.2.6 - # via -r requirements/edx/kernel.in -newrelic==10.3.1 + # via + # -r requirements/edx/kernel.in + # openedx-forum +newrelic==10.4.0 # via edx-django-utils -nh3==0.2.19 +nh3==0.2.20 # via -r requirements/edx/kernel.in nltk==3.9.1 # via chem @@ -804,7 +800,9 @@ openai==0.28.1 # -c requirements/edx/../constraints.txt # edx-enterprise openedx-atlas==0.6.2 - # via -r requirements/edx/kernel.in + # via + # -r requirements/edx/kernel.in + # openedx-forum openedx-calc==4.0.1 # via -r requirements/edx/kernel.in openedx-django-pyfs==3.7.0 @@ -825,11 +823,13 @@ openedx-events==9.15.0 # edx-name-affirmation # event-tracking # ora2 -openedx-filters==1.11.0 +openedx-filters==1.12.0 # via # -r requirements/edx/kernel.in # lti-consumer-xblock # ora2 +openedx-forum==0.1.5 + # via -r requirements/edx/kernel.in openedx-learning==0.18.1 # via # -c requirements/edx/../constraints.txt @@ -840,7 +840,7 @@ optimizely-sdk==4.1.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/bundled.in -ora2==6.14.1 +ora2==6.14.3 # via -r requirements/edx/bundled.in packaging==24.2 # via @@ -856,7 +856,6 @@ path==16.11.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in - # -r requirements/edx/paver.txt # edx-i18n-tools # path-py path-py==12.5.0 @@ -864,17 +863,13 @@ path-py==12.5.0 # edx-enterprise # ora2 # staff-graded-xblock -paver==1.3.4 - # via -r requirements/edx/paver.txt pbr==6.1.0 - # via - # -r requirements/edx/paver.txt - # stevedore + # via stevedore pgpy==0.6.0 # via edx-enterprise piexif==1.1.3 # via -r requirements/edx/kernel.in -pillow==11.0.0 +pillow==11.1.0 # via # -r requirements/edx/kernel.in # edx-enterprise @@ -895,16 +890,16 @@ proto-plus==1.25.0 # via # google-api-core # google-cloud-firestore -protobuf==5.29.1 +protobuf==5.29.2 # via # google-api-core # google-cloud-firestore # googleapis-common-protos # grpcio-status # proto-plus -psutil==6.1.0 +psutil==6.1.1 # via - # -r requirements/edx/paver.txt + # -r requirements/edx/kernel.in # edx-django-utils py2neo @ https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz # via @@ -927,14 +922,12 @@ pycryptodomex==3.21.0 # edx-proctoring # lti-consumer-xblock # pyjwkest -pydantic==2.10.3 +pydantic==2.10.4 # via camel-converter -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via pydantic -pygments==2.18.0 - # via - # -r requirements/edx/bundled.in - # py2neo +pygments==2.19.1 + # via py2neo pyjwkest==1.4.2 # via # -r requirements/edx/kernel.in @@ -957,15 +950,15 @@ pylatexenc==2.10 pylti1p3==2.0.0 # via -r requirements/edx/kernel.in pymemcache==4.0.0 - # via -r requirements/edx/paver.txt + # via -r requirements/edx/kernel.in pymongo==4.4.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in - # -r requirements/edx/paver.txt # edx-opaque-keys # event-tracking # mongoengine + # openedx-forum # openedx-mongodbproxy pynacl==1.5.0 # via @@ -977,7 +970,7 @@ pyopenssl==24.3.0 # via # optimizely-sdk # snowflake-connector-python -pyparsing==3.2.0 +pyparsing==3.2.1 # via # chem # httplib2 @@ -1003,8 +996,6 @@ python-dateutil==2.9.0.post0 # xblock python-ipware==3.0.0 # via django-ipware -python-memcached==1.62 - # via -r requirements/edx/paver.txt python-slugify==8.0.4 # via code-annotations python-swiftclient==4.6.0 @@ -1048,7 +1039,7 @@ random2==1.0.2 # via -r requirements/edx/kernel.in recommender-xblock==3.0.0 # via -r requirements/edx/bundled.in -redis==5.2.0 +redis==5.2.1 # via # -r requirements/edx/kernel.in # walrus @@ -1060,7 +1051,6 @@ regex==2024.11.6 # via nltk requests==2.32.3 # via - # -r requirements/edx/paver.txt # algoliasearch # analytics-python # cachecontrol @@ -1075,6 +1065,7 @@ requests==2.32.3 # mailsnake # meilisearch # openai + # openedx-forum # optimizely-sdk # pyjwkest # pylti1p3 @@ -1105,7 +1096,7 @@ s3transfer==0.10.4 # via boto3 sailthru-client==2.2.3 # via edx-ace -scipy==1.14.1 +scipy==1.15.0 # via # chem # openedx-calc @@ -1123,7 +1114,6 @@ simplejson==3.19.3 six==1.17.0 # via # -r requirements/edx/kernel.in - # -r requirements/edx/paver.txt # analytics-python # codejail-includes # crowdsourcehinter-xblock @@ -1139,9 +1129,7 @@ six==1.17.0 # fs-s3fs # html5lib # interchange - # libsass # optimizely-sdk - # paver # py2neo # pyjwkest # python-dateutil @@ -1172,14 +1160,13 @@ sortedcontainers==2.4.0 # snowflake-connector-python soupsieve==2.6 # via beautifulsoup4 -sqlparse==0.5.2 +sqlparse==0.5.3 # via django staff-graded-xblock==2.3.0 # via -r requirements/edx/bundled.in stevedore==5.4.0 # via # -r requirements/edx/kernel.in - # -r requirements/edx/paver.txt # code-annotations # edx-ace # edx-django-utils @@ -1203,7 +1190,6 @@ tqdm==4.67.1 # openai typing-extensions==4.12.2 # via - # -r requirements/edx/paver.txt # django-countries # edx-opaque-keys # jwcrypto @@ -1227,7 +1213,7 @@ uritemplate==4.1.1 # google-api-python-client urllib3==2.2.3 # via - # -r requirements/edx/paver.txt + # -c requirements/edx/../common_constraints.txt # botocore # elasticsearch # py2neo @@ -1243,8 +1229,6 @@ voluptuous==0.15.2 # via ora2 walrus==0.9.4 # via edx-event-bus-redis -watchdog==6.0.0 - # via -r requirements/edx/paver.txt wcwidth==0.2.13 # via prompt-toolkit web-fragments==2.2.0 @@ -1264,8 +1248,10 @@ webob==1.8.9 # via # -r requirements/edx/kernel.in # xblock +wheel==0.45.1 + # via django-pipeline wrapt==1.17.0 - # via -r requirements/edx/paver.txt + # via -r requirements/edx/kernel.in xblock[django]==5.1.0 # via # -r requirements/edx/kernel.in diff --git a/requirements/edx/bundled.in b/requirements/edx/bundled.in index a9394b809f55..61f6007aa507 100644 --- a/requirements/edx/bundled.in +++ b/requirements/edx/bundled.in @@ -25,7 +25,6 @@ # Follow up issue to remove this fork: https://github.com/openedx/edx-platform/issues/33456 https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz -pygments # Used to support colors in paver command output # i18n_tool is needed at build time for pulling translations edx-i18n-tools>=0.4.6 # Commands for developers and translators to extract, compile and validate translations diff --git a/requirements/edx/coverage.txt b/requirements/edx/coverage.txt index 38acef7c7978..26669aec6cb0 100644 --- a/requirements/edx/coverage.txt +++ b/requirements/edx/coverage.txt @@ -6,15 +6,15 @@ # chardet==5.2.0 # via diff-cover -coverage==7.6.8 +coverage==7.6.10 # via -r requirements/edx/coverage.in -diff-cover==9.2.0 +diff-cover==9.2.1 # via -r requirements/edx/coverage.in -jinja2==3.1.4 +jinja2==3.1.5 # via diff-cover markupsafe==3.0.2 # via jinja2 pluggy==1.5.0 # via diff-cover -pygments==2.18.0 +pygments==2.19.1 # via diff-cover diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 8c9728d408d8..1e81b10f47be 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -21,13 +21,13 @@ aiohappyeyeballs==2.4.4 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # aiohttp -aiohttp==3.11.9 +aiohttp==3.11.11 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # geoip2 # openai -aiosignal==1.3.1 +aiosignal==1.3.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -60,7 +60,7 @@ annotated-types==0.7.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # pydantic -anyio==4.7.0 +anyio==4.8.0 # via # -r requirements/edx/testing.txt # starlette @@ -76,6 +76,7 @@ asgiref==3.8.1 # django # django-cors-headers # django-countries + # django-stubs asn1crypto==1.5.1 # via # -r requirements/edx/doc.txt @@ -89,7 +90,7 @@ astroid==2.13.5 # pylint # pylint-celery # sphinx-autoapi -attrs==24.2.0 +attrs==24.3.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -122,6 +123,7 @@ beautifulsoup4==4.12.3 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt + # openedx-forum # pydata-sphinx-theme # pynliner billiard==4.2.1 @@ -143,14 +145,14 @@ boto==2.49.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -boto3==1.35.76 +boto3==1.35.93 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # django-ses # fs-s3fs # ora2 -botocore==1.35.76 +botocore==1.35.93 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -191,7 +193,7 @@ celery==5.4.0 # edx-enterprise # event-tracking # openedx-learning -certifi==2024.8.30 +certifi==2024.12.14 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -266,7 +268,7 @@ click-repl==0.3.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # celery -code-annotations==2.0.0 +code-annotations==2.1.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -281,7 +283,7 @@ colorama==0.4.6 # via # -r requirements/edx/testing.txt # tox -coverage[toml]==7.6.8 +coverage[toml]==7.6.10 # via # -r requirements/edx/testing.txt # pytest-cov @@ -326,7 +328,7 @@ defusedxml==0.7.1 # ora2 # python3-openid # social-auth-core -diff-cover==9.2.0 +diff-cover==9.2.1 # via -r requirements/edx/testing.txt dill==0.3.9 # via @@ -406,6 +408,7 @@ django==4.2.17 # openedx-django-wiki # openedx-events # openedx-filters + # openedx-forum # openedx-learning # ora2 # social-auth-app-django @@ -417,7 +420,7 @@ django-appconf==1.0.6 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # django-statici18n -django-cache-memoize==0.2.0 +django-cache-memoize==0.2.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -476,7 +479,7 @@ django-ipware==7.0.1 # -r requirements/edx/testing.txt # edx-enterprise # edx-proctoring -django-js-asset==2.2.0 +django-js-asset==3.0.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -529,7 +532,7 @@ django-object-actions==4.3.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-enterprise -django-pipeline==3.1.0 +django-pipeline==4.0.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -547,7 +550,7 @@ django-sekizai==4.1.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # openedx-django-wiki -django-ses==4.3.0 +django-ses==4.3.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -575,7 +578,7 @@ django-storages==1.14.3 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edxval -django-stubs==1.16.0 +django-stubs==5.1.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/development.in @@ -619,13 +622,12 @@ djangorestframework==3.14.0 # edx-organizations # edx-proctoring # edx-submissions + # openedx-forum # openedx-learning # ora2 # super-csv -djangorestframework-stubs==3.14.0 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/development.in +djangorestframework-stubs==3.15.2 + # via -r requirements/edx/development.in djangorestframework-xml==2.0.0 # via # -r requirements/edx/doc.txt @@ -700,7 +702,7 @@ edx-codejail==3.5.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -edx-completion==4.7.6 +edx-completion==4.7.8 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -745,7 +747,7 @@ edx-drf-extensions==10.5.0 # edx-when # edxval # openedx-learning -edx-enterprise==5.4.1 +edx-enterprise==5.5.2 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt @@ -815,11 +817,12 @@ edx-search==4.1.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt + # openedx-forum edx-sga==0.25.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -edx-submissions==3.8.3 +edx-submissions==3.8.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -851,16 +854,18 @@ edx-when==2.5.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-proctoring -edxval==2.6.1 +edxval==2.8.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt elasticsearch==7.9.1 # via # -c requirements/edx/../common_constraints.txt + # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-search + # openedx-forum enmerkar==0.7.1 # via # -r requirements/edx/doc.txt @@ -884,7 +889,7 @@ execnet==2.1.1 # pytest-xdist factory-boy==3.3.1 # via -r requirements/edx/testing.txt -faker==33.1.0 +faker==33.3.0 # via # -r requirements/edx/testing.txt # factory-boy @@ -892,7 +897,7 @@ fastapi==0.115.6 # via # -r requirements/edx/testing.txt # pact-python -fastavro==1.9.7 +fastavro==1.10.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -938,17 +943,17 @@ geoip2==4.8.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -gitdb==4.0.11 +gitdb==4.0.12 # via # -r requirements/edx/doc.txt # gitpython -gitpython==3.1.43 +gitpython==3.1.44 # via -r requirements/edx/doc.txt glob2==0.7 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -google-api-core[grpc]==2.23.0 +google-api-core[grpc]==2.24.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -957,12 +962,12 @@ google-api-core[grpc]==2.23.0 # google-cloud-core # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.154.0 +google-api-python-client==2.157.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # firebase-admin -google-auth==2.36.0 +google-auth==2.37.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1014,13 +1019,13 @@ grimp==3.5 # via # -r requirements/edx/testing.txt # import-linter -grpcio==1.68.1 +grpcio==1.69.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # google-api-core # grpcio-status -grpcio-status==1.68.1 +grpcio-status==1.69.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1101,7 +1106,7 @@ isort==5.13.2 # via # -r requirements/edx/testing.txt # pylint -jinja2==3.1.4 +jinja2==3.1.5 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1179,14 +1184,12 @@ libsass==0.10.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/assets.txt - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt loremipsum==1.0.5 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # ora2 -lti-consumer-xblock==9.12.0 +lti-consumer-xblock==9.12.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1214,7 +1217,7 @@ mailsnake==1.6.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -mako==1.3.7 +mako==1.3.8 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1253,14 +1256,12 @@ meilisearch==0.33.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-search -mistune==3.0.2 +mistune==3.1.0 # via # -r requirements/edx/doc.txt # sphinx-mdinclude mock==5.1.0 - # via - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt + # via -r requirements/edx/testing.txt mongoengine==0.29.1 # via # -r requirements/edx/doc.txt @@ -1292,24 +1293,21 @@ multidict==6.1.0 # -r requirements/edx/testing.txt # aiohttp # yarl -mypy==1.11.2 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/development.in - # django-stubs - # djangorestframework-stubs +mypy==1.14.1 + # via -r requirements/edx/development.in mypy-extensions==1.0.0 # via mypy mysqlclient==2.2.6 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -newrelic==10.3.1 + # openedx-forum +newrelic==10.4.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-django-utils -nh3==0.2.19 +nh3==0.2.20 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1354,6 +1352,7 @@ openedx-atlas==0.6.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt + # openedx-forum openedx-calc==4.0.1 # via # -r requirements/edx/doc.txt @@ -1383,12 +1382,16 @@ openedx-events==9.15.0 # edx-name-affirmation # event-tracking # ora2 -openedx-filters==1.11.0 +openedx-filters==1.12.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # lti-consumer-xblock # ora2 +openedx-forum==0.1.5 + # via + # -r requirements/edx/doc.txt + # -r requirements/edx/testing.txt openedx-learning==0.18.1 # via # -c requirements/edx/../constraints.txt @@ -1403,7 +1406,7 @@ optimizely-sdk==4.1.1 # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -ora2==6.14.1 +ora2==6.14.3 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1421,7 +1424,7 @@ packaging==24.2 # snowflake-connector-python # sphinx # tox -pact-python==2.2.2 +pact-python==2.3.0 # via -r requirements/edx/testing.txt pansi==2024.11.0 # via @@ -1447,10 +1450,6 @@ path-py==12.5.0 # edx-enterprise # ora2 # staff-graded-xblock -paver==1.3.4 - # via - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt pbr==6.1.0 # via # -r requirements/edx/doc.txt @@ -1469,7 +1468,7 @@ piexif==1.1.3 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -pillow==11.0.0 +pillow==11.1.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1515,7 +1514,7 @@ proto-plus==1.25.0 # -r requirements/edx/testing.txt # google-api-core # google-cloud-firestore -protobuf==5.29.1 +protobuf==5.29.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1524,7 +1523,7 @@ protobuf==5.29.1 # googleapis-common-protos # grpcio-status # proto-plus -psutil==6.1.0 +psutil==6.1.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1570,22 +1569,22 @@ pycryptodomex==3.21.0 # edx-proctoring # lti-consumer-xblock # pyjwkest -pydantic==2.10.3 +pydantic==2.10.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # camel-converter # fastapi -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # pydantic -pydata-sphinx-theme==0.16.0 +pydata-sphinx-theme==0.16.1 # via # -r requirements/edx/doc.txt # sphinx-book-theme -pygments==2.18.0 +pygments==2.19.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1659,6 +1658,7 @@ pymongo==4.4.0 # edx-opaque-keys # event-tracking # mongoengine + # openedx-forum # openedx-mongodbproxy pynacl==1.5.0 # via @@ -1676,7 +1676,7 @@ pyopenssl==24.3.0 # -r requirements/edx/testing.txt # optimizely-sdk # snowflake-connector-python -pyparsing==3.2.0 +pyparsing==3.2.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1752,10 +1752,6 @@ python-ipware==3.0.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # django-ipware -python-memcached==1.62 - # via - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt python-slugify==8.0.4 # via # -r requirements/edx/doc.txt @@ -1820,7 +1816,7 @@ recommender-xblock==3.0.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -redis==5.2.0 +redis==5.2.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1855,6 +1851,7 @@ requests==2.32.3 # mailsnake # meilisearch # openai + # openedx-forum # optimizely-sdk # pact-python # pyjwkest @@ -1900,7 +1897,7 @@ sailthru-client==2.2.3 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-ace -scipy==1.14.1 +scipy==1.15.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1949,7 +1946,6 @@ six==1.17.0 # libsass # optimizely-sdk # pact-python - # paver # py2neo # pyjwkest # python-dateutil @@ -1960,7 +1956,7 @@ slumber==0.7.1 # -r requirements/edx/testing.txt # edx-bulk-grades # edx-enterprise -smmap==5.0.1 +smmap==5.0.2 # via # -r requirements/edx/doc.txt # gitdb @@ -2060,7 +2056,7 @@ sphinxcontrib-serializinghtml==2.0.0 # sphinx sphinxext-rediraffe==0.2.7 # via -r requirements/edx/doc.txt -sqlparse==0.5.2 +sqlparse==0.5.3 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -2108,8 +2104,6 @@ tinycss2==1.4.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # bleach -tomli==2.2.1 - # via django-stubs tomlkit==0.13.2 # via # -r requirements/edx/doc.txt @@ -2124,14 +2118,14 @@ tqdm==4.67.1 # -r requirements/edx/testing.txt # nltk # openai -types-pytz==2024.2.0.20241003 - # via django-stubs -types-pyyaml==6.0.12.20240917 +types-pyyaml==6.0.12.20241230 # via # django-stubs # djangorestframework-stubs -types-requests==2.32.0.20241016 +types-requests==2.31.0.6 # via djangorestframework-stubs +types-urllib3==1.26.25.14 + # via types-requests typing-extensions==4.12.2 # via # -r requirements/edx/doc.txt @@ -2176,18 +2170,18 @@ uritemplate==4.1.1 # google-api-python-client urllib3==2.2.3 # via + # -c requirements/edx/../common_constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # botocore # elasticsearch # py2neo # requests - # types-requests user-util==1.1.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -uvicorn==0.32.1 +uvicorn==0.34.0 # via # -r requirements/edx/testing.txt # pact-python @@ -2198,7 +2192,7 @@ vine==5.1.0 # amqp # celery # kombu -virtualenv==20.28.0 +virtualenv==20.28.1 # via # -r requirements/edx/testing.txt # tox @@ -2207,7 +2201,7 @@ voluptuous==0.15.2 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # ora2 -vulture==2.13 +vulture==2.14 # via -r requirements/edx/development.in walrus==0.9.4 # via @@ -2215,10 +2209,7 @@ walrus==0.9.4 # -r requirements/edx/testing.txt # edx-event-bus-redis watchdog==6.0.0 - # via - # -r requirements/edx/development.in - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt + # via -r requirements/edx/development.in wcwidth==0.2.13 # via # -r requirements/edx/doc.txt @@ -2248,6 +2239,9 @@ webob==1.8.9 wheel==0.45.1 # via # -r requirements/edx/../pip-tools.txt + # -r requirements/edx/doc.txt + # -r requirements/edx/testing.txt + # django-pipeline # pip-tools wrapt==1.17.0 # via diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index daa72cd780f0..cf590ac75130 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -14,12 +14,12 @@ aiohappyeyeballs==2.4.4 # via # -r requirements/edx/base.txt # aiohttp -aiohttp==3.11.9 +aiohttp==3.11.11 # via # -r requirements/edx/base.txt # geoip2 # openai -aiosignal==1.3.1 +aiosignal==1.3.2 # via # -r requirements/edx/base.txt # aiohttp @@ -61,7 +61,7 @@ astroid==2.13.5 # via # -c requirements/edx/../constraints.txt # sphinx-autoapi -attrs==24.2.0 +attrs==24.3.0 # via # -r requirements/edx/base.txt # aiohttp @@ -89,6 +89,7 @@ bcrypt==4.2.1 beautifulsoup4==4.12.3 # via # -r requirements/edx/base.txt + # openedx-forum # pydata-sphinx-theme # pynliner billiard==4.2.1 @@ -106,13 +107,13 @@ bleach[css]==6.2.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/base.txt -boto3==1.35.76 +boto3==1.35.93 # via # -r requirements/edx/base.txt # django-ses # fs-s3fs # ora2 -botocore==1.35.76 +botocore==1.35.93 # via # -r requirements/edx/base.txt # boto3 @@ -141,7 +142,7 @@ celery==5.4.0 # edx-enterprise # event-tracking # openedx-learning -certifi==2024.8.30 +certifi==2024.12.14 # via # -r requirements/edx/base.txt # elasticsearch @@ -190,7 +191,7 @@ click-repl==0.3.0 # via # -r requirements/edx/base.txt # celery -code-annotations==2.0.0 +code-annotations==2.1.0 # via # -r requirements/edx/base.txt # -r requirements/edx/doc.in @@ -292,6 +293,7 @@ django==4.2.17 # openedx-django-wiki # openedx-events # openedx-filters + # openedx-forum # openedx-learning # ora2 # social-auth-app-django @@ -302,7 +304,7 @@ django-appconf==1.0.6 # via # -r requirements/edx/base.txt # django-statici18n -django-cache-memoize==0.2.0 +django-cache-memoize==0.2.1 # via # -r requirements/edx/base.txt # edx-enterprise @@ -347,7 +349,7 @@ django-ipware==7.0.1 # -r requirements/edx/base.txt # edx-enterprise # edx-proctoring -django-js-asset==2.2.0 +django-js-asset==3.0.1 # via # -r requirements/edx/base.txt # django-mptt @@ -390,7 +392,7 @@ django-object-actions==4.3.0 # via # -r requirements/edx/base.txt # edx-enterprise -django-pipeline==3.1.0 +django-pipeline==4.0.0 # via -r requirements/edx/base.txt django-push-notifications==3.1.0 # via @@ -402,7 +404,7 @@ django-sekizai==4.1.0 # via # -r requirements/edx/base.txt # openedx-django-wiki -django-ses==4.3.0 +django-ses==4.3.1 # via -r requirements/edx/base.txt django-simple-history==3.4.0 # via @@ -457,6 +459,7 @@ djangorestframework==3.14.0 # edx-organizations # edx-proctoring # edx-submissions + # openedx-forum # openedx-learning # ora2 # super-csv @@ -514,7 +517,7 @@ edx-celeryutils==1.3.0 # super-csv edx-codejail==3.5.2 # via -r requirements/edx/base.txt -edx-completion==4.7.6 +edx-completion==4.7.8 # via -r requirements/edx/base.txt edx-django-release-util==1.4.0 # via @@ -552,7 +555,7 @@ edx-drf-extensions==10.5.0 # edx-when # edxval # openedx-learning -edx-enterprise==5.4.1 +edx-enterprise==5.5.2 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt @@ -601,10 +604,12 @@ edx-rest-api-client==6.0.0 # edx-enterprise # edx-proctoring edx-search==4.1.1 - # via -r requirements/edx/base.txt + # via + # -r requirements/edx/base.txt + # openedx-forum edx-sga==0.25.0 # via -r requirements/edx/base.txt -edx-submissions==3.8.3 +edx-submissions==3.8.4 # via # -r requirements/edx/base.txt # ora2 @@ -630,13 +635,15 @@ edx-when==2.5.0 # via # -r requirements/edx/base.txt # edx-proctoring -edxval==2.6.1 +edxval==2.8.0 # via -r requirements/edx/base.txt elasticsearch==7.9.1 # via # -c requirements/edx/../common_constraints.txt + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # edx-search + # openedx-forum enmerkar==0.7.1 # via # -r requirements/edx/base.txt @@ -650,7 +657,7 @@ event-tracking==3.0.0 # edx-completion # edx-proctoring # edx-search -fastavro==1.9.7 +fastavro==1.10.0 # via # -r requirements/edx/base.txt # openedx-events @@ -683,13 +690,13 @@ future==1.0.0 # pyjwkest geoip2==4.8.1 # via -r requirements/edx/base.txt -gitdb==4.0.11 +gitdb==4.0.12 # via gitpython -gitpython==3.1.43 +gitpython==3.1.44 # via -r requirements/edx/doc.in glob2==0.7 # via -r requirements/edx/base.txt -google-api-core[grpc]==2.23.0 +google-api-core[grpc]==2.24.0 # via # -r requirements/edx/base.txt # firebase-admin @@ -697,11 +704,11 @@ google-api-core[grpc]==2.23.0 # google-cloud-core # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.154.0 +google-api-python-client==2.157.0 # via # -r requirements/edx/base.txt # firebase-admin -google-auth==2.36.0 +google-auth==2.37.0 # via # -r requirements/edx/base.txt # google-api-core @@ -741,12 +748,12 @@ googleapis-common-protos==1.66.0 # -r requirements/edx/base.txt # google-api-core # grpcio-status -grpcio==1.68.1 +grpcio==1.69.0 # via # -r requirements/edx/base.txt # google-api-core # grpcio-status -grpcio-status==1.68.1 +grpcio-status==1.69.0 # via # -r requirements/edx/base.txt # google-api-core @@ -791,7 +798,7 @@ isodate==0.7.2 # via # -r requirements/edx/base.txt # python3-saml -jinja2==3.1.4 +jinja2==3.1.5 # via # -r requirements/edx/base.txt # code-annotations @@ -849,15 +856,11 @@ lazy==1.6 # xblock lazy-object-proxy==1.10.0 # via astroid -libsass==0.10.0 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/base.txt loremipsum==1.0.5 # via # -r requirements/edx/base.txt # ora2 -lti-consumer-xblock==9.12.0 +lti-consumer-xblock==9.12.1 # via -r requirements/edx/base.txt lxml[html-clean]==5.3.0 # via @@ -878,7 +881,7 @@ lxml-html-clean==0.4.1 # lxml mailsnake==1.6.4 # via -r requirements/edx/base.txt -mako==1.3.7 +mako==1.3.8 # via # -r requirements/edx/base.txt # acid-xblock @@ -908,10 +911,8 @@ meilisearch==0.33.0 # via # -r requirements/edx/base.txt # edx-search -mistune==3.0.2 +mistune==3.1.0 # via sphinx-mdinclude -mock==5.1.0 - # via -r requirements/edx/base.txt mongoengine==0.29.1 # via -r requirements/edx/base.txt monotonic==1.6 @@ -937,12 +938,14 @@ multidict==6.1.0 # aiohttp # yarl mysqlclient==2.2.6 - # via -r requirements/edx/base.txt -newrelic==10.3.1 + # via + # -r requirements/edx/base.txt + # openedx-forum +newrelic==10.4.0 # via # -r requirements/edx/base.txt # edx-django-utils -nh3==0.2.19 +nh3==0.2.20 # via -r requirements/edx/base.txt nltk==3.9.1 # via @@ -973,7 +976,9 @@ openai==0.28.1 # -r requirements/edx/base.txt # edx-enterprise openedx-atlas==0.6.2 - # via -r requirements/edx/base.txt + # via + # -r requirements/edx/base.txt + # openedx-forum openedx-calc==4.0.1 # via -r requirements/edx/base.txt openedx-django-pyfs==3.7.0 @@ -995,11 +1000,13 @@ openedx-events==9.15.0 # edx-name-affirmation # event-tracking # ora2 -openedx-filters==1.11.0 +openedx-filters==1.12.0 # via # -r requirements/edx/base.txt # lti-consumer-xblock # ora2 +openedx-forum==0.1.5 + # via -r requirements/edx/base.txt openedx-learning==0.18.1 # via # -c requirements/edx/../constraints.txt @@ -1010,7 +1017,7 @@ optimizely-sdk==4.1.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt -ora2==6.14.1 +ora2==6.14.3 # via -r requirements/edx/base.txt packaging==24.2 # via @@ -1040,8 +1047,6 @@ path-py==12.5.0 # edx-enterprise # ora2 # staff-graded-xblock -paver==1.3.4 - # via -r requirements/edx/base.txt pbr==6.1.0 # via # -r requirements/edx/base.txt @@ -1054,7 +1059,7 @@ picobox==4.0.0 # via sphinxcontrib-openapi piexif==1.1.3 # via -r requirements/edx/base.txt -pillow==11.0.0 +pillow==11.1.0 # via # -r requirements/edx/base.txt # edx-enterprise @@ -1083,7 +1088,7 @@ proto-plus==1.25.0 # -r requirements/edx/base.txt # google-api-core # google-cloud-firestore -protobuf==5.29.1 +protobuf==5.29.2 # via # -r requirements/edx/base.txt # google-api-core @@ -1091,7 +1096,7 @@ protobuf==5.29.1 # googleapis-common-protos # grpcio-status # proto-plus -psutil==6.1.0 +psutil==6.1.1 # via # -r requirements/edx/base.txt # edx-django-utils @@ -1121,17 +1126,17 @@ pycryptodomex==3.21.0 # edx-proctoring # lti-consumer-xblock # pyjwkest -pydantic==2.10.3 +pydantic==2.10.4 # via # -r requirements/edx/base.txt # camel-converter -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via # -r requirements/edx/base.txt # pydantic -pydata-sphinx-theme==0.16.0 +pydata-sphinx-theme==0.16.1 # via sphinx-book-theme -pygments==2.18.0 +pygments==2.19.1 # via # -r requirements/edx/base.txt # accessible-pygments @@ -1171,6 +1176,7 @@ pymongo==4.4.0 # edx-opaque-keys # event-tracking # mongoengine + # openedx-forum # openedx-mongodbproxy pynacl==1.5.0 # via @@ -1184,7 +1190,7 @@ pyopenssl==24.3.0 # -r requirements/edx/base.txt # optimizely-sdk # snowflake-connector-python -pyparsing==3.2.0 +pyparsing==3.2.1 # via # -r requirements/edx/base.txt # chem @@ -1215,8 +1221,6 @@ python-ipware==3.0.0 # via # -r requirements/edx/base.txt # django-ipware -python-memcached==1.62 - # via -r requirements/edx/base.txt python-slugify==8.0.4 # via # -r requirements/edx/base.txt @@ -1266,7 +1270,7 @@ random2==1.0.2 # via -r requirements/edx/base.txt recommender-xblock==3.0.0 # via -r requirements/edx/base.txt -redis==5.2.0 +redis==5.2.1 # via # -r requirements/edx/base.txt # walrus @@ -1296,6 +1300,7 @@ requests==2.32.3 # mailsnake # meilisearch # openai + # openedx-forum # optimizely-sdk # pyjwkest # pylti1p3 @@ -1334,7 +1339,7 @@ sailthru-client==2.2.3 # via # -r requirements/edx/base.txt # edx-ace -scipy==1.14.1 +scipy==1.15.0 # via # -r requirements/edx/base.txt # chem @@ -1370,9 +1375,7 @@ six==1.17.0 # fs-s3fs # html5lib # interchange - # libsass # optimizely-sdk - # paver # py2neo # pyjwkest # python-dateutil @@ -1382,7 +1385,7 @@ slumber==0.7.1 # -r requirements/edx/base.txt # edx-bulk-grades # edx-enterprise -smmap==5.0.1 +smmap==5.0.2 # via gitdb snowballstemmer==2.2.0 # via sphinx @@ -1452,7 +1455,7 @@ sphinxcontrib-serializinghtml==2.0.0 # via sphinx sphinxext-rediraffe==0.2.7 # via -r requirements/edx/doc.in -sqlparse==0.5.2 +sqlparse==0.5.3 # via # -r requirements/edx/base.txt # django @@ -1524,6 +1527,7 @@ uritemplate==4.1.1 # google-api-python-client urllib3==2.2.3 # via + # -c requirements/edx/../common_constraints.txt # -r requirements/edx/base.txt # botocore # elasticsearch @@ -1545,8 +1549,6 @@ walrus==0.9.4 # via # -r requirements/edx/base.txt # edx-event-bus-redis -watchdog==6.0.0 - # via -r requirements/edx/base.txt wcwidth==0.2.13 # via # -r requirements/edx/base.txt @@ -1569,6 +1571,10 @@ webob==1.8.9 # via # -r requirements/edx/base.txt # xblock +wheel==0.45.1 + # via + # -r requirements/edx/base.txt + # django-pipeline wrapt==1.17.0 # via # -r requirements/edx/base.txt diff --git a/requirements/edx/kernel.in b/requirements/edx/kernel.in index 7323c243accf..d2ec04314801 100644 --- a/requirements/edx/kernel.in +++ b/requirements/edx/kernel.in @@ -3,7 +3,6 @@ -c ../constraints.txt -r github.in # Forks and other dependencies not yet on PyPI --r paver.txt # Requirements for running paver commands # DON'T JUST ADD NEW DEPENDENCIES!!! # Please follow these guidelines whenever you change this file: @@ -119,12 +118,14 @@ openedx-calc # Library supporting mathematical calculatio openedx-django-require openedx-events # Open edX Events from Hooks Extension Framework (OEP-50) openedx-filters # Open edX Filters from Hooks Extension Framework (OEP-50) +openedx-forum # Open edX forum v2 application openedx-learning # Open edX Learning core (experimental) openedx-mongodbproxy openedx-django-wiki path piexif # Exif image metadata manipulation, used in the profile_images app Pillow # Image manipulation library; used for course assets, profile images, invoice PDFs, etc. +psutil # Library for retrieving information on running processes and system utilization pycountry pycryptodomex pyjwkest @@ -132,6 +133,7 @@ pyjwkest # PyJWT 1.6.3 contains PyJWTError, which is required by Apple auth in social-auth-core PyJWT>=1.6.3 pylti1p3 # Required by content_libraries core library to support LTI 1.3 launches +pymemcache # Python interface to the memcached memory cache daemon pymongo # MongoDB driver pynliner # Inlines CSS styles into HTML for email notifications python-dateutil @@ -158,5 +160,6 @@ unicodecsv # Easier support for CSV files with unicode user-util # Functionality for retiring users (GDPR compliance) webob web-fragments # Provides the ability to render fragments of web pages +wrapt # Better functools.wrapped. TODO: functools has since improved, maybe we can switch? XBlock[django] # Courseware component architecture xss-utils # https://github.com/openedx/edx-platform/pull/20633 Fix XSS via Translations diff --git a/requirements/edx/paver.in b/requirements/edx/paver.in deleted file mode 100644 index 6987ede82275..000000000000 --- a/requirements/edx/paver.in +++ /dev/null @@ -1,27 +0,0 @@ -# Requirements to run and test Paver -# -# DON'T JUST ADD NEW DEPENDENCIES!!! -# -# If you open a pull request that adds a new dependency, you should: -# * verify that the dependency has a license compatible with AGPLv3 -# * confirm that it has no system requirements beyond what we already install -# * run "make upgrade" to update the detailed requirements files -# - --c ../constraints.txt - -edx-opaque-keys # Create and introspect course and xblock identities -lazy # Lazily-evaluated attributes for Python objects -libsass # Python bindings for the LibSass CSS compiler -markupsafe # XML/HTML/XHTML Markup safe strings -mock # Stub out code with mock objects and make assertions about how they have been used -path # Easier manipulation of filesystem paths -paver # Build, distribution and deployment scripting tool -psutil # Library for retrieving information on running processes and system utilization -pymongo # via edx-opaque-keys -python-memcached # Python interface to the memcached memory cache daemon -pymemcache # Python interface to the memcached memory cache daemon -requests # Simple interface for making HTTP requests -stevedore # Support for runtime plugins, used for XBlocks and edx-platform Django app plugins -watchdog # Used in paver watch_assets -wrapt # Decorator utilities used in the @timed paver task decorator diff --git a/requirements/edx/paver.txt b/requirements/edx/paver.txt deleted file mode 100644 index c9ee8f3aff49..000000000000 --- a/requirements/edx/paver.txt +++ /dev/null @@ -1,65 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# make upgrade -# -certifi==2024.8.30 - # via requests -charset-normalizer==2.0.12 - # via - # -c requirements/edx/../constraints.txt - # requests -dnspython==2.7.0 - # via pymongo -edx-opaque-keys==2.11.0 - # via -r requirements/edx/paver.in -idna==3.10 - # via requests -lazy==1.6 - # via -r requirements/edx/paver.in -libsass==0.10.0 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/paver.in -markupsafe==3.0.2 - # via -r requirements/edx/paver.in -mock==5.1.0 - # via -r requirements/edx/paver.in -path==16.11.0 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/paver.in -paver==1.3.4 - # via -r requirements/edx/paver.in -pbr==6.1.0 - # via stevedore -psutil==6.1.0 - # via -r requirements/edx/paver.in -pymemcache==4.0.0 - # via -r requirements/edx/paver.in -pymongo==4.4.0 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/paver.in - # edx-opaque-keys -python-memcached==1.62 - # via -r requirements/edx/paver.in -requests==2.32.3 - # via -r requirements/edx/paver.in -six==1.17.0 - # via - # libsass - # paver -stevedore==5.4.0 - # via - # -r requirements/edx/paver.in - # edx-opaque-keys -typing-extensions==4.12.2 - # via edx-opaque-keys -urllib3==2.2.3 - # via requests -watchdog==6.0.0 - # via -r requirements/edx/paver.in -wrapt==1.17.0 - # via -r requirements/edx/paver.in diff --git a/requirements/edx/semgrep.txt b/requirements/edx/semgrep.txt index c244159342bc..d7386250db56 100644 --- a/requirements/edx/semgrep.txt +++ b/requirements/edx/semgrep.txt @@ -4,7 +4,7 @@ # # make upgrade # -attrs==24.2.0 +attrs==24.3.0 # via # glom # jsonschema @@ -17,7 +17,7 @@ boltons==21.0.0 # semgrep bracex==2.5.post1 # via wcmatch -certifi==2024.8.30 +certifi==2024.12.14 # via requests charset-normalizer==2.0.12 # via @@ -96,7 +96,7 @@ protobuf==4.25.5 # via # googleapis-common-protos # opentelemetry-proto -pygments==2.18.0 +pygments==2.19.1 # via rich referencing==0.35.1 # via @@ -112,11 +112,11 @@ rpds-py==0.22.3 # via # jsonschema # referencing -ruamel-yaml==0.17.40 +ruamel-yaml==0.18.10 # via semgrep ruamel-yaml-clib==0.2.12 # via ruamel-yaml -semgrep==1.97.0 +semgrep==1.101.0 # via -r requirements/edx/semgrep.in tomli==2.0.2 # via semgrep @@ -126,6 +126,7 @@ typing-extensions==4.12.2 # semgrep urllib3==2.2.3 # via + # -c requirements/edx/../common_constraints.txt # requests # semgrep wcmatch==8.5.2 diff --git a/requirements/edx/testing.in b/requirements/edx/testing.in index b903768f4de6..cf57aeb0fc46 100644 --- a/requirements/edx/testing.in +++ b/requirements/edx/testing.in @@ -28,6 +28,7 @@ freezegun # Allows tests to mock the output of assorted datetime httpretty # Library for mocking HTTP requests, used in many tests import-linter # Tool for making assertions about which modules can import which others isort # For checking and fixing the order of imports +mock # Deprecated alias to standard library `unittest.mock` pycodestyle # Checker for compliance with the Python style guide (PEP 8) polib # Library for manipulating gettext translation files, used to test paver i18n commands pyquery # jQuery-like API for retrieving fragments of HTML and XML files in tests diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 93511523f22d..d4ee1a6c4f65 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -12,12 +12,12 @@ aiohappyeyeballs==2.4.4 # via # -r requirements/edx/base.txt # aiohttp -aiohttp==3.11.9 +aiohttp==3.11.11 # via # -r requirements/edx/base.txt # geoip2 # openai -aiosignal==1.3.1 +aiosignal==1.3.2 # via # -r requirements/edx/base.txt # aiohttp @@ -39,7 +39,7 @@ annotated-types==0.7.0 # via # -r requirements/edx/base.txt # pydantic -anyio==4.7.0 +anyio==4.8.0 # via starlette appdirs==1.4.4 # via @@ -60,7 +60,7 @@ astroid==2.13.5 # -c requirements/edx/../constraints.txt # pylint # pylint-celery -attrs==24.2.0 +attrs==24.3.0 # via # -r requirements/edx/base.txt # aiohttp @@ -87,6 +87,7 @@ beautifulsoup4==4.12.3 # via # -r requirements/edx/base.txt # -r requirements/edx/testing.in + # openedx-forum # pynliner billiard==4.2.1 # via @@ -103,13 +104,13 @@ bleach[css]==6.2.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/base.txt -boto3==1.35.76 +boto3==1.35.93 # via # -r requirements/edx/base.txt # django-ses # fs-s3fs # ora2 -botocore==1.35.76 +botocore==1.35.93 # via # -r requirements/edx/base.txt # boto3 @@ -139,7 +140,7 @@ celery==5.4.0 # edx-enterprise # event-tracking # openedx-learning -certifi==2024.8.30 +certifi==2024.12.14 # via # -r requirements/edx/base.txt # elasticsearch @@ -199,7 +200,7 @@ click-repl==0.3.0 # via # -r requirements/edx/base.txt # celery -code-annotations==2.0.0 +code-annotations==2.1.0 # via # -r requirements/edx/base.txt # -r requirements/edx/testing.in @@ -210,7 +211,7 @@ codejail-includes==1.0.0 # via -r requirements/edx/base.txt colorama==0.4.6 # via tox -coverage[toml]==7.6.8 +coverage[toml]==7.6.10 # via # -r requirements/edx/coverage.txt # pytest-cov @@ -246,7 +247,7 @@ defusedxml==0.7.1 # ora2 # python3-openid # social-auth-core -diff-cover==9.2.0 +diff-cover==9.2.1 # via -r requirements/edx/coverage.txt dill==0.3.9 # via pylint @@ -318,6 +319,7 @@ django==4.2.17 # openedx-django-wiki # openedx-events # openedx-filters + # openedx-forum # openedx-learning # ora2 # social-auth-app-django @@ -328,7 +330,7 @@ django-appconf==1.0.6 # via # -r requirements/edx/base.txt # django-statici18n -django-cache-memoize==0.2.0 +django-cache-memoize==0.2.1 # via # -r requirements/edx/base.txt # edx-enterprise @@ -373,7 +375,7 @@ django-ipware==7.0.1 # -r requirements/edx/base.txt # edx-enterprise # edx-proctoring -django-js-asset==2.2.0 +django-js-asset==3.0.1 # via # -r requirements/edx/base.txt # django-mptt @@ -416,7 +418,7 @@ django-object-actions==4.3.0 # via # -r requirements/edx/base.txt # edx-enterprise -django-pipeline==3.1.0 +django-pipeline==4.0.0 # via -r requirements/edx/base.txt django-push-notifications==3.1.0 # via @@ -428,7 +430,7 @@ django-sekizai==4.1.0 # via # -r requirements/edx/base.txt # openedx-django-wiki -django-ses==4.3.0 +django-ses==4.3.1 # via -r requirements/edx/base.txt django-simple-history==3.4.0 # via @@ -483,6 +485,7 @@ djangorestframework==3.14.0 # edx-organizations # edx-proctoring # edx-submissions + # openedx-forum # openedx-learning # ora2 # super-csv @@ -535,7 +538,7 @@ edx-celeryutils==1.3.0 # super-csv edx-codejail==3.5.2 # via -r requirements/edx/base.txt -edx-completion==4.7.6 +edx-completion==4.7.8 # via -r requirements/edx/base.txt edx-django-release-util==1.4.0 # via @@ -573,7 +576,7 @@ edx-drf-extensions==10.5.0 # edx-when # edxval # openedx-learning -edx-enterprise==5.4.1 +edx-enterprise==5.5.2 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt @@ -624,10 +627,12 @@ edx-rest-api-client==6.0.0 # edx-enterprise # edx-proctoring edx-search==4.1.1 - # via -r requirements/edx/base.txt + # via + # -r requirements/edx/base.txt + # openedx-forum edx-sga==0.25.0 # via -r requirements/edx/base.txt -edx-submissions==3.8.3 +edx-submissions==3.8.4 # via # -r requirements/edx/base.txt # ora2 @@ -653,13 +658,15 @@ edx-when==2.5.0 # via # -r requirements/edx/base.txt # edx-proctoring -edxval==2.6.1 +edxval==2.8.0 # via -r requirements/edx/base.txt elasticsearch==7.9.1 # via # -c requirements/edx/../common_constraints.txt + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # edx-search + # openedx-forum enmerkar==0.7.1 # via # -r requirements/edx/base.txt @@ -677,11 +684,11 @@ execnet==2.1.1 # via pytest-xdist factory-boy==3.3.1 # via -r requirements/edx/testing.in -faker==33.1.0 +faker==33.3.0 # via factory-boy fastapi==0.115.6 # via pact-python -fastavro==1.9.7 +fastavro==1.10.0 # via # -r requirements/edx/base.txt # openedx-events @@ -720,7 +727,7 @@ geoip2==4.8.1 # via -r requirements/edx/base.txt glob2==0.7 # via -r requirements/edx/base.txt -google-api-core[grpc]==2.23.0 +google-api-core[grpc]==2.24.0 # via # -r requirements/edx/base.txt # firebase-admin @@ -728,11 +735,11 @@ google-api-core[grpc]==2.23.0 # google-cloud-core # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.154.0 +google-api-python-client==2.157.0 # via # -r requirements/edx/base.txt # firebase-admin -google-auth==2.36.0 +google-auth==2.37.0 # via # -r requirements/edx/base.txt # google-api-core @@ -774,12 +781,12 @@ googleapis-common-protos==1.66.0 # grpcio-status grimp==3.5 # via import-linter -grpcio==1.68.1 +grpcio==1.69.0 # via # -r requirements/edx/base.txt # google-api-core # grpcio-status -grpcio-status==1.68.1 +grpcio-status==1.69.0 # via # -r requirements/edx/base.txt # google-api-core @@ -835,7 +842,7 @@ isort==5.13.2 # via # -r requirements/edx/testing.in # pylint -jinja2==3.1.4 +jinja2==3.1.5 # via # -r requirements/edx/base.txt # -r requirements/edx/coverage.txt @@ -892,15 +899,11 @@ lazy==1.6 # xblock lazy-object-proxy==1.10.0 # via astroid -libsass==0.10.0 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/base.txt loremipsum==1.0.5 # via # -r requirements/edx/base.txt # ora2 -lti-consumer-xblock==9.12.0 +lti-consumer-xblock==9.12.1 # via -r requirements/edx/base.txt lxml[html-clean]==5.3.0 # via @@ -922,7 +925,7 @@ lxml-html-clean==0.4.1 # lxml mailsnake==1.6.4 # via -r requirements/edx/base.txt -mako==1.3.7 +mako==1.3.8 # via # -r requirements/edx/base.txt # acid-xblock @@ -956,7 +959,7 @@ meilisearch==0.33.0 # -r requirements/edx/base.txt # edx-search mock==5.1.0 - # via -r requirements/edx/base.txt + # via -r requirements/edx/testing.in mongoengine==0.29.1 # via -r requirements/edx/base.txt monotonic==1.6 @@ -982,12 +985,14 @@ multidict==6.1.0 # aiohttp # yarl mysqlclient==2.2.6 - # via -r requirements/edx/base.txt -newrelic==10.3.1 + # via + # -r requirements/edx/base.txt + # openedx-forum +newrelic==10.4.0 # via # -r requirements/edx/base.txt # edx-django-utils -nh3==0.2.19 +nh3==0.2.20 # via -r requirements/edx/base.txt nltk==3.9.1 # via @@ -1018,7 +1023,9 @@ openai==0.28.1 # -r requirements/edx/base.txt # edx-enterprise openedx-atlas==0.6.2 - # via -r requirements/edx/base.txt + # via + # -r requirements/edx/base.txt + # openedx-forum openedx-calc==4.0.1 # via -r requirements/edx/base.txt openedx-django-pyfs==3.7.0 @@ -1040,11 +1047,13 @@ openedx-events==9.15.0 # edx-name-affirmation # event-tracking # ora2 -openedx-filters==1.11.0 +openedx-filters==1.12.0 # via # -r requirements/edx/base.txt # lti-consumer-xblock # ora2 +openedx-forum==0.1.5 + # via -r requirements/edx/base.txt openedx-learning==0.18.1 # via # -c requirements/edx/../constraints.txt @@ -1055,7 +1064,7 @@ optimizely-sdk==4.1.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt -ora2==6.14.1 +ora2==6.14.3 # via -r requirements/edx/base.txt packaging==24.2 # via @@ -1067,7 +1076,7 @@ packaging==24.2 # pytest # snowflake-connector-python # tox -pact-python==2.2.2 +pact-python==2.3.0 # via -r requirements/edx/testing.in pansi==2024.11.0 # via @@ -1089,8 +1098,6 @@ path-py==12.5.0 # edx-enterprise # ora2 # staff-graded-xblock -paver==1.3.4 - # via -r requirements/edx/base.txt pbr==6.1.0 # via # -r requirements/edx/base.txt @@ -1101,7 +1108,7 @@ pgpy==0.6.0 # edx-enterprise piexif==1.1.3 # via -r requirements/edx/base.txt -pillow==11.0.0 +pillow==11.1.0 # via # -r requirements/edx/base.txt # edx-enterprise @@ -1140,7 +1147,7 @@ proto-plus==1.25.0 # -r requirements/edx/base.txt # google-api-core # google-cloud-firestore -protobuf==5.29.1 +protobuf==5.29.2 # via # -r requirements/edx/base.txt # google-api-core @@ -1148,7 +1155,7 @@ protobuf==5.29.1 # googleapis-common-protos # grpcio-status # proto-plus -psutil==6.1.0 +psutil==6.1.1 # via # -r requirements/edx/base.txt # edx-django-utils @@ -1186,16 +1193,16 @@ pycryptodomex==3.21.0 # edx-proctoring # lti-consumer-xblock # pyjwkest -pydantic==2.10.3 +pydantic==2.10.4 # via # -r requirements/edx/base.txt # camel-converter # fastapi -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via # -r requirements/edx/base.txt # pydantic -pygments==2.18.0 +pygments==2.19.1 # via # -r requirements/edx/base.txt # -r requirements/edx/coverage.txt @@ -1251,6 +1258,7 @@ pymongo==4.4.0 # edx-opaque-keys # event-tracking # mongoengine + # openedx-forum # openedx-mongodbproxy pynacl==1.5.0 # via @@ -1264,7 +1272,7 @@ pyopenssl==24.3.0 # -r requirements/edx/base.txt # optimizely-sdk # snowflake-connector-python -pyparsing==3.2.0 +pyparsing==3.2.1 # via # -r requirements/edx/base.txt # chem @@ -1328,8 +1336,6 @@ python-ipware==3.0.0 # via # -r requirements/edx/base.txt # django-ipware -python-memcached==1.62 - # via -r requirements/edx/base.txt python-slugify==8.0.4 # via # -r requirements/edx/base.txt @@ -1377,7 +1383,7 @@ random2==1.0.2 # via -r requirements/edx/base.txt recommender-xblock==3.0.0 # via -r requirements/edx/base.txt -redis==5.2.0 +redis==5.2.1 # via # -r requirements/edx/base.txt # walrus @@ -1407,6 +1413,7 @@ requests==2.32.3 # mailsnake # meilisearch # openai + # openedx-forum # optimizely-sdk # pact-python # pyjwkest @@ -1445,7 +1452,7 @@ sailthru-client==2.2.3 # via # -r requirements/edx/base.txt # edx-ace -scipy==1.14.1 +scipy==1.15.0 # via # -r requirements/edx/base.txt # chem @@ -1484,10 +1491,8 @@ six==1.17.0 # fs-s3fs # html5lib # interchange - # libsass # optimizely-sdk # pact-python - # paver # py2neo # pyjwkest # python-dateutil @@ -1524,7 +1529,7 @@ soupsieve==2.6 # via # -r requirements/edx/base.txt # beautifulsoup4 -sqlparse==0.5.2 +sqlparse==0.5.3 # via # -r requirements/edx/base.txt # django @@ -1608,6 +1613,7 @@ uritemplate==4.1.1 # google-api-python-client urllib3==2.2.3 # via + # -c requirements/edx/../common_constraints.txt # -r requirements/edx/base.txt # botocore # elasticsearch @@ -1615,7 +1621,7 @@ urllib3==2.2.3 # requests user-util==1.1.0 # via -r requirements/edx/base.txt -uvicorn==0.32.1 +uvicorn==0.34.0 # via pact-python vine==5.1.0 # via @@ -1623,7 +1629,7 @@ vine==5.1.0 # amqp # celery # kombu -virtualenv==20.28.0 +virtualenv==20.28.1 # via tox voluptuous==0.15.2 # via @@ -1633,8 +1639,6 @@ walrus==0.9.4 # via # -r requirements/edx/base.txt # edx-event-bus-redis -watchdog==6.0.0 - # via -r requirements/edx/base.txt wcwidth==0.2.13 # via # -r requirements/edx/base.txt @@ -1657,6 +1661,10 @@ webob==1.8.9 # via # -r requirements/edx/base.txt # xblock +wheel==0.45.1 + # via + # -r requirements/edx/base.txt + # django-pipeline wrapt==1.17.0 # via # -r requirements/edx/base.txt diff --git a/requirements/pip.txt b/requirements/pip.txt index 0bdb9d7ac387..ea16b6b4de8b 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -12,5 +12,5 @@ pip==24.2 # via # -c requirements/common_constraints.txt # -r requirements/pip.in -setuptools==75.6.0 +setuptools==75.7.0 # via -r requirements/pip.in diff --git a/scripts/eslint.py b/scripts/eslint.py new file mode 100644 index 000000000000..ad5fbf3f3f1b --- /dev/null +++ b/scripts/eslint.py @@ -0,0 +1,78 @@ +""" # pylint: disable=django-not-configured +Check code quality using eslint. +""" + +import re +import subprocess +import shlex +import sys + + +class BuildFailure(Exception): + pass + + +def fail_quality(message): + """ + Fail the specified quality check. + """ + + raise BuildFailure(message) + + +def run_eslint(): + """ + Runs eslint on static asset directories. + If limit option is passed, fails build if more violations than the limit are found. + """ + violations_limit = 734 + + command = [ + "node", + "--max_old_space_size=4096", + "node_modules/.bin/eslint", + "--ext", ".js", + "--ext", ".jsx", + "--format=compact", + "lms", + "cms", + "common", + "openedx", + "xmodule", + ] + print("Running command:", shlex.join(command)) + result = subprocess.run( + command, + text=True, + check=False, + capture_output=True + ) + + print(result.stdout) + if result.returncode == 0: + fail_quality("No eslint violations found. This is unexpected... are you sure eslint is running correctly?") + elif result.returncode == 1: + last_line = result.stdout.strip().splitlines()[-1] if result.stdout.strip().splitlines() else "" + regex = r'^\d+' + try: + num_violations = int(re.search(regex, last_line).group(0)) if last_line else 0 + # Fail if number of violations is greater than the limit + if num_violations > violations_limit: + fail_quality("FAILURE: Too many eslint violations ({count}).\nThe limit is {violations_limit}.".format(count=num_violations, violations_limit=violations_limit)) + else: + print(f"successfully run eslint with '{num_violations}' violations") + + # An AttributeError will occur if the regex finds no matches. + except (AttributeError, ValueError): + fail_quality(f"FAILURE: Number of eslint violations could not be found in '{last_line}'") + else: + print(f"Unexpected ESLint failure with exit code {result.returncode}.") + fail_quality(f"Unexpected error: {result.stderr.strip()}") + + +if __name__ == "__main__": + try: + run_eslint() + except BuildFailure as e: + print(e) + sys.exit(1) diff --git a/scripts/generic-ci-tests.sh b/scripts/generic-ci-tests.sh deleted file mode 100755 index 54b9cbb9d500..000000000000 --- a/scripts/generic-ci-tests.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env bash -set -e - -############################################################################### -# -# generic-ci-tests.sh -# -# Execute some tests for edx-platform. -# (Most other tests are run by invoking `pytest`, `pylint`, etc. directly) -# -# This script can be called from CI jobs that define -# these environment variables: -# -# `TEST_SUITE` defines which kind of test to run. -# Possible values are: -# -# - "quality": Run the quality (pycodestyle/pylint) checks -# - "js-unit": Run the JavaScript tests -# - "pavelib-js-unit": Run the JavaScript tests and the Python unit -# tests from the pavelib/lib directory -# -############################################################################### - -# Clean up previous builds -git clean -qxfd - -function emptyxunit { - - cat > "reports/$1.xml" < - - - -END - -} - -# if specified tox environment is supported, prepend paver commands -# with tox env invocation -if [ -z ${TOX_ENV+x} ] || [[ ${TOX_ENV} == 'null' ]]; then - echo "TOX_ENV: ${TOX_ENV}" - TOX="" -elif tox -l |grep -q "${TOX_ENV}"; then - if [[ "${TOX_ENV}" == 'quality' ]]; then - TOX="" - else - TOX="tox -r -e ${TOX_ENV} --" - fi -else - echo "${TOX_ENV} is not currently supported. Please review the" - echo "tox.ini file to see which environments are supported" - exit 1 -fi - -PAVER_ARGS="-v" -export SUBSET_JOB=$JOB_NAME - -function run_paver_quality { - QUALITY_TASK=$1 - shift - mkdir -p test_root/log/ - LOG_PREFIX="test_root/log/$QUALITY_TASK" - $TOX paver "$QUALITY_TASK" "$@" 2> "$LOG_PREFIX.err.log" > "$LOG_PREFIX.out.log" || { - echo "STDOUT (last 100 lines of $LOG_PREFIX.out.log):"; - tail -n 100 "$LOG_PREFIX.out.log" - echo "STDERR (last 100 lines of $LOG_PREFIX.err.log):"; - tail -n 100 "$LOG_PREFIX.err.log" - return 1; - } - return 0; -} - -case "$TEST_SUITE" in - - "quality") - EXIT=0 - - mkdir -p reports - - echo "Finding pycodestyle violations and storing report..." - run_paver_quality run_pep8 || { EXIT=1; } - echo "Finding ESLint violations and storing report..." - run_paver_quality run_eslint -l "$ESLINT_THRESHOLD" || { EXIT=1; } - echo "Finding Stylelint violations and storing report..." - run_paver_quality run_stylelint || { EXIT=1; } - echo "Running xss linter report." - run_paver_quality run_xsslint -t "$XSSLINT_THRESHOLDS" || { EXIT=1; } - echo "Running PII checker on all Django models..." - run_paver_quality run_pii_check || { EXIT=1; } - echo "Running reserved keyword checker on all Django models..." - run_paver_quality check_keywords || { EXIT=1; } - - # Need to create an empty test result so the post-build - # action doesn't fail the build. - emptyxunit "stub" - exit "$EXIT" - ;; - - "js-unit") - $TOX paver test_js --coverage - $TOX paver diff_coverage - ;; - - "pavelib-js-unit") - EXIT=0 - $TOX paver test_js --coverage --skip-clean || { EXIT=1; } - paver test_lib --skip-clean $PAVER_ARGS || { EXIT=1; } - - # This is to ensure that the build status of the shard is properly set. - # Because we are running two paver commands in a row, we need to capture - # their return codes in order to exit with a non-zero code if either of - # them fail. We put the || clause there because otherwise, when a paver - # command fails, this entire script will exit, and not run the second - # paver command in this case statement. So instead of exiting, the value - # of a variable named EXIT will be set to 1 if either of the paver - # commands fail. We then use this variable's value as our exit code. - # Note that by default the value of this variable EXIT is not set, so if - # neither command fails then the exit command resolves to simply exit - # which is considered successful. - exit "$EXIT" - ;; -esac diff --git a/scripts/paver_autocomplete.sh b/scripts/paver_autocomplete.sh deleted file mode 100644 index 8b4e8111411c..000000000000 --- a/scripts/paver_autocomplete.sh +++ /dev/null @@ -1,89 +0,0 @@ -# shellcheck disable=all -# ^ Paver in edx-platform is on the way out -# (https://github.com/openedx/edx-platform/issues/31798) -# so we're not going to bother fixing these shellcheck -# violations. - -# Courtesy of Gregory Nicholas - -_subcommand_opts() -{ - local awkfile command cur usage - command=$1 - cur=${COMP_WORDS[COMP_CWORD]} - awkfile=/tmp/paver-option-awkscript-$$.awk - echo ' -BEGIN { - opts = ""; -} - -{ - for (i = 1; i <= NF; i = i + 1) { - # Match short options (-a, -S, -3) - # or long options (--long-option, --another_option) - # in output from paver help [subcommand] - if ($i ~ /^(-[A-Za-z0-9]|--[A-Za-z][A-Za-z0-9_-]*)/) { - opt = $i; - # remove trailing , and = characters. - match(opt, "[,=]"); - if (RSTART > 0) { - opt = substr(opt, 0, RSTART); - } - opts = opts " " opt; - } - } -} - -END { - print opts -}' > $awkfile - - usage=`paver help $command` - options=`echo "$usage"|awk -f $awkfile` - - COMPREPLY=( $(compgen -W "$options" -- "$cur") ) -} - - -_paver() -{ - local cur prev - COMPREPLY=() - # Variable to hold the current word - cur="${COMP_WORDS[COMP_CWORD]}" - prev="${COMP_WORDS[COMP_CWORD - 1]}" - - # Build a list of the available tasks from: `paver --help --quiet` - local cmds=$(paver -hq | awk '/^ ([a-zA-Z][a-zA-Z0-9_]+)/ {print $1}') - - subcmd="${COMP_WORDS[1]}" - # Generate possible matches and store them in the - # array variable COMPREPLY - - if [[ -n $subcmd ]] - then - - if [[ ${#COMP_WORDS[*]} == 3 ]] - then - _subcommand_opts $subcmd - return 0 - else - if [[ "$cur" == -* ]] - then - _subcommand_opts $subcmd - return 0 - else - COMPREPLY=( $(compgen -o nospace -- "$cur") ) - fi - fi - fi - - if [[ ${#COMP_WORDS[*]} == 2 ]] - then - COMPREPLY=( $(compgen -W "${cmds}" -- "$cur") ) - fi -} - -# Assign the auto-completion function for our command. - -complete -F _paver -o default paver diff --git a/scripts/user_retirement/requirements/base.txt b/scripts/user_retirement/requirements/base.txt index 622ffd2bc135..56fe29b90dc3 100644 --- a/scripts/user_retirement/requirements/base.txt +++ b/scripts/user_retirement/requirements/base.txt @@ -6,19 +6,19 @@ # asgiref==3.8.1 # via django -attrs==24.2.0 +attrs==24.3.0 # via zeep backoff==2.2.1 # via -r scripts/user_retirement/requirements/base.in -boto3==1.35.76 +boto3==1.35.93 # via -r scripts/user_retirement/requirements/base.in -botocore==1.35.76 +botocore==1.35.93 # via # boto3 # s3transfer cachetools==5.5.0 # via google-auth -certifi==2024.8.30 +certifi==2024.12.14 # via requests cffi==1.17.1 # via @@ -50,11 +50,11 @@ edx-django-utils==7.1.0 # via edx-rest-api-client edx-rest-api-client==6.0.0 # via -r scripts/user_retirement/requirements/base.in -google-api-core==2.23.0 +google-api-core==2.24.0 # via google-api-python-client -google-api-python-client==2.154.0 +google-api-python-client==2.157.0 # via -r scripts/user_retirement/requirements/base.in -google-auth==2.36.0 +google-auth==2.37.0 # via # google-api-core # google-api-python-client @@ -81,7 +81,7 @@ lxml==5.3.0 # via zeep more-itertools==10.5.0 # via simple-salesforce -newrelic==10.3.1 +newrelic==10.4.0 # via edx-django-utils pbr==6.1.0 # via stevedore @@ -89,12 +89,12 @@ platformdirs==4.3.6 # via zeep proto-plus==1.25.0 # via google-api-core -protobuf==5.29.1 +protobuf==5.29.2 # via # google-api-core # googleapis-common-protos # proto-plus -psutil==6.1.0 +psutil==6.1.1 # via edx-django-utils pyasn1==0.6.1 # via @@ -110,7 +110,7 @@ pyjwt[crypto]==2.10.1 # simple-salesforce pynacl==1.5.0 # via edx-django-utils -pyparsing==3.2.0 +pyparsing==3.2.1 # via httplib2 python-dateutil==2.9.0.post0 # via botocore @@ -146,7 +146,7 @@ six==1.17.0 # via # jenkinsapi # python-dateutil -sqlparse==0.5.2 +sqlparse==0.5.3 # via django stevedore==5.4.0 # via edx-django-utils @@ -158,6 +158,7 @@ uritemplate==4.1.1 # via google-api-python-client urllib3==1.26.20 # via + # -c scripts/user_retirement/requirements/../../../requirements/common_constraints.txt # -r scripts/user_retirement/requirements/base.in # botocore # requests diff --git a/scripts/user_retirement/requirements/testing.txt b/scripts/user_retirement/requirements/testing.txt index efaa4369170f..e63881b77cac 100644 --- a/scripts/user_retirement/requirements/testing.txt +++ b/scripts/user_retirement/requirements/testing.txt @@ -8,17 +8,17 @@ asgiref==3.8.1 # via # -r scripts/user_retirement/requirements/base.txt # django -attrs==24.2.0 +attrs==24.3.0 # via # -r scripts/user_retirement/requirements/base.txt # zeep backoff==2.2.1 # via -r scripts/user_retirement/requirements/base.txt -boto3==1.35.76 +boto3==1.35.93 # via # -r scripts/user_retirement/requirements/base.txt # moto -botocore==1.35.76 +botocore==1.35.93 # via # -r scripts/user_retirement/requirements/base.txt # boto3 @@ -28,7 +28,7 @@ cachetools==5.5.0 # via # -r scripts/user_retirement/requirements/base.txt # google-auth -certifi==2024.8.30 +certifi==2024.12.14 # via # -r scripts/user_retirement/requirements/base.txt # requests @@ -72,13 +72,13 @@ edx-django-utils==7.1.0 # edx-rest-api-client edx-rest-api-client==6.0.0 # via -r scripts/user_retirement/requirements/base.txt -google-api-core==2.23.0 +google-api-core==2.24.0 # via # -r scripts/user_retirement/requirements/base.txt # google-api-python-client -google-api-python-client==2.154.0 +google-api-python-client==2.157.0 # via -r scripts/user_retirement/requirements/base.txt -google-auth==2.36.0 +google-auth==2.37.0 # via # -r scripts/user_retirement/requirements/base.txt # google-api-core @@ -109,7 +109,7 @@ isodate==0.7.2 # zeep jenkinsapi==0.3.13 # via -r scripts/user_retirement/requirements/base.txt -jinja2==3.1.4 +jinja2==3.1.5 # via moto jmespath==1.0.1 # via @@ -132,7 +132,7 @@ more-itertools==10.5.0 # simple-salesforce moto==4.2.14 # via -r scripts/user_retirement/requirements/testing.in -newrelic==10.3.1 +newrelic==10.4.0 # via # -r scripts/user_retirement/requirements/base.txt # edx-django-utils @@ -152,13 +152,13 @@ proto-plus==1.25.0 # via # -r scripts/user_retirement/requirements/base.txt # google-api-core -protobuf==5.29.1 +protobuf==5.29.2 # via # -r scripts/user_retirement/requirements/base.txt # google-api-core # googleapis-common-protos # proto-plus -psutil==6.1.0 +psutil==6.1.1 # via # -r scripts/user_retirement/requirements/base.txt # edx-django-utils @@ -184,7 +184,7 @@ pynacl==1.5.0 # via # -r scripts/user_retirement/requirements/base.txt # edx-django-utils -pyparsing==3.2.0 +pyparsing==3.2.1 # via # -r scripts/user_retirement/requirements/base.txt # httplib2 @@ -248,7 +248,7 @@ six==1.17.0 # -r scripts/user_retirement/requirements/base.txt # jenkinsapi # python-dateutil -sqlparse==0.5.2 +sqlparse==0.5.3 # via # -r scripts/user_retirement/requirements/base.txt # django diff --git a/scripts/xblock/requirements.txt b/scripts/xblock/requirements.txt index 920cf0cf6ac1..9af137853dc5 100644 --- a/scripts/xblock/requirements.txt +++ b/scripts/xblock/requirements.txt @@ -4,7 +4,7 @@ # # make upgrade # -certifi==2024.8.30 +certifi==2024.12.14 # via requests charset-normalizer==2.0.12 # via @@ -15,4 +15,6 @@ idna==3.10 requests==2.32.3 # via -r scripts/xblock/requirements.in urllib3==2.2.3 - # via requests + # via + # -c scripts/xblock/../../requirements/common_constraints.txt + # requests diff --git a/scripts/xsslint/xss_linter.py b/scripts/xsslint/xss_linter.py index a35038c3de6d..b32c54aa5cd7 100755 --- a/scripts/xsslint/xss_linter.py +++ b/scripts/xsslint/xss_linter.py @@ -4,6 +4,316 @@ """ +import argparse +import importlib +import json +import os +import re +import sys + +from functools import reduce +from io import StringIO +from xsslint.reporting import SummaryResults +from xsslint.rules import RuleSet +from xsslint.utils import is_skip_dir + + +class BuildFailure(Exception): + pass + + +def fail_quality(message): + """ + Fail the specified quality check. + """ + + raise BuildFailure(message) + + +def _load_config_module(module_path): + cwd = os.getcwd() + if cwd not in sys.path: + # Enable config module to be imported relative to wherever the script was run from. + sys.path.append(cwd) + return importlib.import_module(module_path) + + +def _build_ruleset(template_linters): + """ + Combines the RuleSets from the provided template_linters into a single, aggregate RuleSet. + + Arguments: + template_linters: A list of linting objects. + + Returns: + The combined RuleSet. + """ + return reduce( + lambda combined, current: combined + current.ruleset, + template_linters, + RuleSet() + ) + + +def _process_file(full_path, template_linters, options, summary_results, out): + """ + For each linter, lints the provided file. This means finding and printing + violations. + + Arguments: + full_path: The full path of the file to lint. + template_linters: A list of linting objects. + options: A list of the options. + summary_results: A SummaryResults with a summary of the violations. + out: output file + + """ + num_violations = 0 + directory = os.path.dirname(full_path) + file_name = os.path.basename(full_path) + try: + for template_linter in template_linters: + results = template_linter.process_file(directory, file_name) + results.print_results(options, summary_results, out) + except BaseException as e: + raise Exception(f"Failed to process path: {full_path}") from e + + +def _process_os_dir(directory, files, template_linters, options, summary_results, out): + """ + Calls out to lint each file in the passed list of files. + + Arguments: + directory: Directory being linted. + files: All files in the directory to be linted. + template_linters: A list of linting objects. + options: A list of the options. + summary_results: A SummaryResults with a summary of the violations. + out: output file + + """ + for current_file in sorted(files, key=lambda s: s.lower()): + full_path = os.path.join(directory, current_file) + _process_file(full_path, template_linters, options, summary_results, out) + + +def _process_os_dirs(starting_dir, template_linters, options, summary_results, out): + """ + For each linter, lints all the directories in the starting directory. + + Arguments: + starting_dir: The initial directory to begin the walk. + template_linters: A list of linting objects. + options: A list of the options. + summary_results: A SummaryResults with a summary of the violations. + out: output file + + """ + skip_dirs = options.get('skip_dirs', ()) + for root, dirs, files in os.walk(starting_dir): + if is_skip_dir(skip_dirs, root): + del dirs + continue + dirs.sort(key=lambda s: s.lower()) + _process_os_dir(root, files, template_linters, options, summary_results, out) + + +def _get_xsslint_counts(result_contents): + """ + This returns a dict of violations from the xsslint report. + + Arguments: + filename: The name of the xsslint report. + + Returns: + A dict containing the following: + rules: A dict containing the count for each rule as follows: + violation-rule-id: N, where N is the number of violations + total: M, where M is the number of total violations + + """ + + rule_count_regex = re.compile(r"^(?P[a-z-]+):\s+(?P\d+) violations", re.MULTILINE) + total_count_regex = re.compile(r"^(?P\d+) violations total", re.MULTILINE) + violations = {'rules': {}} + for violation_match in rule_count_regex.finditer(result_contents): + try: + violations['rules'][violation_match.group('rule_id')] = int(violation_match.group('count')) + except ValueError: + violations['rules'][violation_match.group('rule_id')] = None + try: + violations['total'] = int(total_count_regex.search(result_contents).group('count')) + # An AttributeError will occur if the regex finds no matches. + # A ValueError will occur if the returned regex cannot be cast as a float. + except (AttributeError, ValueError): + violations['total'] = None + return violations + + +def _check_violations(options, results): + xsslint_script = "xss_linter.py" + try: + thresholds_option = options['thresholds'] + # Read the JSON file + with open(thresholds_option, 'r') as file: + violation_thresholds = json.load(file) + + except ValueError: + violation_thresholds = None + if isinstance(violation_thresholds, dict) is False or \ + any(key not in ("total", "rules") for key in violation_thresholds.keys()): + print('xsslint') + fail_quality("""FAILURE: Thresholds option "{thresholds_option}" was not supplied using proper format.\n""" + """Here is a properly formatted example, '{{"total":100,"rules":{{"javascript-escape":0}}}}' """ + """with property names in double-quotes.""".format(thresholds_option=thresholds_option)) + + try: + metrics_str = "Number of {xsslint_script} violations: {num_violations}\n".format( + xsslint_script=xsslint_script, num_violations=int(results['total']) + ) + if 'rules' in results and any(results['rules']): + metrics_str += "\n" + rule_keys = sorted(results['rules'].keys()) + for rule in rule_keys: + metrics_str += "{rule} violations: {count}\n".format( + rule=rule, + count=int(results['rules'][rule]) + ) + except TypeError: + print('xsslint') + fail_quality("FAILURE: Number of {xsslint_script} violations could not be found".format( + xsslint_script=xsslint_script + )) + + error_message = "" + # Test total violations against threshold. + if 'total' in list(violation_thresholds.keys()): + if violation_thresholds['total'] < results['total']: + error_message = "Too many violations total ({count}).\nThe limit is {violations_limit}.".format( + count=results['total'], violations_limit=violation_thresholds['total'] + ) + + # Test rule violations against thresholds. + if 'rules' in violation_thresholds: + threshold_keys = sorted(violation_thresholds['rules'].keys()) + for threshold_key in threshold_keys: + if threshold_key not in results['rules']: + error_message += ( + "\nNumber of {xsslint_script} violations for {rule} could not be found" + ).format( + xsslint_script=xsslint_script, rule=threshold_key + ) + elif violation_thresholds['rules'][threshold_key] < results['rules'][threshold_key]: + error_message += \ + "\nToo many {rule} violations ({count}).\nThe {rule} limit is {violations_limit}.".format( + rule=threshold_key, count=results['rules'][threshold_key], + violations_limit=violation_thresholds['rules'][threshold_key], + ) + + if error_message: + print('xsslint') + fail_quality("FAILURE: XSSLinter Failed.\n{error_message}\n" + "run the following command to hone in on the problem:\n" + "./scripts/xss-commit-linter.sh -h".format(error_message=error_message)) + else: + print("successfully run xsslint") + + +def _lint(file_or_dir, template_linters, options, summary_results, out): + """ + For each linter, lints the provided file or directory. + + Arguments: + file_or_dir: The file or initial directory to lint. + template_linters: A list of linting objects. + options: A list of the options. + summary_results: A SummaryResults with a summary of the violations. + out: output file + + """ + + if file_or_dir is not None and os.path.isfile(file_or_dir): + _process_file(file_or_dir, template_linters, options, summary_results, out) + else: + directory = "." + if file_or_dir is not None: + if os.path.exists(file_or_dir): + directory = file_or_dir + else: + raise ValueError(f"Path [{file_or_dir}] is not a valid file or directory.") + _process_os_dirs(directory, template_linters, options, summary_results, out) + + summary_results.print_results(options, out) + result_output = _get_xsslint_counts(out.getvalue()) + _check_violations(options, result_output) + + +def main(): + """ + Used to execute the linter. Use --help option for help. + + Prints all violations. + """ + epilog = "For more help using the xss linter, including details on how to\n" + epilog += "understand and fix any violations, read the docs here:\n" + epilog += "\n" + # pylint: disable=line-too-long + epilog += " https://edx.readthedocs.org/projects/edx-developer-guide/en/latest/conventions/preventing_xss.html#xss-linter\n" + + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description='Checks that templates are safe.', + epilog=epilog, + ) + parser.add_argument( + '--list-files', dest='list_files', action='store_true', + help='Only display the filenames that contain violations.' + ) + parser.add_argument( + '--rule-totals', dest='rule_totals', action='store_true', + help='Display the totals for each rule.' + ) + parser.add_argument( + '--summary-format', dest='summary_format', + choices=['eslint', 'json'], default='eslint', + help='Choose the display format for the summary.' + ) + parser.add_argument( + '--verbose', dest='verbose', action='store_true', + help='Print multiple lines where possible for additional context of violations.' + ) + parser.add_argument( + '--config', dest='config', action='store', default='xsslint.default_config', + help='Specifies the config module to use. The config module should be in Python package syntax.' + ) + parser.add_argument( + '--thresholds', dest='thresholds', action='store', + help='Specifies the config module to use. The config module should be in Python package syntax.' + ) + parser.add_argument('path', nargs="?", default=None, help='A file to lint or directory to recursively lint.') + + args = parser.parse_args() + config = _load_config_module(args.config) + options = { + 'list_files': args.list_files, + 'rule_totals': args.rule_totals, + 'summary_format': args.summary_format, + 'verbose': args.verbose, + 'skip_dirs': getattr(config, 'SKIP_DIRS', ()), + 'thresholds': args.thresholds + } + template_linters = getattr(config, 'LINTERS', ()) + if not template_linters: + raise ValueError(f"LINTERS is empty or undefined in the config module ({args.config}).") + + ruleset = _build_ruleset(template_linters) + summary_results = SummaryResults(ruleset) + _lint(args.path, template_linters, options, summary_results, out=StringIO()) + + if __name__ == "__main__": - from xsslint.main import main - main() + try: + main() + except BuildFailure as e: + print(e) + sys.exit(1) diff --git a/scripts/xsslint/xsslint/main.py b/scripts/xsslint/xsslint/main.py deleted file mode 100644 index f8f8672b74b3..000000000000 --- a/scripts/xsslint/xsslint/main.py +++ /dev/null @@ -1,187 +0,0 @@ -""" -The main function for the XSS linter. -""" - - -import argparse -import importlib -import os -import sys -from functools import reduce - -from xsslint.reporting import SummaryResults -from xsslint.rules import RuleSet -from xsslint.utils import is_skip_dir - - -def _load_config_module(module_path): - cwd = os.getcwd() - if cwd not in sys.path: - # Enable config module to be imported relative to wherever the script was run from. - sys.path.append(cwd) - return importlib.import_module(module_path) - - -def _build_ruleset(template_linters): - """ - Combines the RuleSets from the provided template_linters into a single, aggregate RuleSet. - - Arguments: - template_linters: A list of linting objects. - - Returns: - The combined RuleSet. - """ - return reduce( - lambda combined, current: combined + current.ruleset, - template_linters, - RuleSet() - ) - - -def _process_file(full_path, template_linters, options, summary_results, out): - """ - For each linter, lints the provided file. This means finding and printing - violations. - - Arguments: - full_path: The full path of the file to lint. - template_linters: A list of linting objects. - options: A list of the options. - summary_results: A SummaryResults with a summary of the violations. - out: output file - - """ - num_violations = 0 - directory = os.path.dirname(full_path) - file_name = os.path.basename(full_path) - try: - for template_linter in template_linters: - results = template_linter.process_file(directory, file_name) - results.print_results(options, summary_results, out) - except BaseException as e: - raise Exception(f"Failed to process path: {full_path}") from e - - -def _process_os_dir(directory, files, template_linters, options, summary_results, out): - """ - Calls out to lint each file in the passed list of files. - - Arguments: - directory: Directory being linted. - files: All files in the directory to be linted. - template_linters: A list of linting objects. - options: A list of the options. - summary_results: A SummaryResults with a summary of the violations. - out: output file - - """ - for current_file in sorted(files, key=lambda s: s.lower()): - full_path = os.path.join(directory, current_file) - _process_file(full_path, template_linters, options, summary_results, out) - - -def _process_os_dirs(starting_dir, template_linters, options, summary_results, out): - """ - For each linter, lints all the directories in the starting directory. - - Arguments: - starting_dir: The initial directory to begin the walk. - template_linters: A list of linting objects. - options: A list of the options. - summary_results: A SummaryResults with a summary of the violations. - out: output file - - """ - skip_dirs = options.get('skip_dirs', ()) - for root, dirs, files in os.walk(starting_dir): - if is_skip_dir(skip_dirs, root): - del dirs - continue - dirs.sort(key=lambda s: s.lower()) - _process_os_dir(root, files, template_linters, options, summary_results, out) - - -def _lint(file_or_dir, template_linters, options, summary_results, out): - """ - For each linter, lints the provided file or directory. - - Arguments: - file_or_dir: The file or initial directory to lint. - template_linters: A list of linting objects. - options: A list of the options. - summary_results: A SummaryResults with a summary of the violations. - out: output file - - """ - - if file_or_dir is not None and os.path.isfile(file_or_dir): - _process_file(file_or_dir, template_linters, options, summary_results, out) - else: - directory = "." - if file_or_dir is not None: - if os.path.exists(file_or_dir): - directory = file_or_dir - else: - raise ValueError(f"Path [{file_or_dir}] is not a valid file or directory.") - _process_os_dirs(directory, template_linters, options, summary_results, out) - - summary_results.print_results(options, out) - - -def main(): - """ - Used to execute the linter. Use --help option for help. - - Prints all violations. - """ - epilog = "For more help using the xss linter, including details on how to\n" - epilog += "understand and fix any violations, read the docs here:\n" - epilog += "\n" - # pylint: disable=line-too-long - epilog += " https://edx.readthedocs.org/projects/edx-developer-guide/en/latest/conventions/preventing_xss.html#xss-linter\n" - - parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, - description='Checks that templates are safe.', - epilog=epilog, - ) - parser.add_argument( - '--list-files', dest='list_files', action='store_true', - help='Only display the filenames that contain violations.' - ) - parser.add_argument( - '--rule-totals', dest='rule_totals', action='store_true', - help='Display the totals for each rule.' - ) - parser.add_argument( - '--summary-format', dest='summary_format', - choices=['eslint', 'json'], default='eslint', - help='Choose the display format for the summary.' - ) - parser.add_argument( - '--verbose', dest='verbose', action='store_true', - help='Print multiple lines where possible for additional context of violations.' - ) - parser.add_argument( - '--config', dest='config', action='store', default='xsslint.default_config', - help='Specifies the config module to use. The config module should be in Python package syntax.' - ) - parser.add_argument('path', nargs="?", default=None, help='A file to lint or directory to recursively lint.') - - args = parser.parse_args() - config = _load_config_module(args.config) - options = { - 'list_files': args.list_files, - 'rule_totals': args.rule_totals, - 'summary_format': args.summary_format, - 'verbose': args.verbose, - 'skip_dirs': getattr(config, 'SKIP_DIRS', ()) - } - template_linters = getattr(config, 'LINTERS', ()) - if not template_linters: - raise ValueError(f"LINTERS is empty or undefined in the config module ({args.config}).") - - ruleset = _build_ruleset(template_linters) - summary_results = SummaryResults(ruleset) - _lint(args.path, template_linters, options, summary_results, out=sys.stdout) diff --git a/stylelint.config.js b/stylelint.config.js deleted file mode 100644 index bd7769911708..000000000000 --- a/stylelint.config.js +++ /dev/null @@ -1,3 +0,0 @@ -module.exports = { - extends: '@edx/stylelint-config-edx' -}; diff --git a/xmodule/js/spec/video/video_control_spec.js b/xmodule/js/spec/video/video_control_spec.js index 248c3c31d2be..49326290716b 100644 --- a/xmodule/js/spec/video/video_control_spec.js +++ b/xmodule/js/spec/video/video_control_spec.js @@ -186,6 +186,26 @@ }); describe('constructor with end-time', function() { + it('displays the correct time when startTime and endTime are specified', function(done) { + state = jasmine.initializePlayer({ + start: 10, + end: 20 + }); + spyOn(state.videoPlayer, 'duration').and.returnValue(60); + + state.videoControl.updateVcrVidTime({ + time: 15, + duration: 60 + }); + + jasmine.waitUntil(function() { + var expectedValue = $('.video-controls').find('.vidtime'); + return expectedValue.text().indexOf('0:05 / 0:20') !== -1; // Expecting 15 seconds - 10 seconds = 5 seconds + }).then(function() { + expect($('.video-controls').find('.vidtime')).toHaveText('0:05 / 0:20'); + }).always(done); + }); + it( 'saved position is 0, timer slider and VCR set to 0:00 ' + 'and ending at specified end-time', diff --git a/xmodule/js/spec/video/video_events_plugin_spec.js b/xmodule/js/spec/video/video_events_plugin_spec.js index 860cfb5e07ba..ac809d054edc 100644 --- a/xmodule/js/spec/video/video_events_plugin_spec.js +++ b/xmodule/js/spec/video/video_events_plugin_spec.js @@ -226,6 +226,55 @@ import '../helper.js'; destroy: plugin.destroy }); }); + + describe('getCurrentTime method', function() { + it('returns current time adjusted by startTime if video starts from a subsection', function() { + spyOn(state.videoPlayer, 'currentTime', 'get').and.returnValue(120); + state.config.startTime = 30; + expect(state.videoEventsPlugin.getCurrentTime()).toBe(90); // 120 - 30 = 90 + }); + + it('returns 0 if currentTime is undefined', function() { + spyOn(state.videoPlayer, 'currentTime', 'get').and.returnValue(undefined); + state.config.startTime = 30; // Start time is irrelevant since current time is undefined + expect(state.videoEventsPlugin.getCurrentTime()).toBe(0); + }); + + it('returns unadjusted current time if startTime is not defined', function() { + spyOn(state.videoPlayer, 'currentTime', 'get').and.returnValue(60); + expect(state.videoEventsPlugin.getCurrentTime()).toBe(60); // Returns current time as is + }); + }); + + describe('log method', function() { + it('logs event with adjusted duration when startTime and endTime are defined', function() { + state.config.startTime = 30; + state.config.endTime = 150; + state.duration = 200; + + state.videoEventsPlugin.log('test_event', {}); + + expect(Logger.log).toHaveBeenCalledWith('test_event', { + id: 'id', + code: this.code, + duration: 120, // 150 - 30 = 120 + }); + }); + + it('logs event with full duration when startTime and endTime are not defined', function() { + state.config.startTime = undefined; + state.config.endTime = undefined; + state.duration = 200; + + state.videoEventsPlugin.log('test_event', {}); + + expect(Logger.log).toHaveBeenCalledWith('test_event', { + id: 'id', + code: this.code, + duration: 200 // Full duration as no start/end time adjustment is needed + }); + }); + }); }); describe('VideoPlayer Events plugin', function() { diff --git a/xmodule/js/src/video/04_video_control.js b/xmodule/js/src/video/04_video_control.js index b8cec53cae6f..c6f2ba240745 100644 --- a/xmodule/js/src/video/04_video_control.js +++ b/xmodule/js/src/video/04_video_control.js @@ -152,10 +152,17 @@ } function updateVcrVidTime(params) { - var endTime = (this.config.endTime !== null) ? this.config.endTime : params.duration; + var endTime = (this.config.endTime !== null) ? this.config.endTime : params.duration, + startTime, currentTime; // in case endTime is accidentally specified as being greater than the video endTime = Math.min(endTime, params.duration); - this.videoControl.vidTimeEl.text(Time.format(params.time) + ' / ' + Time.format(endTime)); + startTime = this.config.startTime > 0 ? this.config.startTime : 0; + // if it's a subsection of video, use the clip duration as endTime + if (startTime && this.config.endTime) { + endTime = this.config.endTime - startTime; + } + currentTime = startTime ? params.time - startTime : params.time; + this.videoControl.vidTimeEl.text(Time.format(currentTime) + ' / ' + Time.format(endTime)); } } ); diff --git a/xmodule/js/src/video/09_events_plugin.js b/xmodule/js/src/video/09_events_plugin.js index d407534c983e..6116f26e22c5 100644 --- a/xmodule/js/src/video/09_events_plugin.js +++ b/xmodule/js/src/video/09_events_plugin.js @@ -143,8 +143,15 @@ }, getCurrentTime: function() { - var player = this.state.videoPlayer; - return player ? player.currentTime : 0; + var player = this.state.videoPlayer, + startTime = this.state.config.startTime, + currentTime; + currentTime = player ? player.currentTime : 0; + // if video didn't start from 0(it's a subsection of video), subtract the additional time at start + if (startTime) { + currentTime = currentTime ? currentTime - startTime : 0; + } + return currentTime; }, getCurrentLanguage: function() { @@ -153,11 +160,15 @@ }, log: function(eventName, data) { + // use startTime and endTime to calculate the duration to handle the case where only a subsection of video is used + var endTime = this.state.config.endTime || this.state.duration, + startTime = this.state.config.startTime; + var logInfo = _.extend({ id: this.state.id, // eslint-disable-next-line no-nested-ternary code: this.state.isYoutubeType() ? this.state.youtubeId() : this.state.canPlayHLS ? 'hls' : 'html5', - duration: this.state.duration + duration: endTime - startTime }, data, this.options.data); Logger.log(eventName, logInfo); } diff --git a/xmodule/templates/html/zooming_image.yaml b/xmodule/templates/html/zooming_image.yaml new file mode 100644 index 000000000000..14e9ef2c2392 --- /dev/null +++ b/xmodule/templates/html/zooming_image.yaml @@ -0,0 +1,239 @@ +--- +metadata: + display_name: Zooming Image Tool +data: | +

Use the Zooming Image Tool to enable learners to see details of large, complex images. With the tool, the learner can move the mouse pointer over a part of the image to enlarge it and see more detail.

+

To set it up, first upload the regular image file and, optionally, a magnified image file to your course. Then refer to them with the following HTML code, replacing the values in italics accordingly:

+ 
+      <div class="zooming-image">
+        <a data-src="(Optional) URL to the magnified image">
+          <img src="URL to the regular image" />
+        </a>
+      </div>
+
+

If a magnified image is not provided, the regular one will be used at its native size.

+

Feel free to modify the example below for your own use, but take care not to remove the included Javascript.

+
+ + + A color wheel. + +
+ diff --git a/xmodule/tests/__init__.py b/xmodule/tests/__init__.py index b2cdd67b71ba..786836c050b5 100644 --- a/xmodule/tests/__init__.py +++ b/xmodule/tests/__init__.py @@ -1,10 +1,5 @@ """ unittests for xmodule - -Run like this: - - paver test_lib -l ./xmodule - """