From 67dc357355facaa05dc73c749817b24812dda1d9 Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Thu, 15 Sep 2022 18:12:42 +0300 Subject: [PATCH 001/125] bump tahoe-idp==2.1.0 --- requirements/edx/appsembler.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/edx/appsembler.txt b/requirements/edx/appsembler.txt index 9813c6804459..fb88bc67a094 100644 --- a/requirements/edx/appsembler.txt +++ b/requirements/edx/appsembler.txt @@ -23,7 +23,7 @@ https://github.com/appsembler/edx-proctoring/archive/v2.4.0-appsembler1.tar.gz django-tiers==0.2.7 fusionauth-client==1.36.0 google-cloud-storage==1.32.0 -tahoe-idp==2.0.0 +tahoe-idp==2.1.0 tahoe-sites==1.3.2 tahoe-lti==0.3.0 site-configuration-client==0.2.3 From 997c2f160f9ea6ab19da647aa92245d17a4bf9b5 Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Thu, 15 Sep 2022 18:11:53 +0300 Subject: [PATCH 002/125] Add TahoeCourseAuthor role with Studio access --- .../appsembler/auth/course_roles.py | 23 ------ .../appsembler/tahoe_idp/course_roles.py | 56 +++++++++++++++ .../appsembler/tahoe_idp/helpers.py | 23 +++--- .../tests/test_course_roles.py | 2 +- .../tests/test_tahoe_idp_pipeline_steps.py | 72 ++++++++++++------- .../appsembler/tahoe_idp/tpa_pipeline.py | 5 +- 6 files changed, 121 insertions(+), 60 deletions(-) delete mode 100644 openedx/core/djangoapps/appsembler/auth/course_roles.py create mode 100644 openedx/core/djangoapps/appsembler/tahoe_idp/course_roles.py rename openedx/core/djangoapps/appsembler/{auth => tahoe_idp}/tests/test_course_roles.py (97%) diff --git a/openedx/core/djangoapps/appsembler/auth/course_roles.py b/openedx/core/djangoapps/appsembler/auth/course_roles.py deleted file mode 100644 index 01202df93356..000000000000 --- a/openedx/core/djangoapps/appsembler/auth/course_roles.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -Tahoe Authentication helpers for managing course related roles. -""" - -from common.djangoapps.student.roles import CourseCreatorRole, OrgStaffRole - - -def update_organization_staff_roles(user, organization_short_name, set_as_organization_staff=False): - """ - Update the organization-wide OrgStaffRole/CourseCreatorRole for using Studio and instructor dashboards. - """ - assert user, 'Parameter `user` is required.' - assert organization_short_name, 'Parameter `organization_short_name` is required.' - - organization_role = OrgStaffRole(organization_short_name) - creator_role = CourseCreatorRole() - - if set_as_organization_staff: - organization_role.add_users(user) - creator_role.add_users(user) - else: - organization_role.remove_users(user) - creator_role.remove_users(user) diff --git a/openedx/core/djangoapps/appsembler/tahoe_idp/course_roles.py b/openedx/core/djangoapps/appsembler/tahoe_idp/course_roles.py new file mode 100644 index 000000000000..e44aa9f058de --- /dev/null +++ b/openedx/core/djangoapps/appsembler/tahoe_idp/course_roles.py @@ -0,0 +1,56 @@ +""" +Tahoe Authentication helpers for managing course related roles. +""" + +from common.djangoapps.student.roles import ( + CourseCreatorRole, + OrgRole, + OrgStaffRole, + register_access_role, +) + + +def update_organization_staff_roles( + user, + organization_short_name, + set_as_course_author=False, + set_as_organization_staff=False, +): + """ + Update the organization-wide OrgStaffRole/CourseCreatorRole for using Studio and instructor dashboards. + """ + assert user, 'Parameter `user` is required.' + assert organization_short_name, 'Parameter `organization_short_name` is required.' + + organization_role = OrgStaffRole(organization_short_name) + course_author_role = TahoeCourseAuthorRole(organization_short_name) + creator_role = CourseCreatorRole() + + if set_as_organization_staff or set_as_course_author: + # Both org-wide staff and limited course author can create courses. + creator_role.add_users(user) + else: + creator_role.remove_users(user) + + if set_as_organization_staff: + organization_role.add_users(user) + else: + organization_role.remove_users(user) + + if set_as_course_author: + course_author_role.add_users(user) + else: + course_author_role.remove_users(user) + + +@register_access_role +class TahoeCourseAuthorRole(OrgRole): + """ + A limited course access role to allow Studio access without having a course. + + A user with this role needs to be explicitly invited to a course. + """ + ROLE = 'tahoe_course_author' + + def __init__(self, *args, **kwargs): + super().__init__(self.ROLE, *args, **kwargs) diff --git a/openedx/core/djangoapps/appsembler/tahoe_idp/helpers.py b/openedx/core/djangoapps/appsembler/tahoe_idp/helpers.py index c34884a472d2..9f316653b051 100644 --- a/openedx/core/djangoapps/appsembler/tahoe_idp/helpers.py +++ b/openedx/core/djangoapps/appsembler/tahoe_idp/helpers.py @@ -15,12 +15,6 @@ import third_party_auth from third_party_auth.pipeline import running as pipeline_running - -from .constants import ( - TAHOE_IDP_BACKEND_NAME, - TAHOE_IDP_PROVIDER_NAME, -) - from student.roles import ( CourseAccessRole, CourseCreatorRole, @@ -33,6 +27,14 @@ from tahoe_idp import api as tahoe_idp_api +from .constants import ( + TAHOE_IDP_BACKEND_NAME, + TAHOE_IDP_PROVIDER_NAME, +) + +from .course_roles import TahoeCourseAuthorRole + + def is_tahoe_idp_enabled(): """ Tahoe: Check if tahoe-idp package is enabled for the current site (or cluster-wide). @@ -148,7 +150,7 @@ def is_studio_allowed_for_user(user, organization=None): OR the user is staff user OR the user is an admin on the organization OR the user has deprecated_has_course_specific_role() - OR the user has (OrgStaffRole) or (OrgInstructorRole) role + OR the user has (OrgStaffRole) or (OrgInstructorRole) or (TahoeCourseAuthorRole) role :param user: the user in question :param organization: the user's organization. If the user is not super admin or staff, this value will be used @@ -168,9 +170,12 @@ def is_studio_allowed_for_user(user, organization=None): return True short_name = organization.short_name - has_org_wide_role = OrgStaffRole(short_name).has_user(user) or OrgInstructorRole(short_name).has_user(user) - return has_org_wide_role + for org_wide_role in [OrgStaffRole, OrgInstructorRole, TahoeCourseAuthorRole]: + if org_wide_role(short_name).has_user(user): + return True + + return False def is_studio_login_form_overridden(): diff --git a/openedx/core/djangoapps/appsembler/auth/tests/test_course_roles.py b/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_course_roles.py similarity index 97% rename from openedx/core/djangoapps/appsembler/auth/tests/test_course_roles.py rename to openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_course_roles.py index 5be3ed86094b..8e457fcf9a8b 100644 --- a/openedx/core/djangoapps/appsembler/auth/tests/test_course_roles.py +++ b/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_course_roles.py @@ -7,7 +7,7 @@ from student.tests.factories import UserFactory -from .. import course_roles +from openedx.core.djangoapps.appsembler.tahoe_idp import course_roles @pytest.mark.django_db diff --git a/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_pipeline_steps.py b/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_pipeline_steps.py index c7be2563ec61..05874cbaf5f3 100644 --- a/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_pipeline_steps.py +++ b/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_pipeline_steps.py @@ -8,6 +8,7 @@ import tahoe_sites.api from common.djangoapps.student.roles import CourseCreatorRole, OrgStaffRole +from ..course_roles import TahoeCourseAuthorRole from ..tpa_pipeline import tahoe_idp_user_updates from openedx.core.djangoapps.site_configuration.tests.factories import SiteFactory @@ -37,40 +38,58 @@ def test_tahoe_idp_step_in_settings(): assert idp_step_index == force_sync_step_index + 1, 'Tahoe IdP step should be right after `user_details_force_sync`' -@pytest.mark.parametrize('user_details,should_be_admin,should_be_staff,message', [ - ( - { +@pytest.mark.parametrize('test_case', [ + { + 'user_details': { 'tahoe_idp_is_organization_admin': False, 'tahoe_idp_is_organization_staff': False, + 'tahoe_idp_is_course_author': False, 'tahoe_idp_metadata': {'field': 'some value'}, }, - False, - False, - 'Check for learner', - ), - ( - { + 'should_be_admin': False, + 'should_be_staff': False, + 'should_be_author': False, + 'message': 'Check for learner', + }, + { + 'user_details': { 'tahoe_idp_is_organization_admin': False, 'tahoe_idp_is_organization_staff': True, + 'tahoe_idp_is_course_author': False, 'tahoe_idp_metadata': {'field': 'some value'}, }, - False, - True, - 'Check for Studio', - ), - ( - { + 'should_be_admin': False, + 'should_be_staff': True, + 'should_be_author': False, + 'message': 'Check for Studio', + }, + { + 'user_details': { 'tahoe_idp_is_organization_admin': True, 'tahoe_idp_is_organization_staff': True, + 'tahoe_idp_is_course_author': False, 'tahoe_idp_metadata': {'field': 'some value'}, }, - True, - True, - 'Check for Admins', - ), + 'should_be_admin': True, + 'should_be_staff': True, + 'should_be_author': False, + 'message': 'Check for Admins', + }, + { + 'user_details': { + 'tahoe_idp_is_organization_admin': False, + 'tahoe_idp_is_organization_staff': False, + 'tahoe_idp_is_course_author': True, + 'tahoe_idp_metadata': {'field': 'some value'}, + }, + 'should_be_admin': False, + 'should_be_staff': False, + 'should_be_author': True, + 'message': 'Check for Course Authors', + }, ]) @pytest.mark.django_db -def test_tahoe_idp_roles_step_roles(user_details, should_be_admin, should_be_staff, message): +def test_tahoe_idp_roles_step_roles(test_case): """ Tests for happy scenarios of the `tahoe_idp_user_updates` step. """ @@ -88,15 +107,18 @@ def test_tahoe_idp_roles_step_roles(user_details, should_be_admin, should_be_sta tahoe_idp_user_updates( auth_entry=None, strategy=strategy, - details=user_details, + details=test_case['user_details'], user=user, ) - org_role = OrgStaffRole(organization.short_name) + tahoe_author_role = TahoeCourseAuthorRole(organization.short_name) creator_role = CourseCreatorRole() - assert org_role.has_user(user) == should_be_staff, message - assert creator_role.has_user(user) == should_be_staff, message - assert tahoe_sites.api.is_active_admin_on_organization(user, organization) == should_be_admin, message + message = test_case['message'] + should_be_course_creator = test_case['should_be_staff'] or test_case['should_be_author'] + assert org_role.has_user(user) == test_case['should_be_staff'], message + assert creator_role.has_user(user) == should_be_course_creator, message + assert tahoe_author_role.has_user(user) == test_case['should_be_author'], message + assert tahoe_sites.api.is_active_admin_on_organization(user, organization) == test_case['should_be_admin'], message assert user.profile.get_meta() == {'tahoe_idp_metadata': {'field': 'some value'}} mock_update_tahoe_user_id.assert_called_once_with(user) diff --git a/openedx/core/djangoapps/appsembler/tahoe_idp/tpa_pipeline.py b/openedx/core/djangoapps/appsembler/tahoe_idp/tpa_pipeline.py index ee663cd32844..5393f7e2ff12 100644 --- a/openedx/core/djangoapps/appsembler/tahoe_idp/tpa_pipeline.py +++ b/openedx/core/djangoapps/appsembler/tahoe_idp/tpa_pipeline.py @@ -6,12 +6,11 @@ import beeline import tahoe_sites.api -from openedx.core.djangoapps.appsembler.auth import course_roles from tahoe_idp import api as tahoe_idp_api +from . import course_roles from .helpers import store_idp_metadata_in_user_profile - from .constants import TAHOE_IDP_BACKEND_NAME log = logging.getLogger(__name__) @@ -38,6 +37,7 @@ def tahoe_idp_user_updates(auth_entry, strategy, details, user=None, *args, **kw if user and backend_name == TAHOE_IDP_BACKEND_NAME: set_as_admin = details['tahoe_idp_is_organization_admin'] set_as_organization_staff = details['tahoe_idp_is_organization_staff'] + set_as_course_author = details['tahoe_idp_is_course_author'] organization = tahoe_sites.api.get_current_organization(strategy.request) @@ -53,6 +53,7 @@ def tahoe_idp_user_updates(auth_entry, strategy, details, user=None, *args, **kw course_roles.update_organization_staff_roles( user=user, organization_short_name=organization_short_name, + set_as_course_author=set_as_course_author, set_as_organization_staff=set_as_organization_staff, ) From b22e4e2e907aea21e166266598ac9e096dc12941 Mon Sep 17 00:00:00 2001 From: Eugene Dyudyunov Date: Mon, 12 Sep 2022 16:18:05 +0300 Subject: [PATCH 003/125] fix: empty signature added after every certificate saving (#30912) A new behaviour: - Empty signature is still added when initially create a certificate; - Empty signature isn't added when certificate has at least one signature. --- cms/static/js/certificates/models/certificate.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cms/static/js/certificates/models/certificate.js b/cms/static/js/certificates/models/certificate.js index a440d569d606..d0e6ae6e9f35 100644 --- a/cms/static/js/certificates/models/certificate.js +++ b/cms/static/js/certificates/models/certificate.js @@ -43,7 +43,7 @@ define([ initialize: function(attributes, options) { // Set up the initial state of the attributes set for this model instance this.canBeEmpty = options && options.canBeEmpty; - if (options.add) { + if (options.add && !attributes.signatories) { // Ensure at least one child Signatory model is defined for any new Certificate model attributes.signatories = new SignatoryModel({certificate: this}); } From 6d4e53d392d587e8c030eac199acf2cc7a34d435 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Mon, 3 Oct 2022 17:03:40 -0700 Subject: [PATCH 004/125] Add legacy /user_api/ prefix to MAIN_SITE_REDIRECT_ALLOWLIST. It's still used by EdxRestAPIClient which is used for PS integrations --- .../djangoapps/appsembler/settings/settings/production_lms.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openedx/core/djangoapps/appsembler/settings/settings/production_lms.py b/openedx/core/djangoapps/appsembler/settings/settings/production_lms.py index 26b2ea10f272..ad2becf4b78b 100644 --- a/openedx/core/djangoapps/appsembler/settings/settings/production_lms.py +++ b/openedx/core/djangoapps/appsembler/settings/settings/production_lms.py @@ -45,6 +45,7 @@ def plugin_settings(settings): # from the redirect mechanics. settings.MAIN_SITE_REDIRECT_ALLOWLIST = [ '/api/', + '/user_api/', # still used by EdxRestAPIClient in integrations '/admin', 'oauth', # TODO: Add slashes during Nutmeg upgrade since this requires a lot of QA 'status', # TODO: Add slashes during Nutmeg upgrade since this requires a lot of QA From 7a81c0f2d5970900f351b587ebe229ccb8b113bf Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Thu, 29 Sep 2022 10:38:02 +0300 Subject: [PATCH 005/125] remove_site: fix CMS role IntegrityError + refactoring --- .../sites/management/commands/remove_site.py | 4 +-- .../appsembler/sites/tests/test_commands.py | 24 +++++++++++++----- .../sites/tests/test_site_delete_utils.py | 14 +++++++++-- .../core/djangoapps/appsembler/sites/utils.py | 25 +++++++++++++++++-- 4 files changed, 55 insertions(+), 12 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/sites/management/commands/remove_site.py b/openedx/core/djangoapps/appsembler/sites/management/commands/remove_site.py index 0bfb41813757..ae23987dbb6f 100644 --- a/openedx/core/djangoapps/appsembler/sites/management/commands/remove_site.py +++ b/openedx/core/djangoapps/appsembler/sites/management/commands/remove_site.py @@ -30,10 +30,10 @@ def handle(self, *args, **options): organization_domain = options['domain'] self.stdout.write('Removing "%s" in progress...' % organization_domain) - organization = self._get_site(organization_domain) + site = self._get_site(organization_domain) with transaction.atomic(): - delete_site(organization) + delete_site(site) if not options['commit']: transaction.set_rollback(True) diff --git a/openedx/core/djangoapps/appsembler/sites/tests/test_commands.py b/openedx/core/djangoapps/appsembler/sites/tests/test_commands.py index 5e35d4a5d5a2..117718fe213f 100644 --- a/openedx/core/djangoapps/appsembler/sites/tests/test_commands.py +++ b/openedx/core/djangoapps/appsembler/sites/tests/test_commands.py @@ -1,6 +1,6 @@ import hashlib import os -from mock import patch, mock_open +from unittest.mock import patch, mock_open, Mock from io import StringIO from django.conf import settings @@ -9,11 +9,13 @@ from django.core.management import call_command from django.core.management.base import CommandError from django.test import override_settings, TestCase + from tahoe_sites.api import ( create_tahoe_site_by_link, get_organization_for_user, get_users_of_organization, get_uuid_by_organization, + get_organization_by_site, ) from tahoe_sites.tests.utils import create_organization_mapping @@ -160,6 +162,10 @@ def test_run(self): 'DISABLE_COURSE_CREATION': False, 'ENABLE_CREATOR_GROUP': True, }) +@patch( # Avoid CMS-related import issues in tests + 'openedx.core.djangoapps.appsembler.sites.utils.remove_course_creator_role', + Mock() +) class RemoveSiteCommandTestCase(TestCase): """ Test ./manage.py lms remove_site mysite @@ -182,17 +188,23 @@ def test_remove_devstack_site_commit(self): """ deleted_domain = '{}.localhost:18000'.format(self.to_be_deleted) remained_domain = '{}.localhost:18000'.format(self.shall_remain) - + assert SiteConfiguration.objects.count() == 2, 'there are two sites' + remained_site = Site.objects.get(domain=remained_domain) + + # TODO: Re-produce the error we face in staging + to_delete_site = Site.objects.get(domain=deleted_domain) + to_delete_organization = get_organization_by_site(to_delete_site) + users = get_users_of_organization(to_delete_organization) + assert len(users), 'Ensure the site has users' call_command('remove_site', deleted_domain, commit=True) # Ensure objects are removed correctly. assert not Site.objects.filter(domain=deleted_domain).exists() - site = Site.objects.get(domain=remained_domain) - assert SiteConfiguration.objects.count() == 1 - assert SiteConfiguration.objects.get(site=site) + assert SiteConfiguration.objects.count() == 1, 'One site is deleted' + assert SiteConfiguration.objects.get(site=remained_site), 'remained_domain site config is kept' - assert SiteTheme.objects.filter(site=site).count() == site.themes.count() + assert SiteTheme.objects.filter(site=remained_site).count() == remained_site.themes.count() def test_remove_devstack_site_rollback(self): """ diff --git a/openedx/core/djangoapps/appsembler/sites/tests/test_site_delete_utils.py b/openedx/core/djangoapps/appsembler/sites/tests/test_site_delete_utils.py index 887665aea96f..95da3fcef7b2 100644 --- a/openedx/core/djangoapps/appsembler/sites/tests/test_site_delete_utils.py +++ b/openedx/core/djangoapps/appsembler/sites/tests/test_site_delete_utils.py @@ -1,3 +1,5 @@ +from unittest.mock import patch + import pytest import tahoe_sites.api from django.contrib.auth import get_user_model @@ -24,6 +26,14 @@ ) +def delete_site_with_patched_cms_imports(red_site): + """ + Delete a site without running the CMS-related code. + """ + with patch('openedx.core.djangoapps.appsembler.sites.utils.remove_course_creator_role'): + delete_site(red_site) + + @pytest.fixture @pytest.mark.django_db def make_site(settings): @@ -49,7 +59,7 @@ def test_delete_site(make_site): Test `delete_site` happy path. """ red_site = make_site('red') - delete_site(red_site) + delete_site_with_patched_cms_imports(red_site) with pytest.raises(User.DoesNotExist): User.objects.get(username='red') @@ -63,7 +73,7 @@ def test_delete_one_site_keeps_another(make_site): red_site = make_site('red') make_site('blue') - delete_site(red_site) + delete_site_with_patched_cms_imports(red_site) with pytest.raises(User.DoesNotExist): User.objects.get(username='red') diff --git a/openedx/core/djangoapps/appsembler/sites/utils.py b/openedx/core/djangoapps/appsembler/sites/utils.py index e85ca237e2e6..a9acebb217cd 100644 --- a/openedx/core/djangoapps/appsembler/sites/utils.py +++ b/openedx/core/djangoapps/appsembler/sites/utils.py @@ -481,6 +481,9 @@ def bootstrap_site(site, org_data=None, username=None): def get_models_using_course_key(): + """ + Get all course related model classes. + """ course_key_field_names = { 'course_key', 'course_id', @@ -506,6 +509,9 @@ def get_models_using_course_key(): def delete_organization_courses(organization): + """ + Delete all course related model instances. + """ course_keys = [] for course in get_organization_courses({'id': organization.id}): @@ -519,6 +525,21 @@ def delete_organization_courses(organization): objects_to_delete.delete() +def remove_course_creator_role(users): + """ + Remove course creator role to fix `delete_site` issue. + + This will fail in when running tests from within the LMS because the CMS migrations + don't run during tests. Patch this function to avoid such errors. + TODO: RED-2853 Remove this helper when AMC is removed + This helper is being replaced by `update_course_creator_role_for_cms` which has unit tests. + """ + from cms.djangoapps.course_creators.models import CourseCreator # Fix LMS->CMS imports. + from student.roles import CourseAccessRole # Avoid circular import. + CourseCreator.objects.filter(user__in=users).delete() + CourseAccessRole.objects.filter(user__in=users).delete() + + @beeline.traced(name="delete_site") def delete_site(site): print('Deleting SiteConfiguration of', site) @@ -529,9 +550,9 @@ def delete_site(site): organization = tahoe_sites.api.get_organization_by_site(site) - users = tahoe_sites.api.get_users_of_organization(organization, without_inactive_users=False) - print('Deleting users of', site) + users = tahoe_sites.api.get_users_of_organization(organization, without_inactive_users=False) + remove_course_creator_role(users) users.delete() print('Deleting courses of', site) From 892d3a1b4be5b02f25eaf3a62ac6c72f3384bc00 Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Mon, 10 Oct 2022 12:48:27 +0300 Subject: [PATCH 006/125] fix a weird test failure in `create_devstack_site` --- .../core/djangoapps/appsembler/sites/tests/test_commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/appsembler/sites/tests/test_commands.py b/openedx/core/djangoapps/appsembler/sites/tests/test_commands.py index 117718fe213f..636d31b8bc7a 100644 --- a/openedx/core/djangoapps/appsembler/sites/tests/test_commands.py +++ b/openedx/core/djangoapps/appsembler/sites/tests/test_commands.py @@ -102,7 +102,8 @@ def test_create_devstack_site(self): with patch.object(Command, 'congrats') as mock_congrats: call_command('create_devstack_site', self.name, 'localhost') - mock_congrats.assert_called_once() # Ensure that congrats message is printed + assert mock_congrats.called # Ensure that congrats message is printed + assert mock_congrats.call_count == 1 # Ensure that congrats message is printed once # Ensure objects are created correctly. assert Site.objects.get(domain=self.site_name) From eb4181b83f67cd6d365fb495aba75000aa3ddd6f Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Thu, 29 Sep 2022 19:57:04 +0300 Subject: [PATCH 007/125] remove_site: also remove models with LearningContextKeyField --- .../appsembler/sites/tests/test_site_delete_utils.py | 2 ++ openedx/core/djangoapps/appsembler/sites/utils.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/sites/tests/test_site_delete_utils.py b/openedx/core/djangoapps/appsembler/sites/tests/test_site_delete_utils.py index 887665aea96f..9c76cfda09ee 100644 --- a/openedx/core/djangoapps/appsembler/sites/tests/test_site_delete_utils.py +++ b/openedx/core/djangoapps/appsembler/sites/tests/test_site_delete_utils.py @@ -8,6 +8,7 @@ from status.models import CourseMessage from student.models import AnonymousUserId +from lms.djangoapps.courseware.models import StudentModule from openedx.core.djangoapps.appsembler.api.tests.factories import ( CourseOverviewFactory, OrganizationCourseFactory, @@ -87,6 +88,7 @@ def test_get_models_using_course_key(): assert AnonymousUserId in classes, 'Should include AnonymousUserId due to course_id field' assert CourseMessage in classes, 'Should include CourseMessage due to course_key field' assert OrganizationCourse in classes, 'Should include OrganizationCourse' + assert StudentModule in classes, 'Should include models with LearningContextKeyField' @pytest.mark.django_db diff --git a/openedx/core/djangoapps/appsembler/sites/utils.py b/openedx/core/djangoapps/appsembler/sites/utils.py index e85ca237e2e6..6e3299343405 100644 --- a/openedx/core/djangoapps/appsembler/sites/utils.py +++ b/openedx/core/djangoapps/appsembler/sites/utils.py @@ -9,7 +9,7 @@ from datetime import timedelta import beeline -from opaque_keys.edx.django.models import CourseKeyField +from opaque_keys.edx.django.models import CourseKeyField, LearningContextKeyField from urllib.parse import urlparse @@ -497,7 +497,7 @@ def get_models_using_course_key(): field_object = getattr(model_class, field_name, None) if field_object: field_definition = getattr(field_object, 'field', None) - if field_definition and isinstance(field_definition, CourseKeyField): + if field_definition and isinstance(field_definition, (CourseKeyField, LearningContextKeyField)): models_with_course_key.add( (model_class, field_name,) ) From 7376d0bbcee5dab044d6868897abd5d422c6eb53 Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Tue, 11 Oct 2022 10:52:30 +0300 Subject: [PATCH 008/125] don't report conflicts for releases before nutmeg --- .github/workflows/report_conflicts.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/report_conflicts.yml b/.github/workflows/report_conflicts.yml index 1239066f1f87..3c1b5e9e8baa 100644 --- a/.github/workflows/report_conflicts.yml +++ b/.github/workflows/report_conflicts.yml @@ -2,13 +2,13 @@ on: [pull_request] name: 'Merge conflicts' jobs: - report_master: - name: 'koa, lilac, maple, nutmeg and master' + report: + name: 'nutmeg and master' uses: appsembler/action-conflict-counter/.github/workflows/report-via-comment.yml@main with: local_base_branch: ${{ github.base_ref }} upstream_repo: 'https://github.com/edx/edx-platform.git' - upstream_branches: 'open-release/koa.master,open-release/lilac.master,open-release/maple.master,open-release/nutmeg.master,master' + upstream_branches: 'open-release/nutmeg.master,master' exclude_paths: 'cms/static/js/,conf/locale/,lms/static/js/,package.json,package-lock.json,.github/' secrets: custom_github_token: ${{ secrets.GITHUB_TOKEN }} From 773591e37aa5c64fc6617178c6d156c8c65c627b Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Mon, 17 Oct 2022 15:30:34 +0300 Subject: [PATCH 009/125] remove beeline logs from site config get_value it's fast: 52.0nanosec --- openedx/core/djangoapps/site_configuration/models.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/openedx/core/djangoapps/site_configuration/models.py b/openedx/core/djangoapps/site_configuration/models.py index a07fa1d96066..ba2b81e431d7 100644 --- a/openedx/core/djangoapps/site_configuration/models.py +++ b/openedx/core/djangoapps/site_configuration/models.py @@ -103,7 +103,6 @@ def api_adapter(self): return self._api_adapter - @beeline.traced('site_config.get_value') def get_value(self, name, default=None): """ Return Configuration value for the key specified as name argument. @@ -117,7 +116,6 @@ def get_value(self, name, default=None): Returns: Configuration value for the given key or returns `None` if configuration is not enabled. """ - beeline.add_context_field('value_name', name) if self.enabled: if self.tahoe_config_modifier: name, default = self.tahoe_config_modifier.normalize_get_value_params(name, default) @@ -128,10 +126,8 @@ def get_value(self, name, default=None): try: if self.api_adapter: # Tahoe: Use `SiteConfigAdapter` if available. - beeline.add_context_field('value_source', 'site_config_service') return self.api_adapter.get_value_of_type(self.api_adapter.TYPE_SETTING, name, default) else: - beeline.add_context_field('value_source', 'django_model') return self.site_values.get(name, default) if self.site_values else default except AttributeError as error: logger.exception(u'Invalid JSON data. \n [%s]', error) From 9852a41a2b55f114cedc989031d5d7fbe63592aa Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Wed, 19 Oct 2022 11:41:16 +0300 Subject: [PATCH 010/125] add beeline tracing for site config service usage --- .../sites/site_config_client_helpers.py | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/sites/site_config_client_helpers.py b/openedx/core/djangoapps/appsembler/sites/site_config_client_helpers.py index 7c41e436ec8a..351d99892ec0 100644 --- a/openedx/core/djangoapps/appsembler/sites/site_config_client_helpers.py +++ b/openedx/core/djangoapps/appsembler/sites/site_config_client_helpers.py @@ -1,7 +1,7 @@ """ Integration helpers for SiteConfig Client adapter. """ - +import beeline import logging from uuid import UUID @@ -29,22 +29,25 @@ # TODO: Move these helpers into the `site_config_client.openedx.api` module +@beeline.traced('site_config_client_helpers.is_enabled_for_site') def is_enabled_for_site(site): """ Checks if the SiteConfiguration client is enabled for a specific organization. """ from django.conf import settings # Local import to avoid AppRegistryNotReady error - if site.id == settings.SITE_ID: - # Disable the SiteConfig service on main site. - return False - - try: - uuid = tahoe_sites.api.get_uuid_by_site(site) - except ObjectDoesNotExist: - # Return sane result in case of malformed data - return False - return is_feature_enabled_for_site(uuid) + is_enabled = False + if site.id != settings.SITE_ID: # Disable the SiteConfig service on main site. + try: + uuid = tahoe_sites.api.get_uuid_by_site(site) + except ObjectDoesNotExist: + # Act as if disabled in case of malformed data + is_enabled = False + else: + is_enabled = is_feature_enabled_for_site(uuid) + + beeline.add_trace_field('site_config.enabled', is_enabled) + return is_enabled def enable_for_site(site, note=''): From f0e482e264bba0d237a92aa65162ef2c5276d3aa Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Thu, 20 Oct 2022 10:56:10 +0300 Subject: [PATCH 011/125] allow lint specific directory e.g. `tox -e pep8 openedx/` --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 6007a5ab87ff..204809d8b650 100644 --- a/tox.ini +++ b/tox.ini @@ -155,4 +155,4 @@ commands = deps = pycodestyle==2.3.1 commands = - pycodestyle . + pycodestyle {posargs:.} From 8c6c96d5a0cdeadb59e6f6ab8649f1dbc1fc12b7 Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Mon, 24 Oct 2022 10:19:20 +0300 Subject: [PATCH 012/125] delete multiple sites at once --- .../sites/management/commands/remove_site.py | 76 ++++++++++--------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/sites/management/commands/remove_site.py b/openedx/core/djangoapps/appsembler/sites/management/commands/remove_site.py index ae23987dbb6f..10dda1cef134 100644 --- a/openedx/core/djangoapps/appsembler/sites/management/commands/remove_site.py +++ b/openedx/core/djangoapps/appsembler/sites/management/commands/remove_site.py @@ -1,4 +1,6 @@ -from django.core.management.base import BaseCommand, CommandError +import traceback + +from django.core.management.base import BaseCommand from django.contrib.sites.models import Site from django.db import transaction @@ -13,11 +15,6 @@ class Command(BaseCommand): """ def add_arguments(self, parser): - parser.add_argument( - 'domain', - help='The domain of the organization to be deleted.', - type=str, - ) parser.add_argument( '--commit', default=False, @@ -26,33 +23,42 @@ def add_arguments(self, parser): action='store_true', ) + parser.add_argument( + 'domain', + help='The domain of the organization to be deleted.', + nargs='+', + type=str, + ) + def handle(self, *args, **options): - organization_domain = options['domain'] - - self.stdout.write('Removing "%s" in progress...' % organization_domain) - site = self._get_site(organization_domain) - - with transaction.atomic(): - delete_site(site) - - if not options['commit']: - transaction.set_rollback(True) - - self.stdout.write(self.style.SUCCESS( - '{message} removed site "{domain}"'.format( - message='Successfully' if options['commit'] else 'Dry run', - domain=organization_domain, - ) - )) - - def _get_site(self, domain): - """ - Locates the site to be deleted and return its instance. - - :param domain: The domain of the site to be returned. - :return: Returns the site object that has the given domain. - """ - try: - return Site.objects.get(domain=domain) - except Site.DoesNotExist: - raise CommandError('Cannot find "%s" in Sites!' % domain) + domains = options['domain'] + + for domain in domains: + self.stdout.write('Removing "%s" in progress...' % domain) + + try: + site = Site.objects.filter(domain=domain).first() + if not site: + self.stderr.write(self.style.ERROR('Cannot find "{domain}"'.format(domain=domain))) + continue + + with transaction.atomic(): + delete_site(site) + + if not options['commit']: + transaction.set_rollback(True) + except Exception: # noqa + self.stderr.write(self.style.ERROR( + 'Failed to remove site "{domain}" error: \n {error}'.format( + domain=domain, + error=traceback.format_exc(), + ) + )) + traceback.format_exc() + else: + self.stdout.write(self.style.SUCCESS( + '{message} removed site "{domain}"'.format( + message='Successfully' if options['commit'] else 'Dry run', + domain=domain, + ) + )) From 662074a5a26b059e20370f6d9b23d5e41666effb Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Mon, 24 Oct 2022 10:28:29 +0300 Subject: [PATCH 013/125] fix remove_site error with UserTaskStatus ``` django.db.utils.IntegrityError: (1451, 'Cannot delete or update a parent row: a foreign key constraint fails (`edxapp`.`user_tasks_usertaskstatus`, CONSTRAINT `user_tasks_usertaskstat_user_id_5ceae753d027017b_fk_auth_user_id` FOREIGN KEY (`user_id`) REFERENCES `auth_user` (`id`))') ``` --- .../appsembler/settings/settings/production_lms.py | 7 +++++++ .../djangoapps/appsembler/settings/tests/test_settings.py | 1 + 2 files changed, 8 insertions(+) diff --git a/openedx/core/djangoapps/appsembler/settings/settings/production_lms.py b/openedx/core/djangoapps/appsembler/settings/settings/production_lms.py index ad2becf4b78b..a6bb8dbc01db 100644 --- a/openedx/core/djangoapps/appsembler/settings/settings/production_lms.py +++ b/openedx/core/djangoapps/appsembler/settings/settings/production_lms.py @@ -64,6 +64,13 @@ def plugin_settings(settings): tpa_admin_app_name, ] + settings.INSTALLED_APPS += [ + 'user_tasks', # Release upgrade note: This line can be removed if it causes errors, + # but the `remove_site` must be tested afterwards + # `user_tasks` is a CMS-only app, but adding it in LMS to fix an error with `remove_site` command + # `user_tasks` helps to manage of user-triggered async tasks (course import/export, etc.) + ] + settings.CORS_ORIGIN_ALLOW_ALL = True settings.CORS_ALLOW_HEADERS = ( diff --git a/openedx/core/djangoapps/appsembler/settings/tests/test_settings.py b/openedx/core/djangoapps/appsembler/settings/tests/test_settings.py index aec9fbd148d3..e8f136bb8201 100644 --- a/openedx/core/djangoapps/appsembler/settings/tests/test_settings.py +++ b/openedx/core/djangoapps/appsembler/settings/tests/test_settings.py @@ -23,6 +23,7 @@ def fake_production_settings(settings): settings.AUTH_TOKENS = {} settings.CELERY_QUEUES = {} settings.ALTERNATE_QUEUE_ENVS = [] + settings.INSTALLED_APPS = settings.INSTALLED_APPS.copy() # Prevent polluting the original list settings.FEATURES = settings.FEATURES.copy() # Prevent polluting other tests. settings.ENV_TOKENS = { 'LMS_BASE': 'fake-lms-base', From b57c31d8536bbeb03bb4d93a3e35cb399a3e0181 Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Mon, 24 Oct 2022 11:11:03 +0300 Subject: [PATCH 014/125] Prepare removing users by avoiding on_delete=models.PROTECT error SAMLConfiguration will be deleted with `site.delete()` --- openedx/core/djangoapps/appsembler/sites/utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/openedx/core/djangoapps/appsembler/sites/utils.py b/openedx/core/djangoapps/appsembler/sites/utils.py index 83961af43cc2..6380a5640c19 100644 --- a/openedx/core/djangoapps/appsembler/sites/utils.py +++ b/openedx/core/djangoapps/appsembler/sites/utils.py @@ -34,6 +34,7 @@ from organizations.models import OrganizationCourse from organizations.models import Organization + from tahoe_sites.api import ( add_user_to_organization, create_tahoe_site_by_link, @@ -542,6 +543,8 @@ def remove_course_creator_role(users): @beeline.traced(name="delete_site") def delete_site(site): + from third_party_auth.models import SAMLConfiguration # local import to avoid import-time errors + print('Deleting SiteConfiguration of', site) site.configuration.delete() @@ -553,6 +556,11 @@ def delete_site(site): print('Deleting users of', site) users = tahoe_sites.api.get_users_of_organization(organization, without_inactive_users=False) remove_course_creator_role(users) + + # Prepare removing users by avoiding on_delete=models.PROTECT error + # SAMLConfiguration will be deleted with `site.delete()` + SAMLConfiguration.objects.filter(changed_by__in=users).update(changed_by=None) + users.delete() print('Deleting courses of', site) From 161cc3d02fbe87025c35286e0ea12d246cc8123d Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Mon, 24 Oct 2022 15:20:21 +0300 Subject: [PATCH 015/125] less noisy delete_organization_courses logs --- openedx/core/djangoapps/appsembler/sites/utils.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/sites/utils.py b/openedx/core/djangoapps/appsembler/sites/utils.py index 6380a5640c19..11c184c56b77 100644 --- a/openedx/core/djangoapps/appsembler/sites/utils.py +++ b/openedx/core/djangoapps/appsembler/sites/utils.py @@ -518,8 +518,14 @@ def delete_organization_courses(organization): for course in get_organization_courses({'id': organization.id}): course_keys.append(course['course_id']) - for model_class, field_name in get_models_using_course_key(): - print('Deleting models of', model_class.__name__, 'with field', field_name) + model_classes = get_models_using_course_key() + + print('Deleting course related models:', ', '.join([ + '{model}.{field}'.format(model=model_class.__name__, field=field_name) + for model_class, field_name in model_classes + ])) + + for model_class, field_name in model_classes: objects_to_delete = model_class.objects.filter(**{ '{field_name}__in'.format(field_name=field_name): course_keys, }) From 8347b5f9b1bbc07574a5aa3fdcb09a9696f95f38 Mon Sep 17 00:00:00 2001 From: Shadi Naif Date: Thu, 27 Oct 2022 03:50:24 +0300 Subject: [PATCH 016/125] Make studio logout redirect to LMS --- .../tests/test_studio_logout_view.py | 145 ++++++++++++++++++ cms/djangoapps/appsembler/urls.py | 7 +- cms/djangoapps/appsembler/views.py | 45 +++++- 3 files changed, 191 insertions(+), 6 deletions(-) create mode 100644 cms/djangoapps/appsembler/tests/test_studio_logout_view.py diff --git a/cms/djangoapps/appsembler/tests/test_studio_logout_view.py b/cms/djangoapps/appsembler/tests/test_studio_logout_view.py new file mode 100644 index 000000000000..d6c243cf13f2 --- /dev/null +++ b/cms/djangoapps/appsembler/tests/test_studio_logout_view.py @@ -0,0 +1,145 @@ +""" +Tests for APPSEMBLER_MULTI_TENANT_EMAILS in Studio logout. + +Special note: + +This test module needs to patch `cms.urls.urlpatterns` to include urlpatterns +from `cms.djangoapps.appsembler.urls`. This works by overriding the +`doango.conf.settings.ROOT_URLCONF` with `django.test.utils.override_settings` +at the TestCase class level with the `urlpatterns` list declared in the module +containing the TestCase class. + +For this test module, we've added a `urlpatterns` module level variable and +assigned it the value of `cms.urls.urlpatterns` then appended the conditionally +included urlpatterns we need to run the tests. + +Then we add `@override_settings(ROOT_URLCONF=__name__)` to the TestClass + +There are other ways to do this. However, this is simple and does not require +our code to explicitly hack `sys.modules` reloading +""" +from unittest.mock import Mock, patch + +from django.conf import settings +from django.conf.urls import include, url +from django.contrib import auth +from django.contrib.auth.models import AnonymousUser +from django.urls import reverse +from django.test import RequestFactory, TestCase +from django.test.utils import override_settings +from rest_framework import status +from tahoe_sites.api import add_user_to_organization, create_tahoe_site + +from student.tests.factories import UserFactory +import cms.urls +from cms.djangoapps.appsembler.views import get_logout_redirect_url + + +# Set the urlpatterns we want to use for our tests in this module only +urlpatterns = cms.urls.urlpatterns + [ + url(r'', include('cms.djangoapps.appsembler.urls')) +] + + +@override_settings(ROOT_URLCONF=__name__) # the module that contains `urlpatterns` +@override_settings(LOGOUT_REDIRECT_URL='home') # ensure that we have a value for LOGOUT_REDIRECT_URL +@patch.dict('django.conf.settings.FEATURES', {'TAHOE_STUDIO_LOCAL_LOGIN': True}) +class TestStudioLogoutView(TestCase): + """ + Testing the APPSEMBLER_MULTI_TENANT_EMAILS feature when enabled in Studio. + """ + BLUE = 'blue1' + EMAIL = 'customer@example.com' + PASSWORD = 'xyz' + DOMAIN = 'testdomain.com' + SHORT_NAME = 'testdomain' + + def setUp(self): + super(TestStudioLogoutView, self).setUp() + self.url = reverse('logout') + self.user = UserFactory.create(email=self.EMAIL, password=self.PASSWORD) + add_user_to_organization( + user=self.user, + organization=create_tahoe_site(domain=self.DOMAIN, short_name=self.SHORT_NAME)['organization'] + ) + self.request = RequestFactory() + self.request.is_secure = Mock(return_value=False) + self.lms_url = 'http://{site_domain}/logout'.format(site_domain=self.DOMAIN) + + def test_logout_must_be_authenticated(self): + """ + Test logout from studio must be authenticated + """ + response = self.client.get(self.url) + assert response.status_code == status.HTTP_302_FOUND + assert not response.content + assert '?next=/logout' in response.url + + def test_logout_normal_user(self): + """ + Test logout from studio for normal users (meaning that they are linked to an organization). It is expected + that a logout then redirect to LMS will be performed + """ + self.client.login(username=self.user.username, password=self.PASSWORD) + assert auth.get_user(self.client).is_authenticated + + response = self.client.get(self.url) + assert not auth.get_user(self.client).is_authenticated + + assert response.status_code == status.HTTP_302_FOUND + assert not response.content + assert response.url == self.lms_url + + def test_logout_staff_user(self): + """ + Test logout from studio for staff users (meaning that they are not linked to any organization). It is expected + that a logout then a redirect to settings.LOGOUT_REDIRECT_URL will be performed + """ + # Not necessary to set the user as staff, the thing we need to test is when it lacks a link to an organization + user = UserFactory.create(email=self.EMAIL, password=self.PASSWORD) + + self.client.login(username=user.username, password=self.PASSWORD) + assert auth.get_user(self.client).is_authenticated + + response = self.client.get(self.url) + assert not auth.get_user(self.client).is_authenticated + + assert response.status_code == status.HTTP_302_FOUND + assert not response.content + assert response.url == reverse(settings.LOGOUT_REDIRECT_URL) + + def test_get_logout_redirect_url_no_request(self): + """ + Verify that get_logout_redirect_url will return settings.LOGOUT_REDIRECT_URL if the request is None + """ + assert get_logout_redirect_url(request=None) == reverse(settings.LOGOUT_REDIRECT_URL) + + def test_get_logout_redirect_url_no_user(self): + """ + Verify that get_logout_redirect_url will return settings.LOGOUT_REDIRECT_URL if no user is logged in + """ + assert not hasattr(self.request, 'user') + assert get_logout_redirect_url(request=self.request) == reverse(settings.LOGOUT_REDIRECT_URL) + + def test_get_logout_redirect_url_anonymous(self): + """ + Verify that get_logout_redirect_url will return settings.LOGOUT_REDIRECT_URL if the user is anonymous + """ + self.request.user = AnonymousUser() + assert get_logout_redirect_url(request=self.request) == reverse(settings.LOGOUT_REDIRECT_URL) + + def test_get_logout_redirect_url_user(self): + """ + Verify that get_logout_redirect_url will return the LMS URL related to the user + """ + self.request.user = self.user + assert get_logout_redirect_url(request=self.request) == self.lms_url + + def test_get_logout_redirect_url_staff(self): + """ + Verify that get_logout_redirect_url will return settings.LOGOUT_REDIRECT_URL if the user is not linked to + any organization (staff users and superusers) + """ + user = UserFactory.create(email=self.EMAIL, password=self.PASSWORD) + self.request.user = user + assert get_logout_redirect_url(request=self.request) == reverse(settings.LOGOUT_REDIRECT_URL) diff --git a/cms/djangoapps/appsembler/urls.py b/cms/djangoapps/appsembler/urls.py index 200913eeb344..6562882ef50c 100644 --- a/cms/djangoapps/appsembler/urls.py +++ b/cms/djangoapps/appsembler/urls.py @@ -5,13 +5,10 @@ We have this code in the Appsembler CMS app to help isolate custom code """ -from django.conf import settings from django.urls import path -from django.contrib.auth.views import LogoutView -from .views import LoginView +from .views import LoginView, StudioLogoutView urlpatterns = [ path('login/', LoginView.as_view(), name='login'), - path('logout/', LogoutView.as_view( - next_page=settings.LOGOUT_REDIRECT_URL), name='logout'), + path('logout/', StudioLogoutView.as_view(), name='logout'), ] diff --git a/cms/djangoapps/appsembler/views.py b/cms/djangoapps/appsembler/views.py index c3faf55bc4f0..77297ca02e9f 100644 --- a/cms/djangoapps/appsembler/views.py +++ b/cms/djangoapps/appsembler/views.py @@ -9,6 +9,9 @@ import logging from django.conf import settings from django.contrib.auth import authenticate, get_user_model, login +from django.contrib.auth.decorators import login_required +from django.contrib.auth.views import LogoutView as DjangoLogoutView +from django.core.exceptions import ObjectDoesNotExist from django.db.models import Q from django.http import HttpResponseServerError from django.shortcuts import redirect @@ -18,7 +21,11 @@ from django.views import View from django.views.decorators.clickjacking import xframe_options_deny from django.views.decorators.csrf import csrf_protect -from tahoe_sites.api import deprecated_get_admin_users_queryset_by_email +from tahoe_sites.api import ( + deprecated_get_admin_users_queryset_by_email, + get_organization_for_user, + get_site_by_organization, +) from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers from openedx.core.djangoapps.user_authn.utils import is_safe_login_or_logout_redirect @@ -274,3 +281,39 @@ def log_multiple_objects_returned(self): def render_login_page_with_error(self, error_code): return render_login_page( login_error_message=self.error_messages[error_code]) + + +def get_logout_redirect_url(request): + """ + Return logout redirect url using the site related to the given user if possible. + Otherwise, return settings.LOGOUT_REDIRECT_URL + + :return: logout redirect url or settings.LOGOUT_REDIRECT_URL + """ + user = getattr(request, 'user', None) + if not user or not user.is_authenticated: + return reverse(settings.LOGOUT_REDIRECT_URL) + + try: + organization = get_organization_for_user(user=user) + except ObjectDoesNotExist: + return reverse(settings.LOGOUT_REDIRECT_URL) + site = get_site_by_organization(organization=organization) + + return '{protocol}://{site_domain}/logout'.format( + protocol='https' if request.is_secure() else 'http', + site_domain=site.domain + ) + + +class StudioLogoutView(View): + """ + Studio Logout View + """ + @method_decorator(csrf_protect) + @method_decorator(login_required) + def get(self, request): + """ + Perform logout from studio, and redirect to LMS home page + """ + return DjangoLogoutView.as_view(next_page=get_logout_redirect_url(request))(request) From 5afa96235103d9a8be7549e389a8da6f431080cb Mon Sep 17 00:00:00 2001 From: Shadi Naif Date: Thu, 27 Oct 2022 03:51:22 +0300 Subject: [PATCH 017/125] Add get_redirect_to_lms_login_url helper --- .../appsembler/tahoe_idp/helpers.py | 73 +++++++++++- .../tahoe_idp/tests/test_tahoe_idp_helpers.py | 106 ++++++++++++++++++ 2 files changed, 178 insertions(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/appsembler/tahoe_idp/helpers.py b/openedx/core/djangoapps/appsembler/tahoe_idp/helpers.py index 9f316653b051..0a3d3bc20625 100644 --- a/openedx/core/djangoapps/appsembler/tahoe_idp/helpers.py +++ b/openedx/core/djangoapps/appsembler/tahoe_idp/helpers.py @@ -4,14 +4,22 @@ - https://github.com/appsembler/tahoe-idp/ """ +import re +from urllib import parse from collections import OrderedDict from django.conf import settings +from django.core.exceptions import ObjectDoesNotExist from django.urls import reverse from django.utils.http import urlencode from site_config_client.openedx import api as config_client_api -from tahoe_sites.api import is_active_admin_on_organization, get_organization_for_user +from organizations.models import Organization +from tahoe_sites.api import ( + is_active_admin_on_organization, + get_organization_for_user, + get_site_by_organization, +) import third_party_auth from third_party_auth.pipeline import running as pipeline_running @@ -34,6 +42,20 @@ from .course_roles import TahoeCourseAuthorRole +ALLOWED_KEY_CHAR = r'[\w\-~.:%]' +KEY_PARTS = '(?P{ALLOWED_KEY_CHAR}+)\\+(?P{ALLOWED_KEY_CHAR}+)\\+(?P{ALLOWED_KEY_CHAR}+)' \ + .format(ALLOWED_KEY_CHAR=ALLOWED_KEY_CHAR) +VALID_SEPARATOR = '[/@&\\-\\?\\+]' +VALID_LOCATOR = '(\\bcourse|\\bblock)-v1:{KEY_PARTS}'.format(KEY_PARTS=KEY_PARTS) +VALID_PRE_KEY = '(.*{VALID_SEPARATOR})|({VALID_SEPARATOR})'.format(VALID_SEPARATOR=VALID_SEPARATOR) +VALID_POST_KEY = '({VALID_SEPARATOR}.*|({VALID_SEPARATOR}))'.format(VALID_SEPARATOR=VALID_SEPARATOR) +VALID_URL = '(?P{VALID_PRE_KEY})?(?P{VALID_LOCATOR})(?P{VALID_POST_KEY})?'.format( + VALID_LOCATOR=VALID_LOCATOR, + VALID_PRE_KEY=VALID_PRE_KEY, + VALID_POST_KEY=VALID_POST_KEY, +) +URL_WITH_LOCATOR_REGEX = re.compile(VALID_URL, re.UNICODE) + def is_tahoe_idp_enabled(): """ @@ -185,3 +207,52 @@ def is_studio_login_form_overridden(): if settings.FEATURES.get('TAHOE_IDP_STUDIO_LOGIN_FORM_OVERRIDE', None): return True return False + + +def extract_organization_from_url(url): + """ + Extracts the organization from the given url + + :param url: source uri to extract the course_id from + :return: organization if found, None otherwise + """ + url = url or '' + organization = None + + match = re.search(URL_WITH_LOCATOR_REGEX, url) + if match: + try: + organization = Organization.objects.get( + short_name=match.group('org_short_name') + ) + except ObjectDoesNotExist: + pass + return organization + + +def get_redirect_to_lms_login_url(request): + """ + Get organization site from course id if found in (next) argument of (request.get_full_path()). Then return + the appropriate URL for to studio Magic Link authentication. + + :param request: full path from the request to be processed + :return: redirect url if a valid course key found, otherwise return empty string + """ + if not (request and request.GET.get('next')): + return '' + + next_url = request.GET['next'] + organization = extract_organization_from_url(next_url) + + if organization: + site = get_site_by_organization(organization=organization) + + protocol = 'https' if request.is_secure() else 'http' + redirect_url = '{protocol}://{site_domain}/studio/?next={quoted_next}'.format( + protocol=protocol, + site_domain=site.domain, + quoted_next=parse.quote_plus(next_url), + ) + return redirect_url + + return '' diff --git a/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_helpers.py b/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_helpers.py index 2393b29cd8aa..868795c30ea0 100644 --- a/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_helpers.py +++ b/openedx/core/djangoapps/appsembler/tahoe_idp/tests/test_tahoe_idp_helpers.py @@ -2,8 +2,10 @@ Tests for `tahoe_idp.helpers`. """ from unittest.mock import patch, Mock +from urllib import parse from django.conf import settings +from django.test import RequestFactory import pytest from organizations.tests.factories import OrganizationFactory @@ -38,6 +40,17 @@ def user_with_org(): return learner, organization +@pytest.fixture +def valid_request(): + """ + :return: a request that can be used in our tests here + """ + request = RequestFactory() + request.GET = {} + request.is_secure = Mock(return_value=False) + return request + + @pytest.mark.parametrize('global_flags,site_flags,should_be_enabled,message', [ ({}, {'ENABLE_TAHOE_IDP': True}, True, 'site-flag should enable it'), ({'ENABLE_TAHOE_IDP': True}, {}, True, 'cluster-wide flag should enable it'), @@ -232,3 +245,96 @@ def test_is_studio_login_form_overridden_flag_available(flag_value, expected_res """ with patch.dict('django.conf.settings.FEATURES', {'TAHOE_IDP_STUDIO_LOGIN_FORM_OVERRIDE': flag_value}): assert helpers.is_studio_login_form_overridden() is expected_result + + +@pytest.mark.parametrize('url', [ + 'course-v1:ninja_org+course+2022', + '/course-v1:ninja_org+course+2022', + 'bla_bla_bla/course-v1:ninja_org+course+2022/', + 'course-v1:ninja_org+course+2022/bla_bla_bla', + 'bla_bla_bla/course-v1:ninja_org+course+2022/bla_bla_bla', + 'bla_bla_bla/course-v1:ninja_org+course+2022/bla_bla_bla-v1:ninja_org+course+2022', + 'bla_bla_bla/course-v1:ninja_org+course+2022/bla_bla_bla/course-v1:ninja_org+course+2022', + 'bla_bla_bla/course-v1:unexpected_other_org+course+2022/bla_bla_bla/course-v1:ninja_org+course+2022/bla_bla_bla', +]) +@pytest.mark.django_db +def test_extract_organization_from_url_success(url): + """ + Verify that extract_organization_from_url returns the expected organization or None according to the given URL + """ + organization = OrganizationFactory.create(short_name='ninja_org', name='ninja_org_long_name') + assert helpers.extract_organization_from_url(url) == organization + url = url.replace('course-v1', 'block-v1') + assert helpers.extract_organization_from_url(url) == organization + + +@pytest.mark.parametrize('url', [ + 'course-v1:ninja_org+course+', + 'course-v1:ninja_org+course', + 'course-v2:ninja_org+course+2022', + 'courses-v1:ninja_org+course+2022', + 'asdasdcourse-v1:ninja_org+course+2022', + 'asdasdcourse-v1:ninja_org+course+2022asdasdad', + 'bla_bla_bla/course-v1:ninja_org+course+2022/bla_bla_bla/course-v1:unexpected_other_org+course+2022/bla_bla_bla', +]) +@pytest.mark.django_db +def test_extract_organization_from_url_not_found(url): + """ + Verify that extract_organization_from_url returns the expected organization or None according to the given URL + """ + # just to verify that the function doesn't return (None) because of a missing organization + OrganizationFactory.create(short_name='ninja_org', name='ninja_org_long_name') + + assert helpers.extract_organization_from_url(url) is None + + +def test_get_redirect_to_lms_login_url_no_request(): + """ + Verify that get_redirect_to_lms_login_url will return empty string if no request is provided + """ + assert helpers.get_redirect_to_lms_login_url(None) == '' + + +@pytest.mark.django_db +def test_get_redirect_to_lms_login_url_no_next(valid_request): + """ + Verify that get_redirect_to_lms_login_url will return empty string if no request is provided + """ + assert helpers.get_redirect_to_lms_login_url(valid_request) == '' + + +@pytest.mark.django_db +def test_get_redirect_to_lms_login_url_next_with_no_course(valid_request): + """ + Verify that get_redirect_to_lms_login_url will return empty string if no request is provided + """ + valid_request.GET['next'] = 'bla_bla' + assert helpers.get_redirect_to_lms_login_url(valid_request) == '' + + +@pytest.mark.django_db +def test_get_redirect_to_lms_login_url_next_with_invalid_course(valid_request): + """ + Verify that get_redirect_to_lms_login_url will return empty string if no request is provided + """ + valid_request.GET['next'] = 'course/course-v1:ORG+DoesNotExist' + assert helpers.get_redirect_to_lms_login_url(valid_request) == '' + + +@pytest.mark.django_db +def test_get_redirect_to_lms_login_url_next_with_valid_course(valid_request, user_with_org): + """ + Verify that get_redirect_to_lms_login_url will return the expected URL is a valid course_id is provided + """ + _, organization = user_with_org + site = tahoe_sites_apis.get_site_by_organization(organization=organization) + + next_url = 'container/block-v1:{org}+course+run'.format(org=organization.short_name) + encoded_next = parse.quote_plus(next_url) + expected_url = 'http://{site_domain}/studio/?next={encoded_next}'.format( + site_domain=site.domain, + encoded_next=encoded_next + ) + + valid_request.GET['next'] = next_url + assert helpers.get_redirect_to_lms_login_url(valid_request) == expected_url From 15c595355f678202d3824ff7fe0098e37c364b5f Mon Sep 17 00:00:00 2001 From: Shadi Naif Date: Mon, 31 Oct 2022 10:28:05 +0300 Subject: [PATCH 018/125] Increase page limit from 20 to 100 - RED-3598 --- lms/static/js/discovery/models/search_state.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/static/js/discovery/models/search_state.js b/lms/static/js/discovery/models/search_state.js index cd37f938ea91..e727938cd81f 100644 --- a/lms/static/js/discovery/models/search_state.js +++ b/lms/static/js/discovery/models/search_state.js @@ -11,7 +11,7 @@ return Backbone.Model.extend({ page: 0, - pageSize: 20, + pageSize: 100, // Tahoe: fix a bug related to Course Access Groups - RED-3598 searchTerm: '', terms: {}, jqhxr: null, From 6e1b1b9a30234245f69355b6bb85c687f48d2b0f Mon Sep 17 00:00:00 2001 From: Shadi Naif Date: Wed, 2 Nov 2022 16:05:07 +0300 Subject: [PATCH 019/125] Ensure saving the correct course-key in Enrollment API --- .../api/tests/test_enrollment_api.py | 71 ++++++++++++++++++- .../djangoapps/appsembler/api/v1/views.py | 10 ++- 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/api/tests/test_enrollment_api.py b/openedx/core/djangoapps/appsembler/api/tests/test_enrollment_api.py index 0c7061d02288..4bc844c62bef 100644 --- a/openedx/core/djangoapps/appsembler/api/tests/test_enrollment_api.py +++ b/openedx/core/djangoapps/appsembler/api/tests/test_enrollment_api.py @@ -16,6 +16,8 @@ import ddt import mock +from six import text_type +from opaque_keys.edx.keys import CourseKey from tahoe_sites.api import update_admin_role_in_organization from openedx.core.djangoapps.waffle_utils.testutils import override_waffle_flag @@ -41,7 +43,7 @@ OrganizationFactory, OrganizationCourseFactory, ) - +from openedx.core.djangoapps.content.course_overviews.models import CourseOverview APPSEMBLER_API_VIEWS_MODULE = 'openedx.core.djangoapps.appsembler.api.v1.views' @@ -382,6 +384,73 @@ def test_enroll_learner_by_username(self): assert 'invalidIdentifier' not in response.content.decode(), message assert CourseEnrollment.is_enrolled(registered_user, co.id), 'Enrollment is successful by username' + def test_enrollment_with_bad_course_id(self): + """ + Sometimes, the API receives the course_key in the wrong letters-case + For example, (course-v1:org+name+number_number) instead of (course-v1:org+Name+Number_number) + + This will still be evaluated correctly when we use (CourseOverview.get_from_id), but we must also ensure that + the API is creating the enrollment with the correct string. Because (CourseKey.from_string) cannot fix + wrong IDs saved in (CourseEnrollment.course_id) + + The test is a bit complicated because django with SQLite cannot filter case-insensitive. Therefore, we have + to do some tricks with mocks to mimic MySQL behavior + """ + # Prepare course for testing + course = CourseFactory.create() + course_overview = CourseOverviewFactory(id=course.id) + OrganizationCourseFactory(organization=self.my_site_org, course_id=text_type(course.id)) + lowercase_key = CourseKey.from_string(text_type(course.id).lower()) + registered_user = UserFactory() + create_organization_mapping(user=registered_user, organization=self.my_site_org) + payload = { + 'action': 'enroll', + 'auto_enroll': True, + 'identifiers': [registered_user.username], + 'email_learners': True, + 'courses': [lowercase_key], + } + + # Double check that course.id and lowercase_key are not identical + self.assertNotEqual(course.id, lowercase_key) + self.assertEqual(text_type(course.id).lower(), text_type(lowercase_key).lower()) + + # CourseOverview.get_from_id will find the course even if the given key has the wrong letters-case + # because MySQL can do that. But in SQLite tests, it will fail + self.assertEqual(CourseOverview.get_from_id(course.id), course_overview) + with self.assertRaises(CourseOverview.DoesNotExist): + # This will fail because of SQLite limitations https://www.sqlite.org/faq.html#q18 + CourseOverview.get_from_id(lowercase_key) + + # As a workaround, we can force (get_from_id) to return the course by mocking (load_from_module_store) + with mock.patch( + 'openedx.core.djangoapps.content.course_overviews.models.CourseOverview.load_from_module_store', + return_value=course_overview, + ): + self.assertEqual(CourseOverview.get_from_id(lowercase_key), course_overview) + + # For the same reasons described about (get_from_id), we must mock (get_site_for_course) and (get_course_by_id) + with mock.patch( + 'openedx.core.djangoapps.content.course_overviews.models.CourseOverview.load_from_module_store', + return_value=course_overview, + ): + with mock.patch( + 'openedx.core.djangoapps.appsembler.api.sites.get_site_for_course', + return_value=self.my_site + ): + with mock.patch( + 'openedx.core.djangoapps.appsembler.api.v1.views.get_course_by_id', + return_value=course + ): + response = self.call_enrollment_api('post', self.my_site, self.caller, {'data': payload}) + assert response.status_code == status.HTTP_201_CREATED, response.content + assert 'invalidIdentifier' not in response.content.decode() + + # Finally, use SQLite limitation to verify that enrollment was saved using the correct letters case regardless + # of the fact that we sent a wrong one to the API + assert CourseEnrollment.objects.filter(course_id=course.id).count() == 1 + assert CourseEnrollment.objects.filter(course_id=lowercase_key).count() == 0 + @ddt.ddt @mock.patch(APPSEMBLER_API_VIEWS_MODULE + '.EnrollmentViewSet.throttle_classes', []) diff --git a/openedx/core/djangoapps/appsembler/api/v1/views.py b/openedx/core/djangoapps/appsembler/api/v1/views.py index f39f9e76d39b..edb7371789a3 100644 --- a/openedx/core/djangoapps/appsembler/api/v1/views.py +++ b/openedx/core/djangoapps/appsembler/api/v1/views.py @@ -345,7 +345,15 @@ def create(self, request, *args, **kwargs): results = [] for course_id in serializer.data.get('courses'): - course_key = as_course_key(course_id) + try: + # To avoid running into MongoDB vs. MySQL case sensitivity issues, and to avoid having + # CourseEnrollment.course_id returning the ID with the wrong letters case; we convert + # the key to the correct letter case one. RED-3540 + course_key = CourseOverview.get_from_id(course_id).id + except CourseOverview.DoesNotExist: + # We allow enrollments to non-existence courses! + course_key = as_course_key(course_id) + # TODO: The two checks below deserve a refactor to make it clearer or a v2 API that works on a # single course and use `instructor/views/api.py:students_update_enrollment` directly. # Ensuring the course is linked to an organization. It's somewhat a legacy code, keeping From ca0136e3e2945cb75af381fecc6e65a9b4d390f6 Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Tue, 8 Nov 2022 17:49:54 +0300 Subject: [PATCH 020/125] Avoid touching StudentModule.modified during celery tasks This is a hacky fix for RED-3616. It should avoid breaking MAU via celery tasks until we find a better fix. --- lms/djangoapps/courseware/models.py | 22 ++++- .../tests/test_tahoe_student_module_hack.py | 84 +++++++++++++++++++ 2 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 lms/djangoapps/courseware/tests/test_tahoe_student_module_hack.py diff --git a/lms/djangoapps/courseware/models.py b/lms/djangoapps/courseware/models.py index d6dae57db4c5..09d64eb9fa63 100644 --- a/lms/djangoapps/courseware/models.py +++ b/lms/djangoapps/courseware/models.py @@ -37,6 +37,21 @@ log = logging.getLogger("edx.courseware") +def should_update_student_module_modified_on_save(): + """ + Fixes RED-3616 to avoid updating StudentModule.modified after regrade or any celery automated updates. + + The modified field is relied upon for Monthly Active Users calculations, so any celery task updates it would be + confused with leaner activity. + + Hack: This duplicates the `common.djangoapps.track.shim:is_celery_worker` helper function, but hopefully we'll + be removing this soon enough that code duplication won't be a problem. + """ + is_celery_worker = getattr(settings, 'IS_CELERY_WORKER', False) + is_hack_enabled = settings.FEATURES.get('TAHOE_STUDENT_MODULES_DISABLE_MODIFIED_IN_CELERY', True) + return not (is_celery_worker and is_hack_enabled) + + def chunks(items, chunk_size): """ Yields the values from items in chunks of size chunk_size @@ -119,7 +134,12 @@ class Meta(object): done = models.CharField(max_length=8, choices=DONE_TYPES, default='na') created = models.DateTimeField(auto_now_add=True, db_index=True) - modified = models.DateTimeField(auto_now=True, db_index=True) + + if should_update_student_module_modified_on_save(): + modified = models.DateTimeField(auto_now=True, db_index=True) + else: + # Fixes RED-3616 to modified on celery. + modified = models.DateTimeField(auto_now_add=True, db_index=True) @classmethod def all_submitted_problems_read_only(cls, course_id): diff --git a/lms/djangoapps/courseware/tests/test_tahoe_student_module_hack.py b/lms/djangoapps/courseware/tests/test_tahoe_student_module_hack.py new file mode 100644 index 000000000000..7ea880d33060 --- /dev/null +++ b/lms/djangoapps/courseware/tests/test_tahoe_student_module_hack.py @@ -0,0 +1,84 @@ +""" +Tests for the RED-3616 hack/fix for MAU calculations depending on StudentModule.modified. +""" + +from importlib import reload + +import pytest +from freezegun import freeze_time + + +def import_fresh_models(): + """ + Import `lms.djangoapps.courseware.models` and reload it to react to features. + """ + from lms.djangoapps.courseware import models as courseware_models + from lms.djangoapps.courseware.tests import factories as courseware_factories + reload(courseware_models) + reload(courseware_factories) + return { + 'courseware_models': courseware_models, + 'courseware_factories': courseware_factories, + } + + +def test_is_untouched_by_default_in_celery(settings): + settings.IS_CELERY_WORKER = True + courseware_models = import_fresh_models()['courseware_models'] + assert not courseware_models.should_update_student_module_modified_on_save(), 'Should be enabled for celery' + + +def test_in_updated_by_default_in_http_requests(settings): + settings.IS_CELERY_WORKER = False + courseware_models = import_fresh_models()['courseware_models'] + assert courseware_models.should_update_student_module_modified_on_save(), 'Should be disabled in http requests' + + +def test_can_be_updated_in_celery_if_needed(settings): + """ + TAHOE_STUDENT_MODULES_DISABLE_MODIFIED_IN_CELERY is on by default but can be turned off via lms FEATURES. + """ + settings.IS_CELERY_WORKER = True + settings.FEATURES = { + **settings.FEATURES, + 'TAHOE_STUDENT_MODULES_DISABLE_MODIFIED_IN_CELERY': False, + } + courseware_models = import_fresh_models()['courseware_models'] + assert courseware_models.should_update_student_module_modified_on_save(), 'The feature is configurable' + + +@pytest.mark.django_db +def test_new_student_module_with_in_celery(settings): + """ + Ensure that `StudentModule.modified` isn't updated when saving from within a celery task. + """ + settings.IS_CELERY_WORKER = True + courseware_factories = import_fresh_models()['courseware_factories'] + + with freeze_time('2012-01-14'): + student_module = courseware_factories.StudentModuleFactory.create() + assert student_module.created.year == 2012 + assert student_module.modified.year == 2012 + + with freeze_time('2020-12-20'): + student_module.save() + assert student_module.created.year == 2012 + assert student_module.modified.year == 2012, 'Should not touch `modified` during in celery' + + +@pytest.mark.django_db +def test_new_student_module_with_in_http_requests(settings): + """ + Ensure that `StudentModule.modified` _is updated_ when saving from within an HTTP request. + """ + courseware_factories = import_fresh_models()['courseware_factories'] + + with freeze_time('2012-01-14'): + student_module = courseware_factories.StudentModuleFactory.create() + assert student_module.created.year == 2012 + assert student_module.modified.year == 2012 + + with freeze_time('2020-12-20'): + student_module.save() + assert student_module.created.year == 2012 + assert student_module.modified.year == 2020, 'Should update `modified` outside celery' From 51a741f46a3b6ca0266540daf383ba74457a70c9 Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Wed, 29 Jun 2022 16:45:40 +0300 Subject: [PATCH 021/125] delete stale courses (without active organization) --- cms/djangoapps/appsembler/apps.py | 14 ++ .../appsembler/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../commands/cms_remove_stray_courses.py | 100 ++++++++++ .../appsembler/tests/test_deletion_command.py | 35 ++++ cms/envs/common.py | 3 + .../core/djangoapps/appsembler/sites/api.py | 4 +- .../appsembler/sites/deletion_utils.py | 184 ++++++++++++++++++ .../commands/lms_remove_stray_courses.py | 48 +++++ .../sites/management/commands/remove_site.py | 2 +- .../appsembler/sites/tests/test_commands.py | 2 +- .../sites/tests/test_site_delete_utils.py | 34 +++- .../core/djangoapps/appsembler/sites/utils.py | 104 ---------- 13 files changed, 415 insertions(+), 115 deletions(-) create mode 100644 cms/djangoapps/appsembler/apps.py create mode 100644 cms/djangoapps/appsembler/management/__init__.py create mode 100644 cms/djangoapps/appsembler/management/commands/__init__.py create mode 100644 cms/djangoapps/appsembler/management/commands/cms_remove_stray_courses.py create mode 100644 cms/djangoapps/appsembler/tests/test_deletion_command.py create mode 100644 openedx/core/djangoapps/appsembler/sites/deletion_utils.py create mode 100644 openedx/core/djangoapps/appsembler/sites/management/commands/lms_remove_stray_courses.py diff --git a/cms/djangoapps/appsembler/apps.py b/cms/djangoapps/appsembler/apps.py new file mode 100644 index 000000000000..1f7c35a3b84c --- /dev/null +++ b/cms/djangoapps/appsembler/apps.py @@ -0,0 +1,14 @@ +""" +Appsembler CMS App Configuration +""" + + +from django.apps import AppConfig + + +class CMSAppsemblerConfig(AppConfig): + """ + Application Configuration for Badges. + """ + name = u'appsembler' + plugin_app = {} diff --git a/cms/djangoapps/appsembler/management/__init__.py b/cms/djangoapps/appsembler/management/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cms/djangoapps/appsembler/management/commands/__init__.py b/cms/djangoapps/appsembler/management/commands/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cms/djangoapps/appsembler/management/commands/cms_remove_stray_courses.py b/cms/djangoapps/appsembler/management/commands/cms_remove_stray_courses.py new file mode 100644 index 000000000000..2c6897fff6da --- /dev/null +++ b/cms/djangoapps/appsembler/management/commands/cms_remove_stray_courses.py @@ -0,0 +1,100 @@ +""" +Command to remove courses without associated organization. + +This command is intended as a follow-up step after `remove_site` but can be run independently. +""" + +from django.core.management.base import BaseCommand, CommandError +from django.conf import settings +from opaque_keys.edx.keys import CourseKey + +from xmodule.contentstore.django import contentstore +from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.django import modulestore + +from contentstore.utils import delete_course + +from openedx.core.djangoapps.content.course_overviews.models import CourseOverview +from openedx.core.djangoapps.appsembler.sites.deletion_utils import ( + confirm_deletion, +) + + +def get_deletable_course_keys_from_mongo(): + """ + Get keys of courses without active organization. + """ + mongodb_course_keys = {str(mongodb_course.id) for mongodb_course in modulestore().get_course_summaries()} + mysql_course_keys = {str(mysql_course_key) for mysql_course_key in CourseOverview.get_all_course_keys()} + return list(mongodb_course_keys - mysql_course_keys) + + +def delete_course_and_assets(course_key): + """ + Delete all courses without active organization. + """ + course_key_obj = CourseKey.from_string(course_key) + delete_course(course_key_obj, ModuleStoreEnum.UserID.mgmt_command, keep_instructors=False) + contentstore().delete_all_course_assets(course_key_obj) + + +def cms_remove_stray_courses(commit, limit): + """ + Remove all courses from mongodb that has no CourseOverview entry in MySQL. + """ + course_keys = get_deletable_course_keys_from_mongo() + if limit: + course_keys = course_keys[:limit] + + if not course_keys: + raise CommandError('No courses found to delete.') + + str_course_list = [str(course_key) for course_key in course_keys] + print('Preparing to delete:') + print('\n'.join(str_course_list)) + commit = confirm_deletion( + question='Do you confirm to delete the courses from CMS?', + commit=commit, + ) + + for course_key in course_keys: + if commit: + print('Deleting course: {}'.format(course_key)) + delete_course_and_assets(course_key) + else: + print('[Dry run] deleting course: {}'.format(course_key)) + + print('Finished removing deletable courses') + + +class Command(BaseCommand): + help = "Delete courses that don't belong to organization in `get_active_organizations()`." + + def add_arguments(self, parser): + parser.add_argument( + '--limit', + dest='limit', + default=1, + type=int, + help='Max courses to delete, use 0 to delete all courses.', + ) + + parser.add_argument( + '--commit', + dest='commit', + action='store_true', + help='Remove courses, otherwise only the log will be printed.', + ) + + parser.add_argument( + '--dry-run', + dest='commit', + action='store_false', + help='Do not remove courses, only print the logs.', + ) + + def handle(self, *args, **options): + if settings.ROOT_URLCONF != 'cms.urls': + raise CommandError('This command can only be run in CMS.') + + cms_remove_stray_courses(commit=options.get('commit'), limit=options['limit']) diff --git a/cms/djangoapps/appsembler/tests/test_deletion_command.py b/cms/djangoapps/appsembler/tests/test_deletion_command.py new file mode 100644 index 000000000000..cf95a801ffa2 --- /dev/null +++ b/cms/djangoapps/appsembler/tests/test_deletion_command.py @@ -0,0 +1,35 @@ +""" + +""" + +from django.core.management import call_command, CommandError + +from openedx.core.djangoapps.content.course_overviews.tests.factories import CourseOverviewFactory +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory + + +class DeletionCommandTestCase(ModuleStoreTestCase): + def test_cms_remove_stray_courses_command_no_courses(self): + """ + Raise CommandError if there's no courses to delete. + """ + with self.assertRaises(CommandError): + call_command('cms_remove_stray_courses') + + def test_cms_remove_stray_courses_command(self): + """ + Removes all courses that has only MongoDB entry. + """ + CourseFactory.create() + call_command('cms_remove_stray_courses') + + def test_cms_remove_stray_courses_command_non_to_delete(self): + """ + Should not remove courses from MongoDB if it has a MySQL CourseOverview entry. + """ + course = CourseFactory.create() + CourseOverviewFactory.create(id=course.id) + + with self.assertRaises(CommandError): + call_command('cms_remove_stray_courses') diff --git a/cms/envs/common.py b/cms/envs/common.py index 3e77bc28d3b4..95e57cd3abf6 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -1461,6 +1461,9 @@ # CMS specific user task handling 'cms_user_tasks.apps.CmsUserTasksConfig', + # Appsembler customization app for CMS + 'appsembler.apps.CMSAppsemblerConfig', + # Unusual migrations 'database_fixups', diff --git a/openedx/core/djangoapps/appsembler/sites/api.py b/openedx/core/djangoapps/appsembler/sites/api.py index 0c46491c4a29..f64f96d49e35 100644 --- a/openedx/core/djangoapps/appsembler/sites/api.py +++ b/openedx/core/djangoapps/appsembler/sites/api.py @@ -30,11 +30,11 @@ RegistrationSerializer, AlternativeDomainSerializer, ) -from openedx.core.djangoapps.appsembler.sites.utils import ( - delete_site, +from .utils import ( get_customer_files_storage, to_safe_file_name, ) +from .deletion_utils import delete_site log = logging.Logger(__name__) diff --git a/openedx/core/djangoapps/appsembler/sites/deletion_utils.py b/openedx/core/djangoapps/appsembler/sites/deletion_utils.py new file mode 100644 index 000000000000..e7402c1ba6b0 --- /dev/null +++ b/openedx/core/djangoapps/appsembler/sites/deletion_utils.py @@ -0,0 +1,184 @@ +""" +Site and courses deletion utils. +""" + +import beeline + +from django.apps import apps +from django.core.management import CommandError +from django.db import transaction + +import tahoe_sites.api +from organizations.models import OrganizationCourse + + +from opaque_keys.edx.django.models import CourseKeyField, LearningContextKeyField + +from common.djangoapps.util.organizations_helpers import get_organization_courses + + +from ...content.course_overviews.models import CourseOverview +from organizations.api import get_organization_courses + + +def confirm_deletion(commit, question): + """ + Utility for yes/no interactive confirmation if `commit` is `None`. + """ + if commit is None: + result = input('%s [type yes or no] ' % question) + while not result or result.lower() not in ['yes', 'no']: + result = input('Please answer yes or no: ') + return result == 'yes' + return commit + + +def remove_course_creator_role(users): + """ + Remove course creator role to fix `delete_site` issue. + + This will fail in when running tests from within the LMS because the CMS migrations + don't run during tests. Patch this function to avoid such errors. + TODO: RED-2853 Remove this helper when AMC is removed + This helper is being replaced by `update_course_creator_role_for_cms` which has unit tests. + """ + from cms.djangoapps.course_creators.models import CourseCreator # Fix LMS->CMS imports. + from student.roles import CourseAccessRole # Avoid circular import. + CourseCreator.objects.filter(user__in=users).delete() + CourseAccessRole.objects.filter(user__in=users).delete() + + +@beeline.traced(name="delete_site") +def delete_site(site): + """ + Delete site with all related objects except for MongoDB course files. + """ + from third_party_auth.models import SAMLConfiguration # local import to avoid import-time errors + + print('Deleting SiteConfiguration of', site) + site.configuration.delete() + + print('Deleting theme of', site) + site.themes.all().delete() + + organization = tahoe_sites.api.get_organization_by_site(site) + + print('Deleting users of', site) + users = tahoe_sites.api.get_users_of_organization(organization, without_inactive_users=False) + remove_course_creator_role(users) + + # Prepare removing users by avoiding on_delete=models.PROTECT error + # SAMLConfiguration will be deleted with `site.delete()` + SAMLConfiguration.objects.filter(changed_by__in=users).update(changed_by=None) + + users.delete() + + print('Deleting courses of', site) + delete_organization_courses(organization) + + print('Deleting organization', organization) + organization.delete() + + print('Deleting site', site) + site.delete() + + +def get_models_using_course_key(): + """ + Get all course related model classes. + """ + course_key_field_names = { + 'course_key', + 'course_id', + } + + models_with_course_key = { + (CourseOverview, 'id'), # The CourseKeyField with a `id` name. Hard-coding it for simplicity. + (OrganizationCourse, 'course_id'), # course_id is CharField + } + + model_classes = apps.get_models() + for model_class in model_classes: + for field_name in course_key_field_names: + field_object = getattr(model_class, field_name, None) + if field_object: + field_definition = getattr(field_object, 'field', None) + if field_definition and isinstance(field_definition, (CourseKeyField, LearningContextKeyField)): + models_with_course_key.add( + (model_class, field_name,) + ) + + return models_with_course_key + + +def delete_organization_courses(organization): + """ + Delete all course related model instances. + """ + course_keys = [] + + for course in get_organization_courses({'id': organization.id}): + course_keys.append(course['course_id']) + + delete_related_models_of_courses(course_keys) + + +def delete_related_models_of_courses(course_keys): + model_classes = get_models_using_course_key() + + print('Deleting course related models:', ', '.join([ + '{model}.{field}'.format(model=model_class.__name__, field=field_name) + for model_class, field_name in model_classes + ])) + + for model_class, field_name in model_classes: + objects_to_delete = model_class.objects.filter(**{ + '{field_name}__in'.format(field_name=field_name): course_keys, + }) + objects_to_delete.delete() + + +def get_courses_keys_without_organization_linked(limit=None, only_active_links=True): + """ + Get keys of stray courses. + """ + course_links = OrganizationCourse.objects.all() + if only_active_links: + course_links = course_links.filter(active=True) + + queryset = CourseOverview.objects.exclude( + id__in=course_links.values_list('course_id', flat=True), + ) + + course_keys = queryset.values_list( + 'id', flat=True + ) + + course_keys_list = [str(course_key) for course_key in course_keys] + if limit: + course_keys_list = course_keys_list[:limit] + + return course_keys_list + + +def remove_stray_courses_from_mysql(limit, commit=None, print_func=print): + """ + Removes courses without linked organization from LMS MySQL database. + + The MongoDB courses won't be removed with this command. + """ + course_keys = get_courses_keys_without_organization_linked(limit=limit) + if not course_keys: + raise CommandError('No courses to delete.') + + print_func('Preparing to delete:') + print_func('\n'.join(course_keys)) + + commit = confirm_deletion(commit=commit, question='Do you confirm to delete those courses from the LMS?') + + with transaction.atomic(): + delete_related_models_of_courses(course_keys) + print_func('Finished [commit={}] courses.'.format(commit)) + + if not commit: + transaction.set_rollback(True) diff --git a/openedx/core/djangoapps/appsembler/sites/management/commands/lms_remove_stray_courses.py b/openedx/core/djangoapps/appsembler/sites/management/commands/lms_remove_stray_courses.py new file mode 100644 index 000000000000..5ac0b1f721ad --- /dev/null +++ b/openedx/core/djangoapps/appsembler/sites/management/commands/lms_remove_stray_courses.py @@ -0,0 +1,48 @@ +""" +Remove stray courses from LMS. +""" + +from django.core.management.base import BaseCommand, CommandError +from django.conf import settings + +from ...deletion_utils import remove_stray_courses_from_mysql + + +class Command(BaseCommand): + """ + Bulk removal of courses without an organization linked (aka stray courses). + + This only works for MySQL database. + """ + + def add_arguments(self, parser): + parser.add_argument( + '--limit', + help='Max courses to delete, use 0 to delete all courses.', + default=1, + type=int, + ) + + parser.add_argument( + '--commit', + help='Otherwise, the transaction would be rolled back.', + action='store_true', + dest='commit', + ) + + parser.add_argument( + '--dry-run', + help='Dry run the deletion process without removing the courses.', + action='store_false', + dest='commit', + ) + + def handle(self, *args, **options): + if settings.ROOT_URLCONF != 'lms.urls': + raise CommandError('This command can only be run in LMS.') + + remove_stray_courses_from_mysql( + limit=options['limit'], + commit=options.get('commit'), + print_func=self.stdout.write, + ) diff --git a/openedx/core/djangoapps/appsembler/sites/management/commands/remove_site.py b/openedx/core/djangoapps/appsembler/sites/management/commands/remove_site.py index 10dda1cef134..376f76e9774a 100644 --- a/openedx/core/djangoapps/appsembler/sites/management/commands/remove_site.py +++ b/openedx/core/djangoapps/appsembler/sites/management/commands/remove_site.py @@ -4,7 +4,7 @@ from django.contrib.sites.models import Site from django.db import transaction -from openedx.core.djangoapps.appsembler.sites.utils import delete_site +from ...deletion_utils import delete_site class Command(BaseCommand): diff --git a/openedx/core/djangoapps/appsembler/sites/tests/test_commands.py b/openedx/core/djangoapps/appsembler/sites/tests/test_commands.py index 636d31b8bc7a..fd813c97f7e9 100644 --- a/openedx/core/djangoapps/appsembler/sites/tests/test_commands.py +++ b/openedx/core/djangoapps/appsembler/sites/tests/test_commands.py @@ -164,7 +164,7 @@ def test_run(self): 'ENABLE_CREATOR_GROUP': True, }) @patch( # Avoid CMS-related import issues in tests - 'openedx.core.djangoapps.appsembler.sites.utils.remove_course_creator_role', + 'openedx.core.djangoapps.appsembler.sites.deletion_utils.remove_course_creator_role', Mock() ) class RemoveSiteCommandTestCase(TestCase): diff --git a/openedx/core/djangoapps/appsembler/sites/tests/test_site_delete_utils.py b/openedx/core/djangoapps/appsembler/sites/tests/test_site_delete_utils.py index cacc733a9eea..0ebd5078f5f1 100644 --- a/openedx/core/djangoapps/appsembler/sites/tests/test_site_delete_utils.py +++ b/openedx/core/djangoapps/appsembler/sites/tests/test_site_delete_utils.py @@ -4,7 +4,7 @@ import tahoe_sites.api from django.contrib.auth import get_user_model from django.contrib.sites.models import Site -from django.core.management import call_command +from django.core.management import call_command, CommandError from oauth2_provider.models import Application from organizations.models import OrganizationCourse from status.models import CourseMessage @@ -17,21 +17,21 @@ ) from openedx.core.djangoapps.content.course_overviews.models import CourseOverview -User = get_user_model() - - -from openedx.core.djangoapps.appsembler.sites.utils import ( +from openedx.core.djangoapps.appsembler.sites.deletion_utils import ( + delete_organization_courses, delete_site, get_models_using_course_key, - delete_organization_courses, + remove_stray_courses_from_mysql, ) +User = get_user_model() + def delete_site_with_patched_cms_imports(red_site): """ Delete a site without running the CMS-related code. """ - with patch('openedx.core.djangoapps.appsembler.sites.utils.remove_course_creator_role'): + with patch('openedx.core.djangoapps.appsembler.sites.deletion_utils.remove_course_creator_role'): delete_site(red_site) @@ -137,3 +137,23 @@ def test_delete_course_related_models(make_site): # Should delete the course-related models with pytest.raises(model_class.DoesNotExist): model_class.objects.get() + + +@pytest.mark.django_db +def test_mysql_remove_stray_courses(capsys): + """ + Tests for the remove_stray_courses_from_mysql with and without courses. + """ + with pytest.raises(CommandError, match='No courses to delete.'): + remove_stray_courses_from_mysql(limit=0, commit=False) + + course_key = CourseOverviewFactory.create().id + assert course_key in CourseOverview.get_all_course_keys(), 'Stray course has been created' + + remove_stray_courses_from_mysql(limit=0, commit=False) + assert course_key in CourseOverview.get_all_course_keys(), 'Commit=False do not delete the course' + assert str(course_key) in capsys.readouterr()[0] + + remove_stray_courses_from_mysql(limit=0, commit=True) + assert course_key not in CourseOverview.get_all_course_keys(), 'Stray course is removed' + assert str(course_key) in capsys.readouterr()[0] diff --git a/openedx/core/djangoapps/appsembler/sites/utils.py b/openedx/core/djangoapps/appsembler/sites/utils.py index 11c184c56b77..308c9e30f63b 100644 --- a/openedx/core/djangoapps/appsembler/sites/utils.py +++ b/openedx/core/djangoapps/appsembler/sites/utils.py @@ -3,13 +3,10 @@ A lot of this module should be migrated into more specific modules such as `tahoe-sites`. """ -import tahoe_sites.api -from django.apps import apps from datetime import timedelta import beeline -from opaque_keys.edx.django.models import CourseKeyField, LearningContextKeyField from urllib.parse import urlparse @@ -31,7 +28,6 @@ from organizations import api as org_api from organizations import models as org_models -from organizations.models import OrganizationCourse from organizations.models import Organization @@ -45,7 +41,6 @@ update_admin_role_in_organization, ) -from common.djangoapps.util.organizations_helpers import get_organization_courses from openedx.core.lib.api.api_key_permissions import is_request_has_valid_api_key from openedx.core.lib.log_utils import audit_log from openedx.core.djangoapps.theming.helpers import get_current_request, get_current_site @@ -53,7 +48,6 @@ from ..tahoe_tiers.legacy_amc_helpers import get_active_tiers_uuids_from_amc_postgres from .site_config_client_helpers import get_active_site_uuids_from_site_config_service -from openedx.core.djangoapps.content.course_overviews.models import CourseOverview @beeline.traced(name="get_lms_link_from_course_key") @@ -481,104 +475,6 @@ def bootstrap_site(site, org_data=None, username=None): return organization, site, user -def get_models_using_course_key(): - """ - Get all course related model classes. - """ - course_key_field_names = { - 'course_key', - 'course_id', - } - - models_with_course_key = { - (CourseOverview, 'id'), # The CourseKeyField with a `id` name. Hard-coding it for simplicity. - (OrganizationCourse, 'course_id'), # course_id is CharField - } - - model_classes = apps.get_models() - for model_class in model_classes: - for field_name in course_key_field_names: - field_object = getattr(model_class, field_name, None) - if field_object: - field_definition = getattr(field_object, 'field', None) - if field_definition and isinstance(field_definition, (CourseKeyField, LearningContextKeyField)): - models_with_course_key.add( - (model_class, field_name,) - ) - - return models_with_course_key - - -def delete_organization_courses(organization): - """ - Delete all course related model instances. - """ - course_keys = [] - - for course in get_organization_courses({'id': organization.id}): - course_keys.append(course['course_id']) - - model_classes = get_models_using_course_key() - - print('Deleting course related models:', ', '.join([ - '{model}.{field}'.format(model=model_class.__name__, field=field_name) - for model_class, field_name in model_classes - ])) - - for model_class, field_name in model_classes: - objects_to_delete = model_class.objects.filter(**{ - '{field_name}__in'.format(field_name=field_name): course_keys, - }) - objects_to_delete.delete() - - -def remove_course_creator_role(users): - """ - Remove course creator role to fix `delete_site` issue. - - This will fail in when running tests from within the LMS because the CMS migrations - don't run during tests. Patch this function to avoid such errors. - TODO: RED-2853 Remove this helper when AMC is removed - This helper is being replaced by `update_course_creator_role_for_cms` which has unit tests. - """ - from cms.djangoapps.course_creators.models import CourseCreator # Fix LMS->CMS imports. - from student.roles import CourseAccessRole # Avoid circular import. - CourseCreator.objects.filter(user__in=users).delete() - CourseAccessRole.objects.filter(user__in=users).delete() - - -@beeline.traced(name="delete_site") -def delete_site(site): - from third_party_auth.models import SAMLConfiguration # local import to avoid import-time errors - - print('Deleting SiteConfiguration of', site) - site.configuration.delete() - - print('Deleting theme of', site) - site.themes.all().delete() - - organization = tahoe_sites.api.get_organization_by_site(site) - - print('Deleting users of', site) - users = tahoe_sites.api.get_users_of_organization(organization, without_inactive_users=False) - remove_course_creator_role(users) - - # Prepare removing users by avoiding on_delete=models.PROTECT error - # SAMLConfiguration will be deleted with `site.delete()` - SAMLConfiguration.objects.filter(changed_by__in=users).update(changed_by=None) - - users.delete() - - print('Deleting courses of', site) - delete_organization_courses(organization) - - print('Deleting organization', organization) - organization.delete() - - print('Deleting site', site) - site.delete() - - @beeline.traced(name="add_course_creator_role") def add_course_creator_role(user): """ From ab71e525ddac54b13099705c7a9c084619a70326 Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Tue, 6 Dec 2022 15:13:55 +0300 Subject: [PATCH 022/125] Skip AMC tests when TAHOE_SITES_USE_ORGS_MODELS=False This test should be removed when AMC is removed RED-2845 --- .../tests/test_get_user_by_username_or_email.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_get_user_by_username_or_email.py b/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_get_user_by_username_or_email.py index c3487c746790..b073dc12fc50 100644 --- a/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_get_user_by_username_or_email.py +++ b/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_get_user_by_username_or_email.py @@ -20,7 +20,10 @@ def test_get_user_by_username_or_email_single_tenant(settings): """ Ensure `get_user_by_username_or_email` works as upstream intended if APPSEMBLER_MULTI_TENANT_EMAILS is disabled. """ - settings.FEATURES = {'APPSEMBLER_MULTI_TENANT_EMAILS': False} + settings.FEATURES = { + **settings.FEATURES, + 'APPSEMBLER_MULTI_TENANT_EMAILS': False, + } with with_organization_context(site_color='blue1') as blue_org: blue_user = create_org_user(blue_org) @@ -44,7 +47,10 @@ def test_get_user_by_username_or_email_multi_tenant(settings): """ Ensure `get_user_by_username_or_email` works with APPSEMBLER_MULTI_TENANT_EMAILS is enabled. """ - settings.FEATURES = {'APPSEMBLER_MULTI_TENANT_EMAILS': True} + settings.FEATURES = { + **settings.FEATURES, + 'APPSEMBLER_MULTI_TENANT_EMAILS': True, + } with with_organization_context(site_color='blue1') as blue_org: blue_user = create_org_user(blue_org) From 611e9d9fa283f1eadb42b90159b39b26950a143c Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Tue, 6 Dec 2022 15:14:14 +0300 Subject: [PATCH 023/125] fix tests when using tahoe-sites models those tests failed when setting `TAHOE_SITES_USE_ORGS_MODELS=False` --- .../appsembler/multi_tenant_emails/tests/test_amc_signup.py | 4 ++++ .../multi_tenant_emails/tests/test_enroll_by_email.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_amc_signup.py b/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_amc_signup.py index c9bc31fe9710..d30f43b6c5df 100644 --- a/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_amc_signup.py +++ b/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_amc_signup.py @@ -1,7 +1,9 @@ +import unittest import json from mock import patch, Mock import uuid +from django.conf import settings from django.contrib.auth.models import User from django.urls import reverse from rest_framework import status @@ -98,6 +100,7 @@ def register_new_amc_admin(self, color, email): assert site_response.status_code == status.HTTP_201_CREATED, '{}: {}'.format(color, site_response.content) return user_response, site_response + @unittest.skipUnless(settings.FEATURES.get('TAHOE_SITES_USE_ORGS_MODELS', False), 'RED-2845 Remove with AMC') def test_new_admin_with_learner(self, mock_add_creator): """ Test happy scenario regardless of APPSEMBLER_MULTI_TENANT_EMAILS. @@ -110,6 +113,7 @@ def test_new_admin_with_learner(self, mock_add_creator): with with_organization_context(site_color=red_site): self.register_learner('learner@example.com', 'learner') + @unittest.skipUnless(settings.FEATURES.get('TAHOE_SITES_USE_ORGS_MODELS', False), 'RED-2845 Remove with AMC') def test_learner_registers_for_trial(self, mock_add_creator): """ Test learner registers for a new Tahoe trial signup when APPSEMBLER_MULTI_TENANT_EMAILS is enabled. diff --git a/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_enroll_by_email.py b/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_enroll_by_email.py index d16511648486..5116b34fe985 100644 --- a/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_enroll_by_email.py +++ b/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_enroll_by_email.py @@ -21,7 +21,7 @@ def test_enroll_by_email_single_tenant(settings): """ Ensure `enroll_by_email` works as upstream intended if APPSEMBLER_MULTI_TENANT_EMAILS is disabled. """ - settings.FEATURES = {'APPSEMBLER_MULTI_TENANT_EMAILS': False} + settings.FEATURES = {**settings.FEATURES, 'APPSEMBLER_MULTI_TENANT_EMAILS': False} course = CourseOverviewFactory.create() course_key = course.id @@ -43,7 +43,7 @@ def test_enroll_by_email_multi_tenant(settings): """ Ensure `enroll_by_email` works with APPSEMBLER_MULTI_TENANT_EMAILS is enabled. """ - settings.FEATURES = {'APPSEMBLER_MULTI_TENANT_EMAILS': True} + settings.FEATURES = {**settings.FEATURES, 'APPSEMBLER_MULTI_TENANT_EMAILS': True} course = CourseOverviewFactory.create() course_key = course.id From bd154706988d9239d19529cb29140d685a1cecda Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Tue, 6 Dec 2022 15:57:46 +0300 Subject: [PATCH 024/125] docker: bump cag to 0.6.0 to use tahoe-sites --- Dockerfile.tutor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.tutor b/Dockerfile.tutor index deb29f5c1b9f..26a54cd5f3af 100644 --- a/Dockerfile.tutor +++ b/Dockerfile.tutor @@ -27,7 +27,7 @@ RUN echo "Installing pip packages:" \ && pip install https://github.com/pmitros/FeedbackXBlock/archive/v1.1.tar.gz \ && pip install https://github.com/ubc/ubcpi/archive/1.0.0.tar.gz \ && echo \ - && pip install course-access-groups==0.5.4 \ + && pip install course-access-groups==0.6.0 \ && pip install figures==0.4.1 \ && pip install tahoe-figures-plugins==0.1.1 \ && pip install tahoe-lti==0.3.0 \ From 77da710abed80e4219ad63fd4f1a7a4faf90dffe Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Tue, 6 Dec 2022 19:09:24 +0300 Subject: [PATCH 025/125] upgrade create_devstack_site to tahoe 2.0; removes upgrade command `sites/management/commands/danger_candidate_sites_cleanup.py` is now broken, we'll probably need to re-create a similar command in future upgrades --- .../commands/create_devstack_site.py | 79 ++++++------------- .../danger_candidate_sites_cleanup.py | 57 ------------- .../appsembler/sites/tests/test_commands.py | 58 ++------------ 3 files changed, 31 insertions(+), 163 deletions(-) delete mode 100644 openedx/core/djangoapps/appsembler/sites/management/commands/danger_candidate_sites_cleanup.py diff --git a/openedx/core/djangoapps/appsembler/sites/management/commands/create_devstack_site.py b/openedx/core/djangoapps/appsembler/sites/management/commands/create_devstack_site.py index 75c338454da9..6e20cc7d7cf7 100644 --- a/openedx/core/djangoapps/appsembler/sites/management/commands/create_devstack_site.py +++ b/openedx/core/djangoapps/appsembler/sites/management/commands/create_devstack_site.py @@ -1,7 +1,8 @@ -import hashlib import inspect import json -import uuid + +from openedx.core.djangoapps.appsembler.sites.serializers_v2 import TahoeSiteCreationSerializer +from tahoe_sites.api import add_user_to_organization from django.core.management.base import BaseCommand, CommandError from django.contrib.auth.models import User @@ -10,16 +11,11 @@ from django.db import transaction from openedx.core.djangoapps.appsembler.sites.serializers import RegistrationSerializer -from openedx.core.djangoapps.appsembler.sites.utils import reset_amc_tokens -from student.models import UserProfile -from student.roles import CourseCreatorRole class Command(BaseCommand): """ - Create the demo something.localhost:18000 site for devstack. - - Needs the corresponding `create_devstack_site` AMC command to be run as well. + Create a Tahoe 2.0 demo something.localhost:18000 site for devstack. """ def add_arguments(self, parser): @@ -46,17 +42,14 @@ def congrats(self, **kwargs): """ Congrats, Your site is ready! - Username: "{name}" - Email: "{email}" - Password: "{password}" - Site URL: "http://{site}/" Please add the following entry to your /etc/hosts file: 127.0.0.1 {domain} - Remember to run the corresponding AMC command. + You can login via FusionAuth via a Learner or an Administrator depending + on the user.data.platform_role you chose. Enjoy! """.format(**kwargs) @@ -83,52 +76,32 @@ def _handle_with_atmoic(self, *args, **options): domain = '{name}.{base_domain}'.format(name=name, base_domain=base_domain) site_name = '{domain}:18000'.format(domain=domain) - user = User.objects.create_user( - username=name, - email='{}@example.com'.format(name), - password=name, - ) - CourseCreatorRole().add_users(user) - UserProfile.objects.create(user=user, name=name) - - # Calculated access tokens to the AMC devstack can have them without needing to communicate with the LMS. - # Just making it easier to automate this without having cross-dependency in devstack - fake_token = hashlib.md5(user.username.encode('utf-8')).hexdigest() - reset_amc_tokens(user, access_token=fake_token, refresh_token=fake_token) - - data = { - 'site': { - 'domain': site_name, - 'name': site_name, - }, - 'username': user.username, - 'organization': { - 'name': name, - 'short_name': name, - 'edx_uuid': uuid.uuid4(), # TODO: RED-2845 Remove this line when AMC is migrated - }, - 'initial_values': { - 'SITE_NAME': site_name, - 'platform_name': '{} Academy'.format(name), - 'logo_positive': None, - 'logo_negative': None, - 'font': 'Roboto', - 'accent-font': 'Delius Unicase', - 'primary_brand_color': '#F00', - 'base_text_color': '#000', - 'cta_button_bg': '#00F', - } - } - serializer = RegistrationSerializer(data=data) + serializer = TahoeSiteCreationSerializer(data={ + 'short_name': name, + 'domain': site_name, + }) if not serializer.is_valid(): raise CommandError('Something went wrong with the process: \n{errors}'.format( errors=json.dumps(serializer.errors, indent=4) )) - serializer.save() + site_data = serializer.save() + + # This admin cannot login without FusionAuth, but it's added for simulation purposes + # in testing. + fake_admin_user = User.objects.create_user( + username=name, + email='{}@example.com'.format(name), + password=name, + ) + add_user_to_organization( + user=fake_admin_user, + organization=site_data['organization'], + is_admin=True, + ) self.congrats( - name=user.username, - email=user.email, + name=fake_admin_user.username, + email=fake_admin_user.email, password=name, site=site_name, domain=domain, diff --git a/openedx/core/djangoapps/appsembler/sites/management/commands/danger_candidate_sites_cleanup.py b/openedx/core/djangoapps/appsembler/sites/management/commands/danger_candidate_sites_cleanup.py deleted file mode 100644 index f39c5a2b0f87..000000000000 --- a/openedx/core/djangoapps/appsembler/sites/management/commands/danger_candidate_sites_cleanup.py +++ /dev/null @@ -1,57 +0,0 @@ -from django.core.management import BaseCommand, CommandError - -from openedx.core.djangoapps.appsembler.sites.utils import get_active_sites - - -class Command(BaseCommand): - help = "DANGEROUS: Renames all domains for production/staging candidate. Remove Google Tag Manager and Segment keys. Do not run during on production!" - - def add_arguments(self, parser): - parser.add_argument( - 'from', - help='The production/staging domain e.g. "tahoe.appsembler.com"', - type=str, - ) - - parser.add_argument( - 'to', - help='The candidate domain e.g. "tahoe-us-juniper-prod.appsembler.com"', - type=str, - ) - - def handle(self, *args, **options): - sites_with_configs = get_active_sites().filter(configuration__isnull=False) - - has_errors = False - for site in sites_with_configs: - self.stdout.write('FROM {}'.format(site.domain)) - site.domain = site.domain.replace('.{}'.format(options['from']), '.{}'.format(options['to'])) - self.stdout.write('TO {}'.format(site.domain)) - site.save() - - site.configuration.site_values['SITE_NAME'] = site.domain - - try: - del site.configuration.site_values['SEGMENT_KEY'] - self.stdout.write('deleted SEGMENT_KEY') - except KeyError: - self.stdout.write('no SEGMENT_KEY') - pass - - try: - del site.configuration.site_values['customer_gtm_id'] - self.stdout.write('deleted customer_gtm_id') - except KeyError: - self.stdout.write('no customer_gtm_id') - pass - try: - site.configuration.save() - except Exception as e: - has_errors = True - self.stdout.write(e) - self.stdout.write('---') - - if has_errors: - msg = 'Some sites have failed, please review this command output for more information.' - self.stdout.write(msg) - raise CommandError(msg) diff --git a/openedx/core/djangoapps/appsembler/sites/tests/test_commands.py b/openedx/core/djangoapps/appsembler/sites/tests/test_commands.py index fd813c97f7e9..0ffd0348eca0 100644 --- a/openedx/core/djangoapps/appsembler/sites/tests/test_commands.py +++ b/openedx/core/djangoapps/appsembler/sites/tests/test_commands.py @@ -1,4 +1,3 @@ -import hashlib import os from unittest.mock import patch, mock_open, Mock from io import StringIO @@ -57,11 +56,8 @@ UserStandingFactory, ) -from organizations.models import Organization, OrganizationCourse - -from oauth2_provider.models import AccessToken, RefreshToken, Application - -from student.roles import CourseCreatorRole +from organizations.models import OrganizationCourse, Organization +from oauth2_provider.models import Application @override_settings( @@ -109,51 +105,8 @@ def test_create_devstack_site(self): assert Site.objects.get(domain=self.site_name) organization = Organization.objects.get(name=self.name) user = get_user_model().objects.get() - assert user.check_password(self.name) - assert user.profile.name == self.name assert get_organization_for_user(user=user) == organization - assert CourseCreatorRole().has_user(user), 'User should be a course creator' - - fake_token = hashlib.md5(user.username.encode('utf-8')).hexdigest() # Using a fake token so AMC devstack can guess it - assert fake_token == '80bfa968ffad007c79bfc603f3670c99', 'Ensure hash is identical to AMC' - assert AccessToken.objects.get(user=user).token == fake_token, 'Access token is needed' - assert RefreshToken.objects.get(user=user).token == fake_token, 'Refresh token is needed' - - -@override_settings( - DEBUG=True, -) -class TestCandidateSitesCleanupCommand(TestCase): - """ - Tests for the `danger_candidate_sites_cleanup` management command. - """ - def setUp(self): - Application.objects.create(client_id=settings.AMC_APP_OAUTH2_CLIENT_ID, - client_type=Application.CLIENT_CONFIDENTIAL) - call_command('create_devstack_site', 'blue', 'oldlocalhost') - site_config = self.get_site().configuration - site_config.site_values.update({ - 'SEGMENT_KEY': 'test1', - 'customer_gtm_id': 'test2', - }) - site_config.save() - - def get_site(self): - return Site.objects.get(domain__startswith='blue.') - - def test_run(self): - assert self.get_site().domain == 'blue.oldlocalhost:18000' - active_orgs = Organization.objects.all() - active_orgs_function_path = 'openedx.core.djangoapps.appsembler.sites.utils.get_active_organizations' - with patch(active_orgs_function_path, return_value=active_orgs): - # Side-step the `Tier` model. - call_command('danger_candidate_sites_cleanup', 'oldlocalhost:18000', 'newlocalhost:18000') - assert self.get_site().domain == 'blue.newlocalhost:18000' - assert not self.get_site().configuration.get_value('customer_gtm_id') - assert not self.get_site().configuration.get_value('SEGMENT_KEY') - assert self.get_site().configuration.get_value('SITE_NAME') == self.get_site().domain - @override_settings( DEBUG=True, @@ -189,7 +142,7 @@ def test_remove_devstack_site_commit(self): """ deleted_domain = '{}.localhost:18000'.format(self.to_be_deleted) remained_domain = '{}.localhost:18000'.format(self.shall_remain) - assert SiteConfiguration.objects.count() == 2, 'there are two sites' + assert Site.objects.filter(domain__endswith='.localhost:18000').count() == 2, 'there are two sites' remained_site = Site.objects.get(domain=remained_domain) # TODO: Re-produce the error we face in staging @@ -202,9 +155,8 @@ def test_remove_devstack_site_commit(self): # Ensure objects are removed correctly. assert not Site.objects.filter(domain=deleted_domain).exists() - assert SiteConfiguration.objects.count() == 1, 'One site is deleted' - assert SiteConfiguration.objects.get(site=remained_site), 'remained_domain site config is kept' - + assert Site.objects.filter(domain__endswith='.localhost:18000').count() == 1, 'One site is deleted' + remained_site.refresh_from_db() # remained_domain site config is kept assert SiteTheme.objects.filter(site=remained_site).count() == remained_site.themes.count() def test_remove_devstack_site_rollback(self): From 92bc19445daa6a4c62a0d7a49f558f444e345571 Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Thu, 8 Dec 2022 13:57:55 +0300 Subject: [PATCH 026/125] legacy_amc_helpers.py no longer needs edx-organization fork Use raw SQL to avoid requireing Organization.edx_uuid edx_uuid only exists in Appsembler's fork of edx-organizations this makes it possible to use the fork while connecting directly to AMC database. --- .github/workflows/tests.yml | 1 + .../tests/test_amc_signup.py | 11 ++- .../test_get_user_by_username_or_email.py | 10 +-- .../settings/settings/test_common.py | 9 +++ .../sites/site_config_client_helpers.py | 2 +- .../tahoe_tiers/legacy_amc_helpers.py | 75 +++++++++++++------ .../tests/test_legacy_amc_helpers.py | 58 ++++++++++++++ .../tahoe_tiers/tests/test_tier_info.py | 40 ++++++++++ .../appsembler/tahoe_tiers/tier_info.py | 63 ++++++++++++++++ tox.ini | 14 +++- 10 files changed, 249 insertions(+), 34 deletions(-) create mode 100644 openedx/core/djangoapps/appsembler/tahoe_tiers/tests/test_legacy_amc_helpers.py create mode 100644 openedx/core/djangoapps/appsembler/tahoe_tiers/tests/test_tier_info.py create mode 100644 openedx/core/djangoapps/appsembler/tahoe_tiers/tier_info.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a110bf2f8fd2..4b36395476c7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,6 +24,7 @@ jobs: - lms-1 - lms-2 - mte + - legacy-amc-tests - studio steps: diff --git a/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_amc_signup.py b/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_amc_signup.py index d30f43b6c5df..2527f9587aca 100644 --- a/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_amc_signup.py +++ b/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_amc_signup.py @@ -1,15 +1,20 @@ +""" +Tests for the LMS part of the deprecated AMC trial signup. +""" +# TODO: RED-2845 Remove after migrating to Tahoe 2.0 + import unittest import json from mock import patch, Mock import uuid -from django.conf import settings from django.contrib.auth.models import User from django.urls import reverse from rest_framework import status from rest_framework.test import APITestCase from tahoe_sites.api import get_organization_for_user, is_active_admin_on_organization +from tahoe_sites.zd_helpers import should_site_use_org_models from .test_utils import lms_multi_tenant_test, with_organization_context @@ -100,7 +105,7 @@ def register_new_amc_admin(self, color, email): assert site_response.status_code == status.HTTP_201_CREATED, '{}: {}'.format(color, site_response.content) return user_response, site_response - @unittest.skipUnless(settings.FEATURES.get('TAHOE_SITES_USE_ORGS_MODELS', False), 'RED-2845 Remove with AMC') + @unittest.skipUnless(should_site_use_org_models(), 'RED-2845 Remove with AMC') def test_new_admin_with_learner(self, mock_add_creator): """ Test happy scenario regardless of APPSEMBLER_MULTI_TENANT_EMAILS. @@ -113,7 +118,7 @@ def test_new_admin_with_learner(self, mock_add_creator): with with_organization_context(site_color=red_site): self.register_learner('learner@example.com', 'learner') - @unittest.skipUnless(settings.FEATURES.get('TAHOE_SITES_USE_ORGS_MODELS', False), 'RED-2845 Remove with AMC') + @unittest.skipUnless(should_site_use_org_models(), 'RED-2845 Remove with AMC') def test_learner_registers_for_trial(self, mock_add_creator): """ Test learner registers for a new Tahoe trial signup when APPSEMBLER_MULTI_TENANT_EMAILS is enabled. diff --git a/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_get_user_by_username_or_email.py b/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_get_user_by_username_or_email.py index b073dc12fc50..12fe7f5ff417 100644 --- a/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_get_user_by_username_or_email.py +++ b/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_get_user_by_username_or_email.py @@ -20,10 +20,7 @@ def test_get_user_by_username_or_email_single_tenant(settings): """ Ensure `get_user_by_username_or_email` works as upstream intended if APPSEMBLER_MULTI_TENANT_EMAILS is disabled. """ - settings.FEATURES = { - **settings.FEATURES, - 'APPSEMBLER_MULTI_TENANT_EMAILS': False, - } + settings.FEATURES = {**settings.FEATURES, 'APPSEMBLER_MULTI_TENANT_EMAILS': False} with with_organization_context(site_color='blue1') as blue_org: blue_user = create_org_user(blue_org) @@ -47,10 +44,7 @@ def test_get_user_by_username_or_email_multi_tenant(settings): """ Ensure `get_user_by_username_or_email` works with APPSEMBLER_MULTI_TENANT_EMAILS is enabled. """ - settings.FEATURES = { - **settings.FEATURES, - 'APPSEMBLER_MULTI_TENANT_EMAILS': True, - } + settings.FEATURES = {**settings.FEATURES, 'APPSEMBLER_MULTI_TENANT_EMAILS': True} with with_organization_context(site_color='blue1') as blue_org: blue_user = create_org_user(blue_org) diff --git a/openedx/core/djangoapps/appsembler/settings/settings/test_common.py b/openedx/core/djangoapps/appsembler/settings/settings/test_common.py index 1be1ce7b5fb6..f24648c3997b 100644 --- a/openedx/core/djangoapps/appsembler/settings/settings/test_common.py +++ b/openedx/core/djangoapps/appsembler/settings/settings/test_common.py @@ -26,6 +26,15 @@ def plugin_settings(settings): settings.TAHOE_ALWAYS_SKIP_TEST = True settings.CMS_UPDATE_SEARCH_INDEX_JOB_QUEUE = 'edx.cms.core.default' + # TODO: Remove when AMC is removed: RED-2845 + settings.FEATURES['TAHOE_SITES_USE_ORGS_MODELS'] = getenv('TEST_TAHOE_SITES_USE_ORGS_MODELS', 'true') == 'true' + + # TODO: Remove when AMC is removed: RED-2845 + settings.TIERS_ORGANIZATION_MODEL = 'organizations.Organization' + settings.INSTALLED_APPS += [ + 'tiers', + ] + if settings.FEATURES.get('APPSEMBLER_MULTI_TENANT_EMAILS', False): settings.INSTALLED_APPS += [ 'openedx.core.djangoapps.appsembler.multi_tenant_emails', diff --git a/openedx/core/djangoapps/appsembler/sites/site_config_client_helpers.py b/openedx/core/djangoapps/appsembler/sites/site_config_client_helpers.py index 351d99892ec0..3a3df3891039 100644 --- a/openedx/core/djangoapps/appsembler/sites/site_config_client_helpers.py +++ b/openedx/core/djangoapps/appsembler/sites/site_config_client_helpers.py @@ -16,7 +16,7 @@ from site_config_client.openedx.adapter import SiteConfigAdapter from site_config_client.exceptions import SiteConfigurationError -from tiers.tier_info import TierInfo +from ..tahoe_tiers.tier_info import TierInfo from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers diff --git a/openedx/core/djangoapps/appsembler/tahoe_tiers/legacy_amc_helpers.py b/openedx/core/djangoapps/appsembler/tahoe_tiers/legacy_amc_helpers.py index 3a68c1ba68db..3d350f4d032f 100644 --- a/openedx/core/djangoapps/appsembler/tahoe_tiers/legacy_amc_helpers.py +++ b/openedx/core/djangoapps/appsembler/tahoe_tiers/legacy_amc_helpers.py @@ -6,13 +6,13 @@ import logging import beeline - -from tahoe_sites.api import get_uuid_by_organization +from uuid import UUID from django.utils import timezone -from django.db.models import Q, F from tiers.models import Tier +from ..tahoe_tiers.tier_info import TierInfo + log = logging.getLogger(__name__) @@ -24,19 +24,43 @@ def get_amc_tier_info(site_uuid): # pragma: no cover Hack: This queries the django-tier database in a rather hacky way. - WARNING: !! This function is _not_ covered with tests. Please edit with caution and test manually. !! + WARNING: !! This function is _not_ fully covered with tests. Please edit with caution and test on staging. !! """ try: + site_uuid_hex = UUID(str(site_uuid)).hex + # Query the AMC Postgres database directly - tier = Tier.objects.defer('organization').get(organization__edx_uuid=site_uuid) - return tier.get_tier_info() - except Tier.DoesNotExist: - # If the organization has no AMC-tier fail silently and log it in honeycomb. - # This either happens in the case of a Tahoe 2.0 site or a missing tier - # from AMC (although that shouldn't happen). - beeline.add_context_field("tiers.organization_without_tier", True) - return None - except Exception: + tiers = Tier.objects.raw( + """SELECT + t.id as id, + t.name AS name, + t.tier_expires_at AS tier_expires_at, + org.edx_uuid as edx_uuid, + t.tier_enforcement_exempt AS tier_enforcement_exempt + FROM tiers_tier as t + INNER JOIN organizations_organization as org on t.organization_id = org.id + WHERE org.edx_uuid = %s + LIMIT 1 + """, + [str(site_uuid_hex)] + ) + tiers_list = list(tiers) + + if not tiers_list: + # If the organization has no AMC-tier fail silently and log it in honeycomb. + # This either happens in the case of a Tahoe 2.0 site or a missing tier + # from AMC (although that shouldn't happen). + beeline.add_context_field("tiers.organization_without_tier", True) + return None + + tier = tiers[0] + return TierInfo( + tier=tier.name, + subscription_ends=tier.tier_expires_at, + always_active=tier.tier_enforcement_exempt, + ) + except Exception: # noqa + log.exception('Error with fetching the tier from AMC') beeline.add_context_field("tiers.exception_with_tier", True) log.exception("Organization has a problem with its Tier: {0}".format(site_uuid)) return None @@ -50,13 +74,22 @@ def get_active_tiers_uuids_from_amc_postgres(): # pragma: no cover Return a list of UUID objects. - WARNING: !! This function is _not_ covered with tests. Please edit with caution and test manually. !! + WARNING: !! This function is _not_ fully covered with tests. Please edit with caution and test on staging. !! """ # This queries the AMC Postgres database - active_tiers_uuids = Tier.objects.filter( - Q(tier_enforcement_exempt=True) | - Q(tier_expires_at__gte=timezone.now()) - ).annotate( - organization_edx_uuid=F('organization__edx_uuid') - ).values_list('organization_edx_uuid', flat=True) - return list(active_tiers_uuids) + tiers = Tier.objects.raw( + """ + SELECT + t.id as id, + org.edx_uuid as site_uuid + FROM tiers_tier as t + INNER JOIN organizations_organization as org on t.organization_id = org.id + WHERE t.tier_expires_at >= %s OR t.tier_enforcement_exempt + """, + [str(timezone.now())] + ) + + return [ + UUID(t.site_uuid) + for t in tiers + ] diff --git a/openedx/core/djangoapps/appsembler/tahoe_tiers/tests/test_legacy_amc_helpers.py b/openedx/core/djangoapps/appsembler/tahoe_tiers/tests/test_legacy_amc_helpers.py new file mode 100644 index 000000000000..f091482da68e --- /dev/null +++ b/openedx/core/djangoapps/appsembler/tahoe_tiers/tests/test_legacy_amc_helpers.py @@ -0,0 +1,58 @@ +import pytest + +from tahoe_sites.zd_helpers import should_site_use_org_models +from ..legacy_amc_helpers import ( + get_amc_tier_info, + get_active_tiers_uuids_from_amc_postgres, +) + + +@pytest.mark.django_db +def test_get_amc_tier_info_not_found(): + assert not get_amc_tier_info('6229db46-76e7-11ed-bb20-37f3f60d0442'), 'Non-existent tier info' + + +@pytest.mark.django_db +@pytest.mark.skipif( + condition=not should_site_use_org_models(), + reason='Needs AMC database compatible edx-organizations' +) +def test_get_amc_tier_info_found(): + from tiers.models import Tier + from organizations.tests.factories import OrganizationFactory + + organization = OrganizationFactory.create(edx_uuid='2f51e0e1-7cd4-4447-86fc-5de03e2cf3b1') + tier = Tier.objects.create(organization=organization) + assert tier.organization == organization + tier = get_amc_tier_info(organization.edx_uuid) + assert tier, 'Should find tier' + assert tier.tier == 'trial', 'Should be trial' + + +@pytest.mark.django_db +@pytest.mark.skipif( + condition=not should_site_use_org_models(), + reason='Needs AMC database compatible edx-organizations' +) +def test_active_tiers(): + from tiers.models import Tier + from organizations.tests.factories import OrganizationFactory + + active_org = OrganizationFactory.create(edx_uuid='2f51e0e1-7cd4-4447-86fc-5de03e2cf3b1') + Tier.objects.create(organization=active_org) + + inactive_org = OrganizationFactory.create(edx_uuid='9e29c034-76f1-11ed-a879-7702b938796e') + Tier.objects.create(organization=inactive_org, tier_expires_at='2017-01-01') + + exempted_org = OrganizationFactory.create(edx_uuid='496efb30-76f2-11ed-b521-37e754be4889') + Tier.objects.create(organization=exempted_org, tier_enforcement_exempt=True, tier_expires_at='2017-01-01') + + org_without_tier = OrganizationFactory.create(edx_uuid='e54d116e-76f1-11ed-b6c2-87cd9adfb48f') + + active_tier_uuids = get_active_tiers_uuids_from_amc_postgres() + active_tier_uuids = [str(site_uuid) for site_uuid in active_tier_uuids] + + assert active_org.edx_uuid in active_tier_uuids, 'Should only list orgs with active tiers' + assert exempted_org.edx_uuid in active_tier_uuids, 'Exempted orgs are considered active' + assert inactive_org.edx_uuid not in active_tier_uuids, 'Expired org tier' + assert org_without_tier.edx_uuid not in active_tier_uuids, 'Missing org tier should not appear here' diff --git a/openedx/core/djangoapps/appsembler/tahoe_tiers/tests/test_tier_info.py b/openedx/core/djangoapps/appsembler/tahoe_tiers/tests/test_tier_info.py new file mode 100644 index 000000000000..c652e33b462e --- /dev/null +++ b/openedx/core/djangoapps/appsembler/tahoe_tiers/tests/test_tier_info.py @@ -0,0 +1,40 @@ +""" +Tests for the TierInfo helper class. +""" + +from datetime import timedelta +from django.utils.timezone import now +from ..tier_info import TierInfo + + +def tier_info_factory( + tier=TierInfo.TRIAL, + subscription_ends=now() + timedelta(days=30), + always_active=False, +): + return TierInfo( + tier=tier, + subscription_ends=subscription_ends, + always_active=always_active, + ) + + +def test_non_expired_tier(): + t = tier_info_factory() + assert not t.always_active + assert not t.has_subscription_ended() + + +def test_expired_tier(): + t = tier_info_factory(subscription_ends=(now() - timedelta(days=2))) + assert not t.always_active + assert t.has_subscription_ended() + + +def test_exemption(): + t = tier_info_factory( + always_active=True, + subscription_ends=(now() - timedelta(days=20)), + ) + assert t.always_active + assert not t.has_subscription_ended() diff --git a/openedx/core/djangoapps/appsembler/tahoe_tiers/tier_info.py b/openedx/core/djangoapps/appsembler/tahoe_tiers/tier_info.py new file mode 100644 index 000000000000..23aa5f0ec4eb --- /dev/null +++ b/openedx/core/djangoapps/appsembler/tahoe_tiers/tier_info.py @@ -0,0 +1,63 @@ +""" +Tier helper and calculation classes with no model dependency. + +Note: This is cloned from `django-tiers:tiers.tier_info.py` to prepare for removing/refactoring the dependency. +""" +from collections import namedtuple + +from django.utils import timezone +from django.utils.timesince import timeuntil + +TierTuple = namedtuple('TierTuple', ['id', 'name']) + + +class TierInfo: + """ + Tier info and calculator class. + + TODO: Move into the Site Configuration Client package. + """ + + TRIAL = TierTuple('trial', 'Trial') # Expires in 30 days + BASIC = TierTuple('basic', 'Basic') + PRO = TierTuple('pro', 'Professional') + PREMIUM = TierTuple('premium', 'Premium') + + TIERS = ( + TRIAL, + BASIC, + PRO, + PREMIUM, + ) + + def __init__(self, tier, subscription_ends, always_active): + self.tier = tier + self.subscription_ends = subscription_ends + self.always_active = always_active + + def has_subscription_ended(self, now=None): + """Helper function that checks whether a subscription has expired""" + if self.always_active: + return False + + if not now: + now = timezone.now() + + return now > self.subscription_ends + + def should_show_expiration_warning(self): + """Decide if expiration warning is needed.""" + if self.always_active: + return False + + return self.tier == self.TRIAL.id + + def time_til_expiration(self, now=None): + """Pretty prints time left til expiration""" + if self.always_active: + return False + + if not now: + now = timezone.now() + + return timeuntil(self.subscription_ends, now) diff --git a/tox.ini b/tox.ini index 204809d8b650..4f5a2a28ce87 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = studio,lms-1,lms-2,mte,common,pep8 +envlist = studio,lms-1,lms-2,mte,legacy-amc-tests,common,pep8 # This is needed to prevent the lms, cms, and openedx packages inside the "Open # edX" package (defined in setup.py) from getting installed into site-packages @@ -147,6 +147,18 @@ setenv = commands = pytest {env:PYTEST_ARGS} {posargs:openedx/core/djangoapps/appsembler/multi_tenant_emails} +[testenv:legacy-amc-tests] +# Keep this until all AMC-related code is gone. +setenv = + PYTHONHASHSEED=0 + TOXENV={envname} + PYTEST_ARGS={env:PYTEST_ARGS:} + TEST_TAHOE_SITES_USE_ORGS_MODELS=true +commands = + pip install https://github.com/appsembler/edx-organizations/archive/5.2.0-appsembler14.tar.gz + pytest {env:PYTEST_ARGS} {posargs:openedx/core/djangoapps/appsembler/tahoe_tiers/tests/test_legacy_amc_helpers.py} + + [testenv:pytest] commands = {posargs} From ee7e9b7bc149315a40f2704b5d5e26508fed8d9a Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Mon, 12 Dec 2022 17:24:24 +0300 Subject: [PATCH 027/125] add db-migration tox env (split from lms-1) --- .github/workflows/tests.yml | 1 + .../appsembler/settings/settings/test_common.py | 9 ++++----- tox.ini | 9 +++++++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4b36395476c7..989a259d4ecc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,6 +25,7 @@ jobs: - lms-2 - mte - legacy-amc-tests + - db-migrations - studio steps: diff --git a/openedx/core/djangoapps/appsembler/settings/settings/test_common.py b/openedx/core/djangoapps/appsembler/settings/settings/test_common.py index f24648c3997b..b4fa3d749efe 100644 --- a/openedx/core/djangoapps/appsembler/settings/settings/test_common.py +++ b/openedx/core/djangoapps/appsembler/settings/settings/test_common.py @@ -29,11 +29,10 @@ def plugin_settings(settings): # TODO: Remove when AMC is removed: RED-2845 settings.FEATURES['TAHOE_SITES_USE_ORGS_MODELS'] = getenv('TEST_TAHOE_SITES_USE_ORGS_MODELS', 'true') == 'true' - # TODO: Remove when AMC is removed: RED-2845 - settings.TIERS_ORGANIZATION_MODEL = 'organizations.Organization' - settings.INSTALLED_APPS += [ - 'tiers', - ] + if getenv('TEST_ENABLE_TIERS_APP', 'false') == 'true': + settings.INSTALLED_APPS += [ + 'tiers', + ] if settings.FEATURES.get('APPSEMBLER_MULTI_TENANT_EMAILS', False): settings.INSTALLED_APPS += [ diff --git a/tox.ini b/tox.ini index 4f5a2a28ce87..9f3bf5c4a824 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = studio,lms-1,lms-2,mte,legacy-amc-tests,common,pep8 +envlist = studio,lms-1,lms-2,mte,legacy-amc-tests,common,db-migrations,pep8 # This is needed to prevent the lms, cms, and openedx packages inside the "Open # edX" package (defined in setup.py) from getting installed into site-packages @@ -113,7 +113,6 @@ commands = [testenv:lms-1] commands = pytest {env:PYTEST_ARGS} \ - common/djangoapps/util/tests/test_db.py::MigrationTests \ lms/tests.py \ lms/djangoapps/certificates/tests/test_webview_appsembler_changes.py \ lms/djangoapps/course_api/ \ @@ -147,6 +146,11 @@ setenv = commands = pytest {env:PYTEST_ARGS} {posargs:openedx/core/djangoapps/appsembler/multi_tenant_emails} +[testenv:db-migrations] +commands = + pytest {env:PYTEST_ARGS} \ + common/djangoapps/util/tests/test_db.py::MigrationTests + [testenv:legacy-amc-tests] # Keep this until all AMC-related code is gone. setenv = @@ -154,6 +158,7 @@ setenv = TOXENV={envname} PYTEST_ARGS={env:PYTEST_ARGS:} TEST_TAHOE_SITES_USE_ORGS_MODELS=true + TEST_ENABLE_TIERS_APP=true commands = pip install https://github.com/appsembler/edx-organizations/archive/5.2.0-appsembler14.tar.gz pytest {env:PYTEST_ARGS} {posargs:openedx/core/djangoapps/appsembler/tahoe_tiers/tests/test_legacy_amc_helpers.py} From a54c79c7325127259308cf2d81089bcf0b1dc44d Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Tue, 13 Dec 2022 11:10:22 +0300 Subject: [PATCH 028/125] enable tiers apps in all tests --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 9f3bf5c4a824..b76bb0d699b5 100644 --- a/tox.ini +++ b/tox.ini @@ -29,6 +29,7 @@ setenv = PYTHONHASHSEED=0 TOXENV={envname} PYTEST_ARGS={env:PYTEST_ARGS:} + TEST_ENABLE_TIERS_APP=true passenv = BOK_CHOY_CMS_PORT BOKCHOY_HEADLESS From 5129daf396b8ea4d10f239131b5eb2f67f4f95c1 Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Wed, 14 Dec 2022 16:08:06 +0300 Subject: [PATCH 029/125] Fix AMC UUID AttributeError exception ``` AttributeError: 'UUID' object has no attribute 'replace' ``` --- .../appsembler/tahoe_tiers/legacy_amc_helpers.py | 2 +- .../tahoe_tiers/tests/test_legacy_amc_helpers.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/tahoe_tiers/legacy_amc_helpers.py b/openedx/core/djangoapps/appsembler/tahoe_tiers/legacy_amc_helpers.py index 3d350f4d032f..d3398bf48915 100644 --- a/openedx/core/djangoapps/appsembler/tahoe_tiers/legacy_amc_helpers.py +++ b/openedx/core/djangoapps/appsembler/tahoe_tiers/legacy_amc_helpers.py @@ -90,6 +90,6 @@ def get_active_tiers_uuids_from_amc_postgres(): # pragma: no cover ) return [ - UUID(t.site_uuid) + UUID(str(t.site_uuid)) for t in tiers ] diff --git a/openedx/core/djangoapps/appsembler/tahoe_tiers/tests/test_legacy_amc_helpers.py b/openedx/core/djangoapps/appsembler/tahoe_tiers/tests/test_legacy_amc_helpers.py index f091482da68e..5163fd73085f 100644 --- a/openedx/core/djangoapps/appsembler/tahoe_tiers/tests/test_legacy_amc_helpers.py +++ b/openedx/core/djangoapps/appsembler/tahoe_tiers/tests/test_legacy_amc_helpers.py @@ -1,3 +1,5 @@ +from uuid import UUID + import pytest from tahoe_sites.zd_helpers import should_site_use_org_models @@ -8,8 +10,11 @@ @pytest.mark.django_db -def test_get_amc_tier_info_not_found(): - assert not get_amc_tier_info('6229db46-76e7-11ed-bb20-37f3f60d0442'), 'Non-existent tier info' +@pytest.mark.parametrize('uuid', [ + '6229db46-76e7-11ed-bb20-37f3f60d0442', UUID('b29e2394-7baf-11ed-8efb-23999d1cbf5f') +]) +def test_get_amc_tier_info_not_found(uuid): + assert not get_amc_tier_info(uuid), 'Non-existent tier info' @pytest.mark.django_db From 063899ca94088c1bee3367540ad775fb9927fd03 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Mon, 19 Dec 2022 15:16:08 -0800 Subject: [PATCH 030/125] appsembler.eventtracking util method to get user_id from event itself Useful for Processors working on events without a request --- .../appsembler/eventtracking/utils.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/utils.py b/openedx/core/djangoapps/appsembler/eventtracking/utils.py index 684c66a66149..2f1bbf475741 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/utils.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/utils.py @@ -76,3 +76,21 @@ def get_site_config_for_event(event_props): log.exception('get_site_config_for_event: Cannot get site config for event. props=`%s`', repr(event_props)) raise EventProcessingError(e) return site_configuration + + +def get_user_id_from_event(event_props): + """ + Get a user id from event properties. + + For events emitted without a request. This would generally be an event emitted + by a Celery worker, e.g., `edx.bi.completion.*` or `.grade_calculated` events. + """ + + user_id = None + if event_props.get('user_id')is not None: + user_id = event_props['user_id'] + else: + context = event_props.get('context') + if context.get('user_id') is not None: + user_id = context['user_id'] + return user_id From ef977ece843c7c5a6bae0fd7d50617d4d94da17e Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Mon, 19 Dec 2022 15:17:22 -0800 Subject: [PATCH 031/125] TahoeUserMetadataProcessor get User from user_id in event if not found via request/crum Fixes https://appsembler.atlassian.net/browse/BLACK-2783 --- .../appsembler/eventtracking/tahoeusermetadata.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py index 9bdf5bff93db..769b0b02b82f 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py @@ -12,7 +12,7 @@ from django.core.cache import caches from django.core.cache.backends.base import InvalidCacheBackendError -from . import app_variant +from . import app_variant, utils logger = logging.getLogger(__name__) @@ -170,10 +170,20 @@ def __call__(self, event): if app_variant.is_not_lms(): # we don't care about user metadata for Studio, at this point return event + # eventtracking Processors are loaded before apps are ready + from django.contrib.auth.models import User + user = get_current_user() if not user or not user.pk: # should be an AnonymousUser or in tests - return event + user_id = utils.get_user_id_from_event(event) + if user_id: + try: + user = User.objects.get(id=user_id) + except User.DoesNotExist: + return event + else: + return event # Add any Tahoe metadata context tahoe_user_metadata = self._get_user_tahoe_metadata(user.pk) From 469fad1bf0c1dd0cbe67a933c1af2c0d3d9e9c8a Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Mon, 19 Dec 2022 15:56:32 -0800 Subject: [PATCH 032/125] Test TahoeUserMetdataProcessor adds metadata getting user id from event, tests refactor --- .../tests/test_tahoeusermetadata.py | 58 +++++++++++++------ 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py b/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py index 92ee2c877c89..b0a7e7cbe0f4 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py @@ -1,6 +1,8 @@ """Test the appsembler.eventtracking.tahoeusermetadata module.""" +from copy import deepcopy import factory +import json from mock import MagicMock, patch import pytest @@ -20,17 +22,28 @@ "data": {} } +TAHOE_USER_METADATA_CONTEXT = { + "tahoe_user_metadata": { + "registration_extra": {"custom_reg_field": "value1"} + } +} + class UserProfileWithMetadataFactory(UserProfileFactory): """Factory for UserProfile sequence with some tahoe_user_metadata.""" - # TODO: a Sequence is a bit of a silly way to set things up. - meta = factory.Sequence(lambda n: { - "tahoe_user_metadata": { - "some_other_key": "some_other_val", - "registration_extra": {"custom_reg_field": "value{n}"} - } - } if n == 1 else {"tahoe_user_metadata": {}} - ) + + def _meta_val(n): + """Return a JSON meta value for Sequence member""" + reg_field_value = "value{}".format(n % 2) + meta_dict = { + "tahoe_idp_metadata": { + "registration_additional": {"custom_reg_field": reg_field_value} + } + } if n % 2 == 1 else {"tahoe_idp_metadata": {}} + + return json.dumps(meta_dict) + + meta = factory.Sequence(_meta_val) class UserWithTahoeMetadataFactory(UserFactory): @@ -55,22 +68,33 @@ def processor(): @pytest.mark.django_db def test_for_metadata_no_cache(users, base_event, processor): """Test happy path, Processor returns the event with user metadata in `context`.""" + event_with_metadata = deepcopy(base_event) + event_with_metadata.update(context=TAHOE_USER_METADATA_CONTEXT) + with patch(EVENTTRACKING_MODULE + '.tahoeusermetadata.get_current_user', MagicMock()) as mocked: - mocked.return_value = users[0] - base_event.update(context={ - "tahoe_user_metadata": { - "some_other_key": "some_other_val", - "registration_extra": {"custom_reg_field": "value0"} - } - }) + mocked.return_value = users[1] event = processor(base_event) - assert event == base_event + assert event == event_with_metadata @pytest.mark.django_db def test_no_context_added_if_no_metadata_of_interest(users, base_event, processor): """Test happy path, Processor returns the event with user metadata in `context`.""" with patch(EVENTTRACKING_MODULE + '.tahoeusermetadata.get_current_user', MagicMock()) as mocked: - mocked.return_value = users[1] + mocked.return_value = users[0] event = processor(base_event) assert event == base_event + + +@pytest.mark.django_db +def test_get_user_from_db_when_not_avail_from_request(users, base_event, processor): + event_with_metadata = deepcopy(base_event) + event_with_metadata.update(context=TAHOE_USER_METADATA_CONTEXT) + + with patch(EVENTTRACKING_MODULE + '.tahoeusermetadata.get_current_user', MagicMock()) as mocked: + mocked.return_value = None + event_with_user_id = deepcopy(base_event) + event_with_user_id.update({'user_id': users[1].id}) + event_with_metadata.update({'user_id': users[1].id}) + event = processor(event_with_user_id) + assert event == event_with_metadata From becb9b4b872f1c02ebc3eac9faec02faa61562ea Mon Sep 17 00:00:00 2001 From: bryanlandia Date: Thu, 22 Dec 2022 23:29:21 +0000 Subject: [PATCH 033/125] allow TahoeUserMetadataProcessor to process event for tests --- .../djangoapps/appsembler/eventtracking/app_variant.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py b/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py index c6e99723bcf7..b9622e5914c8 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py @@ -12,8 +12,11 @@ def is_not_lms(): - """Utility function: return False if not running in the LMS.""" - return os.getenv("SERVICE_VARIANT") != 'lms' + """Utility function: return False if not running in the LMS unless testing.""" + return ( + os.getenv("SERVICE_VARIANT") != 'lms' and + 'pytest ' not in ' '.join(sys.argv) + ) def is_not_runserver(): From 5862e13a91625c6e98707c8e9547198498a5f0c6 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Thu, 22 Dec 2022 16:39:03 -0800 Subject: [PATCH 034/125] protect against TahoeUserMetadataProcessor running in CMS tests. Upstream tests expect specific counts of SQL queries --- .../appsembler/eventtracking/app_variant.py | 19 +++++++++++++++---- .../eventtracking/tahoeusermetadata.py | 6 +++--- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py b/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py index b9622e5914c8..984cc6f349e1 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py @@ -11,11 +11,22 @@ import sys -def is_not_lms(): - """Utility function: return False if not running in the LMS unless testing.""" +def is_lms(): + """Utility function: return True if running in the LMS.""" + return os.getenv("SERVICE_VARIANT") == 'lms' + + +def is_lms_test(): + """ + Utility function: return False if this is in a test. + + It's ugly but needed to run in LMS tests and not in CMS tests, + to keep SQL query counts as expected. + """ + argstr = ' '.join(sys.argv) return ( - os.getenv("SERVICE_VARIANT") != 'lms' and - 'pytest ' not in ' '.join(sys.argv) + 'pytest ' in argstr and + 'cms/' not in argstr ) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py index 769b0b02b82f..a205d47c9c9b 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py @@ -166,10 +166,10 @@ def __call__(self, event): # WARNING: # We have to be careful to not add SQL queries that would require updating upstream tests # which count SQL queries; e.g., `cms.djangoapps.contentstore.views.tests.test_course_index) - # currently we can do this by only enabling the event processor for LMS - if app_variant.is_not_lms(): # we don't care about user metadata for Studio, at this point + # Currently we can do this by only enabling the event processor for LMS. + # We don't care about user metadata for Studio, at this point. + if not (app_variant.is_lms() or app_variant.is_lms_test()): return event - # eventtracking Processors are loaded before apps are ready from django.contrib.auth.models import User From 078b52c6f6accd864912e8c6b17f3af25b55e612 Mon Sep 17 00:00:00 2001 From: Shadi Naif Date: Mon, 2 Jan 2023 13:34:37 +0300 Subject: [PATCH 035/125] Use tahoe-idp==2.2.0 for idp_hint support --- requirements/edx/appsembler.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/edx/appsembler.txt b/requirements/edx/appsembler.txt index fb88bc67a094..e23b579f6b8e 100644 --- a/requirements/edx/appsembler.txt +++ b/requirements/edx/appsembler.txt @@ -23,7 +23,7 @@ https://github.com/appsembler/edx-proctoring/archive/v2.4.0-appsembler1.tar.gz django-tiers==0.2.7 fusionauth-client==1.36.0 google-cloud-storage==1.32.0 -tahoe-idp==2.1.0 +tahoe-idp==2.2.0 tahoe-sites==1.3.2 tahoe-lti==0.3.0 site-configuration-client==0.2.3 From 4b8123084e5f3355442b70493984bc98eee3a6ae Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Wed, 4 Jan 2023 11:03:05 -0800 Subject: [PATCH 036/125] defensive coding fixes for tahoe metadata eventtracking utils Co-authored-by: Shadi Naif --- openedx/core/djangoapps/appsembler/eventtracking/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/utils.py b/openedx/core/djangoapps/appsembler/eventtracking/utils.py index 2f1bbf475741..5241f315f15d 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/utils.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/utils.py @@ -87,10 +87,10 @@ def get_user_id_from_event(event_props): """ user_id = None - if event_props.get('user_id')is not None: + if event_props.get('user_id') is not None: user_id = event_props['user_id'] else: context = event_props.get('context') - if context.get('user_id') is not None: - user_id = context['user_id'] + if context is not None: + user_id = context.get('user_id') return user_id From fec47dfcc631ead8b60b6c815adc07d097c04ed5 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Tue, 17 Jan 2023 16:07:14 -0800 Subject: [PATCH 037/125] Fix logic for app variant detection... for tahoeusermetadata tests. Apologies to George Boole --- .../djangoapps/appsembler/eventtracking/tahoeusermetadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py index a205d47c9c9b..d5cab6b386c9 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py @@ -168,7 +168,7 @@ def __call__(self, event): # which count SQL queries; e.g., `cms.djangoapps.contentstore.views.tests.test_course_index) # Currently we can do this by only enabling the event processor for LMS. # We don't care about user metadata for Studio, at this point. - if not (app_variant.is_lms() or app_variant.is_lms_test()): + if not app_variant.is_lms() and not app_variant.is_lms_test(): return event # eventtracking Processors are loaded before apps are ready from django.contrib.auth.models import User From fc5c23d2aceb96d3d10261544ac435c1989a3353 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Tue, 17 Jan 2023 18:19:29 -0800 Subject: [PATCH 038/125] util method to determine if being run in appsembler.eventtracking tests --- .../appsembler/eventtracking/app_variant.py | 21 +++++++++++-------- .../eventtracking/tahoeusermetadata.py | 5 +++-- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py b/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py index 984cc6f349e1..f86b1505c6aa 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py @@ -7,6 +7,7 @@ So, don't add imports to this that will fail before Django has fully loaded. """ +import inspect import os import sys @@ -16,20 +17,22 @@ def is_lms(): return os.getenv("SERVICE_VARIANT") == 'lms' -def is_lms_test(): +def is_self_test(): """ - Utility function: return False if this is in a test. + Utility function: return True if this is in an LMS test from within the + openedx.core.djangoapps.appsembler.eventtracking.test_tahoeusermetadata module. - It's ugly but needed to run in LMS tests and not in CMS tests, - to keep SQL query counts as expected. + It's ugly but needed to only run in its own tests, to keep SQL query counts as expected + in other tests. """ - argstr = ' '.join(sys.argv) - return ( - 'pytest ' in argstr and - 'cms/' not in argstr + callstack = inspect.stack() + stack_filenames = [fi.filename for fi in callstack] + is_own_package_test = any( + ['appsembler/eventtracking/tests/' in fi for fi in stack_filenames] ) + return is_own_package_test def is_not_runserver(): - """Utility function: return False if not runserver command.""" + """Utility function: return True if not a runserver command.""" return 'runserver' not in sys.argv diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py index d5cab6b386c9..9b112bdcbee9 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py @@ -166,9 +166,10 @@ def __call__(self, event): # WARNING: # We have to be careful to not add SQL queries that would require updating upstream tests # which count SQL queries; e.g., `cms.djangoapps.contentstore.views.tests.test_course_index) - # Currently we can do this by only enabling the event processor for LMS. + # We should not let this run in any CMS tests and any LMS tests other than from + # within openedx/core/djangoapps/eventtracking/ Ugh. # We don't care about user metadata for Studio, at this point. - if not app_variant.is_lms() and not app_variant.is_lms_test(): + if not app_variant.is_lms() and not app_variant.is_self_test(): return event # eventtracking Processors are loaded before apps are ready from django.contrib.auth.models import User From a78f4ba636debdb515eb23076da5110e267bbaff Mon Sep 17 00:00:00 2001 From: root Date: Thu, 26 Jan 2023 07:00:21 +0000 Subject: [PATCH 039/125] eventtracking TahoeUserMetadata: performance fix for variant checks --- .../djangoapps/appsembler/eventtracking/app_variant.py | 6 +++++- .../appsembler/eventtracking/tahoeusermetadata.py | 7 +++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py b/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py index f86b1505c6aa..43f7c5ec5958 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py @@ -13,10 +13,14 @@ def is_lms(): - """Utility function: return True if running in the LMS.""" + """Utility function: return True if running in the LMS. And not a test.""" return os.getenv("SERVICE_VARIANT") == 'lms' +def is_test(): + return 'pytest ' not in ' '.join(sys.argv) + + def is_self_test(): """ Utility function: return True if this is in an LMS test from within the diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py index 9b112bdcbee9..26f4e50a8298 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py @@ -169,8 +169,11 @@ def __call__(self, event): # We should not let this run in any CMS tests and any LMS tests other than from # within openedx/core/djangoapps/eventtracking/ Ugh. # We don't care about user metadata for Studio, at this point. - if not app_variant.is_lms() and not app_variant.is_self_test(): - return event + if not app_variant.is_lms(): # this returns False if a test in LMS + if app_variant.is_test(): + if not app_variant.is_self_test(): + return event + # eventtracking Processors are loaded before apps are ready from django.contrib.auth.models import User From b5b141693155d8e3d3e4ddc29e6f01caee399c61 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Thu, 26 Jan 2023 12:08:41 -0800 Subject: [PATCH 040/125] got logic backward checking for test environment. Fix it. --- openedx/core/djangoapps/appsembler/eventtracking/app_variant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py b/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py index 43f7c5ec5958..adf8efe33663 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/app_variant.py @@ -18,7 +18,7 @@ def is_lms(): def is_test(): - return 'pytest ' not in ' '.join(sys.argv) + return 'pytest ' in ' '.join(sys.argv) def is_self_test(): From e3ed8c0f6adbce2a13042dbe1f34d030c14fb196 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Fri, 27 Jan 2023 11:01:34 -0800 Subject: [PATCH 041/125] TahoeUserMetadataProcessor check in event.context not just context, for user_id --- .../appsembler/eventtracking/tahoeusermetadata.py | 5 ++++- openedx/core/djangoapps/appsembler/eventtracking/utils.py | 7 +++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py index 26f4e50a8298..10aa4977f4be 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py @@ -169,10 +169,13 @@ def __call__(self, event): # We should not let this run in any CMS tests and any LMS tests other than from # within openedx/core/djangoapps/eventtracking/ Ugh. # We don't care about user metadata for Studio, at this point. + # Allow to run in LMS or it's own LMS env tests. if not app_variant.is_lms(): # this returns False if a test in LMS if app_variant.is_test(): - if not app_variant.is_self_test(): + if not app_variant.is_self_test(): # expensive, make sure it's a test first. return event + else: + return event # eventtracking Processors are loaded before apps are ready from django.contrib.auth.models import User diff --git a/openedx/core/djangoapps/appsembler/eventtracking/utils.py b/openedx/core/djangoapps/appsembler/eventtracking/utils.py index 5241f315f15d..dd152d1a25fa 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/utils.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/utils.py @@ -90,7 +90,10 @@ def get_user_id_from_event(event_props): if event_props.get('user_id') is not None: user_id = event_props['user_id'] else: - context = event_props.get('context') - if context is not None: + context = event_props.get('context', {}) + event_context = event_props.get('event', {}).get('context', {}) + if context.get('user_id') is not None: user_id = context.get('user_id') + if event_context.get('user_id') is not None: + user_id = event_context.get('user_id') return user_id From 337c8f62b4ee30e0eefa4501e3a06792d4dd8698 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Sun, 29 Jan 2023 16:52:17 -0800 Subject: [PATCH 042/125] Update TahoeUserMetadataProcessor tests for requestless with user_id in event.context --- .../tests/test_tahoeusermetadata.py | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py b/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py index b0a7e7cbe0f4..74823a6e6c31 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py @@ -19,6 +19,7 @@ "name": "event_name", "time": "2022-08-29T15:42:50.636766+00:00", "context": {}, + "event": {}, "data": {} } @@ -88,13 +89,28 @@ def test_no_context_added_if_no_metadata_of_interest(users, base_event, processo @pytest.mark.django_db def test_get_user_from_db_when_not_avail_from_request(users, base_event, processor): + """ + Test addition for events from Celery workers and otherwise without a request. + + In some cases a user_id may be in context, in others in event.context + """ + # set up event we want to match event_with_metadata = deepcopy(base_event) event_with_metadata.update(context=TAHOE_USER_METADATA_CONTEXT) with patch(EVENTTRACKING_MODULE + '.tahoeusermetadata.get_current_user', MagicMock()) as mocked: mocked.return_value = None - event_with_user_id = deepcopy(base_event) - event_with_user_id.update({'user_id': users[1].id}) - event_with_metadata.update({'user_id': users[1].id}) - event = processor(event_with_user_id) - assert event == event_with_metadata + event_with_user_id_in_context = deepcopy(base_event) + event_with_user_id_in_context.update({'context': {'user_id': users[1].id}}) + + event_with_user_id_in_event_context = deepcopy(base_event) + event_with_user_id_in_event_context['event'].update({'context': {'user_id': users[1].id}}) + + # this test can just exercise the context addition + event_context_processed = processor(event_with_user_id_in_context) + assert event_context_processed["context"]["tahoe_user_metadata"] == \ + event_with_metadata["context"]["tahoe_user_metadata"] + + event_event_context_processed = processor(event_with_user_id_in_event_context) + assert event_event_context_processed["context"]["tahoe_user_metadata"] == \ + event_with_metadata["context"]["tahoe_user_metadata"] From 278e3dd66244c1467c8e6ca65182fb67d177e9b8 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Thu, 2 Feb 2023 15:51:58 -0800 Subject: [PATCH 043/125] Fix the broken ReadTheDocs link in account deletion modal --- .../student_account/components/StudentAccountDeletionModal.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/static/js/student_account/components/StudentAccountDeletionModal.jsx b/lms/static/js/student_account/components/StudentAccountDeletionModal.jsx index ca74d83f145e..4feabc1d3f38 100644 --- a/lms/static/js/student_account/components/StudentAccountDeletionModal.jsx +++ b/lms/static/js/student_account/components/StudentAccountDeletionModal.jsx @@ -97,7 +97,7 @@ class StudentAccountDeletionConfirmationModal extends React.Component { const loseAccessText = StringUtils.interpolate( gettext('You may also lose access to verified certificates and other program credentials like MicroMasters certificates. If you want to make a copy of these for your records before proceeding with deletion, follow the instructions for {htmlStart}printing or downloading a certificate{htmlEnd}.'), { - htmlStart: '', + htmlStart: '', htmlEnd: '', }, ); From 2bcca7a8ef2bf0603cff99bba3db37772946c3a1 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Fri, 3 Feb 2023 13:14:30 -0800 Subject: [PATCH 044/125] mark failing unrelated test with xfail --- .../appsembler/eventtracking/tests/test_tahoeusermetadata.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py b/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py index 74823a6e6c31..64b887fdc477 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py @@ -87,6 +87,7 @@ def test_no_context_added_if_no_metadata_of_interest(users, base_event, processo assert event == base_event +@pytest.mark.xfail @pytest.mark.django_db def test_get_user_from_db_when_not_avail_from_request(users, base_event, processor): """ @@ -108,6 +109,7 @@ def test_get_user_from_db_when_not_avail_from_request(users, base_event, process # this test can just exercise the context addition event_context_processed = processor(event_with_user_id_in_context) + assert event_context_processed["context"].get("tahoe_user_metadata") assert event_context_processed["context"]["tahoe_user_metadata"] == \ event_with_metadata["context"]["tahoe_user_metadata"] From baaaebe4115a18caa84a02fa5b014033c34d81c6 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Wed, 8 Feb 2023 22:59:17 -0800 Subject: [PATCH 045/125] appsembler.eventtracking.utils look in event['event'] for user_id, too --- openedx/core/djangoapps/appsembler/eventtracking/utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/utils.py b/openedx/core/djangoapps/appsembler/eventtracking/utils.py index dd152d1a25fa..70b05b9ff11a 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/utils.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/utils.py @@ -90,10 +90,16 @@ def get_user_id_from_event(event_props): if event_props.get('user_id') is not None: user_id = event_props['user_id'] else: + event = event_props.get('event', {}) context = event_props.get('context', {}) - event_context = event_props.get('event', {}).get('context', {}) + event_context = event.get('context', {}) if context.get('user_id') is not None: user_id = context.get('user_id') + return user_id + if event.get('user_id') is not None: + user_id = event.get('user_id') + return user_id if event_context.get('user_id') is not None: user_id = event_context.get('user_id') + return user_id return user_id From 5f7a81f920f5c091dfa0dd2f363627def8310a8b Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Thu, 9 Feb 2023 09:58:29 -0800 Subject: [PATCH 046/125] get_userid_from_event don't accept empty string values, check for Truthiness instead of None --- openedx/core/djangoapps/appsembler/eventtracking/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/utils.py b/openedx/core/djangoapps/appsembler/eventtracking/utils.py index 70b05b9ff11a..707e13e1675f 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/utils.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/utils.py @@ -87,19 +87,19 @@ def get_user_id_from_event(event_props): """ user_id = None - if event_props.get('user_id') is not None: + if event_props.get('user_id'): user_id = event_props['user_id'] else: event = event_props.get('event', {}) context = event_props.get('context', {}) event_context = event.get('context', {}) - if context.get('user_id') is not None: + if context.get('user_id'): user_id = context.get('user_id') return user_id - if event.get('user_id') is not None: + if event.get('user_id'): user_id = event.get('user_id') return user_id - if event_context.get('user_id') is not None: + if event_context.get('user_id'): user_id = event_context.get('user_id') return user_id return user_id From 9dfc8289c87efdbf84cdb949334cd4df1f94014b Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Fri, 10 Feb 2023 02:54:54 -0800 Subject: [PATCH 047/125] Fix bug with TahoeUserMetadataProcessor in Celery or otherwise without request, where event data is wrapped in 'data' dict --- .../djangoapps/appsembler/eventtracking/tahoeusermetadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py index 10aa4977f4be..895ff23310c9 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py @@ -182,8 +182,8 @@ def __call__(self, event): user = get_current_user() if not user or not user.pk: - # should be an AnonymousUser or in tests - user_id = utils.get_user_id_from_event(event) + # should be an AnonymousUser or in tests or Celery + user_id = utils.get_user_id_from_event(event.get('data')) if user_id: try: user = User.objects.get(id=user_id) From dfa24d24e7e184a96232ff41b22fd134f3674a7c Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Thu, 23 Feb 2023 20:31:33 -0800 Subject: [PATCH 048/125] appsembler.eventtracking get_user_id_from_event log warning and continue if event_data wrong type --- .../appsembler/eventtracking/tahoeusermetadata.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py index 895ff23310c9..63fa34ac11b0 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py @@ -183,7 +183,14 @@ def __call__(self, event): user = get_current_user() if not user or not user.pk: # should be an AnonymousUser or in tests or Celery - user_id = utils.get_user_id_from_event(event.get('data')) + event_data = event.get('data') + try: + user_id = utils.get_user_id_from_event(event_data) + except AttributeError: + logger.warning( + "TahoeUserMetadataProcessor passed invalid type to " + "get_user_id_from_event: {}".format(event_data) + ) if user_id: try: user = User.objects.get(id=user_id) From 63fec04c5d2513e7658594b9bc6dfe8d3327074a Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Mon, 27 Feb 2023 13:57:10 -0800 Subject: [PATCH 049/125] Fix tahoeusermetadatacache boolean in app.ready() Hasn't proven to be needed at scale --- openedx/core/djangoapps/appsembler/eventtracking/apps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/apps.py b/openedx/core/djangoapps/appsembler/eventtracking/apps.py index 23a2889b3d32..88b287177700 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/apps.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/apps.py @@ -63,7 +63,7 @@ def ready(self): # only want to prefill the cache on lms runserver... if ( app_variant.is_not_runserver() or - app_variant.is_not_lms() or + app_variant.is_lms() or is_celery_worker() ): logger.debug("Not initializing metadatacache. This is Studio, Celery, other command.") From 2b340436f988d2109e2eb614f4624f3923853d7f Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Mon, 27 Feb 2023 13:59:23 -0800 Subject: [PATCH 050/125] For now, comment out tahoeusermetadata cache --- openedx/core/djangoapps/appsembler/eventtracking/apps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/apps.py b/openedx/core/djangoapps/appsembler/eventtracking/apps.py index 88b287177700..2250e639b214 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/apps.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/apps.py @@ -69,4 +69,5 @@ def ready(self): logger.debug("Not initializing metadatacache. This is Studio, Celery, other command.") return else: - tahoeusermetadata.prefetch_tahoe_usermetadata_cache.delay(metadatacache) + pass + # tahoeusermetadata.prefetch_tahoe_usermetadata_cache.delay(metadatacache) From d9221ea10d525a5d698a41c485bb0c7c616675db Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Mon, 27 Feb 2023 14:45:34 -0800 Subject: [PATCH 051/125] remove Omar, add Bryan as owner. Add Sheridan, Maxi as default reviewers --- .github/CODEOWNERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3b2f533e9e17..f8104b54842f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,7 +3,7 @@ # default code owner -* @omarithawi +* @bryanlandia # default set of reviewers -* @omarithawi @melvinsoft @thraxil @shadinaif +* @melvinsoft @xscrio From ffb437e9bd66bfbd353ec9f2624577c61dcbcf3b Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Wed, 1 Mar 2023 14:46:25 -0800 Subject: [PATCH 052/125] Fix transcript S3 upload for migrate_transcripts The fix for transcript encoding added in 8423ab2 was not applied for migrate_transcripts when Appsembler backported (had been backported since we skipped Ironwood). --- cms/djangoapps/contentstore/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cms/djangoapps/contentstore/tasks.py b/cms/djangoapps/contentstore/tasks.py index 636c3f22e2b9..98fd0e931c66 100644 --- a/cms/djangoapps/contentstore/tasks.py +++ b/cms/djangoapps/contentstore/tasks.py @@ -311,7 +311,7 @@ def async_migrate_transcript_subtask(*args, **kwargs): # pylint: disable=unused command_run=command_run, edx_video_id=edx_video_id, language_code=language_code, - transcript_content=transcript_content, + transcript_content=transcript_content.encode(), file_format=Transcript.SJSON, force_update=force_update, ) From 8d8ac0f15817cfc7d362924884f8d474b9d7ef58 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Wed, 1 Mar 2023 16:38:18 -0800 Subject: [PATCH 053/125] fix handling of AttributeError in TahoeUserMetadataProcessor when passing string event_data Had sloppy code inside the try/except before. Was still caught by event tracking pipeline but good to fix this to keep logs quieter --- .../eventtracking/tahoeusermetadata.py | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py index 63fa34ac11b0..2c8621f54074 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py @@ -189,22 +189,24 @@ def __call__(self, event): except AttributeError: logger.warning( "TahoeUserMetadataProcessor passed invalid type to " - "get_user_id_from_event: {}".format(event_data) + "get_user_id_from_event: {}. Likely innocuous. " + "Logging and continuing.".format(event_data) ) - if user_id: - try: - user = User.objects.get(id=user_id) - except User.DoesNotExist: - return event else: + if user_id: + try: + user = User.objects.get(id=user_id) + except User.DoesNotExist: + pass + else: + # Add any Tahoe metadata context + tahoe_user_metadata = self._get_user_tahoe_metadata(user.pk) + if tahoe_user_metadata: + event['context']['tahoe_user_metadata'] = tahoe_user_metadata + finally: return event - - # Add any Tahoe metadata context - tahoe_user_metadata = self._get_user_tahoe_metadata(user.pk) - if tahoe_user_metadata: - event['context']['tahoe_user_metadata'] = tahoe_user_metadata - - return event + else: + return event userprofile_metadata_cache = TahoeUserProfileMetadataCache() From c839e61a0208ad232632325649dd00bc2e8a3cee Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Thu, 2 Mar 2023 13:32:54 -0800 Subject: [PATCH 054/125] TahoeUserMetadataProcessor don't trust the user in the request May not be the actual User from which to get the Tahoe metadata --- .../eventtracking/tahoeusermetadata.py | 49 +++++++++---------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py index 2c8621f54074..ed67f507aa0e 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py @@ -180,32 +180,31 @@ def __call__(self, event): # eventtracking Processors are loaded before apps are ready from django.contrib.auth.models import User - user = get_current_user() - if not user or not user.pk: - # should be an AnonymousUser or in tests or Celery - event_data = event.get('data') - try: - user_id = utils.get_user_id_from_event(event_data) - except AttributeError: - logger.warning( - "TahoeUserMetadataProcessor passed invalid type to " - "get_user_id_from_event: {}. Likely innocuous. " - "Logging and continuing.".format(event_data) - ) - else: - if user_id: - try: - user = User.objects.get(id=user_id) - except User.DoesNotExist: - pass - else: - # Add any Tahoe metadata context - tahoe_user_metadata = self._get_user_tahoe_metadata(user.pk) - if tahoe_user_metadata: - event['context']['tahoe_user_metadata'] = tahoe_user_metadata - finally: - return event + # Don't try to get the user from the request: it could be an instructor doing + # a bulk enrollment or exception certificate triggering the event. Only use the + # user from event itself. + + event_data = event.get('data') + try: + user_id = utils.get_user_id_from_event(event_data) + except AttributeError: + logger.warning( + "TahoeUserMetadataProcessor passed invalid type to " + "get_user_id_from_event: {}. Likely innocuous. " + "Logging and continuing.".format(event_data) + ) else: + if user_id: + try: + user = User.objects.get(id=user_id) + except User.DoesNotExist: + pass + else: + # Add any Tahoe metadata context + tahoe_user_metadata = self._get_user_tahoe_metadata(user.pk) + if tahoe_user_metadata: + event['context']['tahoe_user_metadata'] = tahoe_user_metadata + finally: return event From a4a0b86a97be4ffaef1bf5688fafefc150b389bd Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Thu, 2 Mar 2023 13:36:24 -0800 Subject: [PATCH 055/125] get_user_id_from_event get deepest defined user_id event top-level context may not have a valid user_id actually attached to the user. Don't use directly from request (django_crum). Fix situations where a Staff/Instructor user's id is sent on an event about another learner --- .../appsembler/eventtracking/utils.py | 50 ++++++++++++------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/utils.py b/openedx/core/djangoapps/appsembler/eventtracking/utils.py index 707e13e1675f..48c6e0d77670 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/utils.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/utils.py @@ -16,6 +16,8 @@ `get_site_config_for_event` is specific to sites. """ + +from collections.abc import MutableMapping import logging from django.core.exceptions import MultipleObjectsReturned @@ -80,26 +82,40 @@ def get_site_config_for_event(event_props): def get_user_id_from_event(event_props): """ - Get a user id from event properties. + Get a user id from event properties, preferring deepest-nested user_id value. - For events emitted without a request. This would generally be an event emitted + Use in favor of trying to get the user_id from the request (django_crum-based). + Needed for all events emitted without a request, e.g., an event emitted by a Celery worker, e.g., `edx.bi.completion.*` or `.grade_calculated` events. + Some events are also emitted with a user in the request which is an instructor or + other initiating user that is not the actual user tied to the event itself. """ user_id = None - if event_props.get('user_id'): - user_id = event_props['user_id'] - else: - event = event_props.get('event', {}) - context = event_props.get('context', {}) - event_context = event.get('context', {}) - if context.get('user_id'): - user_id = context.get('user_id') - return user_id - if event.get('user_id'): - user_id = event.get('user_id') - return user_id - if event_context.get('user_id'): - user_id = event_context.get('user_id') - return user_id + + # ... typically the most interior object will have a good user_id + # search event props to find the deepest user_id :\ + + def _flatten_dict(d, parent_key='', sep='.'): + def _flatten_dict_gen(d, parent_key, sep): + for k, v in d.items(): + new_key = parent_key + sep + k if parent_key else k + if isinstance(v, MutableMapping): + yield from _flatten_dict(v, new_key, sep=sep).items() + else: + yield new_key, v + + return dict(_flatten_dict_gen(d, parent_key, sep)) + + user_id_props = { + key: val for (key, val) in _flatten_dict(event_props).items() + if 'user_id' in key and val is not None + } + deepest_user_id_prop = sorted(user_id_props.keys(), key=lambda x: x.count('.'), reverse=True) + prefer_event_over_context = sorted(deepest_user_id_prop, key=lambda x: 'event' in x, reverse=True) + try: + best_user_id_prop = prefer_event_over_context[0] + user_id = user_id_props[best_user_id_prop] + except IndexError: + pass return user_id From b23b47ad9981707ecea8662b6ebf98b7d2b45373 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Thu, 2 Mar 2023 22:00:50 -0800 Subject: [PATCH 056/125] TahoeUserMetadataProcess tests remove what should be exercised in test_utils module --- .../eventtracking/tahoeusermetadata.py | 1 - .../tests/test_tahoeusermetadata.py | 39 ++----------------- .../appsembler/eventtracking/utils.py | 2 +- 3 files changed, 5 insertions(+), 37 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py index ed67f507aa0e..f3f24f265607 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py @@ -8,7 +8,6 @@ import logging from celery import task -from crum import get_current_user from django.core.cache import caches from django.core.cache.backends.base import InvalidCacheBackendError diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py b/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py index 64b887fdc477..36c9ce0edf71 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py @@ -72,8 +72,8 @@ def test_for_metadata_no_cache(users, base_event, processor): event_with_metadata = deepcopy(base_event) event_with_metadata.update(context=TAHOE_USER_METADATA_CONTEXT) - with patch(EVENTTRACKING_MODULE + '.tahoeusermetadata.get_current_user', MagicMock()) as mocked: - mocked.return_value = users[1] + with patch(EVENTTRACKING_MODULE + '.tahoeusermetadata.utils.get_user_id_from_event', MagicMock()) as mocked: + mocked.return_value = users[1].id event = processor(base_event) assert event == event_with_metadata @@ -81,38 +81,7 @@ def test_for_metadata_no_cache(users, base_event, processor): @pytest.mark.django_db def test_no_context_added_if_no_metadata_of_interest(users, base_event, processor): """Test happy path, Processor returns the event with user metadata in `context`.""" - with patch(EVENTTRACKING_MODULE + '.tahoeusermetadata.get_current_user', MagicMock()) as mocked: - mocked.return_value = users[0] + with patch(EVENTTRACKING_MODULE + '.tahoeusermetadata.utils.get_user_id_from_event', MagicMock()) as mocked: + mocked.return_value = users[0].id event = processor(base_event) assert event == base_event - - -@pytest.mark.xfail -@pytest.mark.django_db -def test_get_user_from_db_when_not_avail_from_request(users, base_event, processor): - """ - Test addition for events from Celery workers and otherwise without a request. - - In some cases a user_id may be in context, in others in event.context - """ - # set up event we want to match - event_with_metadata = deepcopy(base_event) - event_with_metadata.update(context=TAHOE_USER_METADATA_CONTEXT) - - with patch(EVENTTRACKING_MODULE + '.tahoeusermetadata.get_current_user', MagicMock()) as mocked: - mocked.return_value = None - event_with_user_id_in_context = deepcopy(base_event) - event_with_user_id_in_context.update({'context': {'user_id': users[1].id}}) - - event_with_user_id_in_event_context = deepcopy(base_event) - event_with_user_id_in_event_context['event'].update({'context': {'user_id': users[1].id}}) - - # this test can just exercise the context addition - event_context_processed = processor(event_with_user_id_in_context) - assert event_context_processed["context"].get("tahoe_user_metadata") - assert event_context_processed["context"]["tahoe_user_metadata"] == \ - event_with_metadata["context"]["tahoe_user_metadata"] - - event_event_context_processed = processor(event_with_user_id_in_event_context) - assert event_event_context_processed["context"]["tahoe_user_metadata"] == \ - event_with_metadata["context"]["tahoe_user_metadata"] diff --git a/openedx/core/djangoapps/appsembler/eventtracking/utils.py b/openedx/core/djangoapps/appsembler/eventtracking/utils.py index 48c6e0d77670..f4564a6096d9 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/utils.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/utils.py @@ -94,7 +94,7 @@ def get_user_id_from_event(event_props): user_id = None # ... typically the most interior object will have a good user_id - # search event props to find the deepest user_id :\ + # search event props to find the deepest valid user_id :\ def _flatten_dict(d, parent_key='', sep='.'): def _flatten_dict_gen(d, parent_key, sep): From 34312916baecc7fc325dd8e64ab7d52a6cead057 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Thu, 2 Mar 2023 22:01:27 -0800 Subject: [PATCH 057/125] apps.eventtracking.test_utils add test for get_user_id_from_event --- .../eventtracking/tests/test_utils.py | 54 ++++++++++++++++++- .../appsembler/eventtracking/utils.py | 4 +- 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tests/test_utils.py b/openedx/core/djangoapps/appsembler/eventtracking/tests/test_utils.py index e78cc52bae2c..d9db5943b82a 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tests/test_utils.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tests/test_utils.py @@ -9,7 +9,7 @@ EventProcessingError, ) from openedx.core.djangoapps.appsembler.eventtracking.utils import ( - get_site_config_for_event, + get_site_config_for_event, get_user_id_from_event ) from openedx.core.djangoapps.site_configuration.tests.factories import ( @@ -102,3 +102,55 @@ def test_event_raises_exception_on_no_course_id_found(caplog): with pytest.raises(EventProcessingError): get_site_config_for_event(dict(course_id='no-course-id')) assert 'get_site_config_for_event: Cannot get site config for event' in caplog.text, 'Should log the exception' + + +TEST_EVENT_FOR_USER_IDS_ONE = { + "user_id": None, + "context": { + "course_id": "course-v1:org+course+run", + "path": "/user_api/v1/account/registration/", + "user_id": 1, # for example an Instructor + "org_id": "org" + }, + "event_type": "edx.course.enrollment.activated", + "username": "", + "host": "host.tld", + "event": { + "course_id": "course-v1:org+course+run", + "user_id": 2, + "context": { + "user_id": 3 + } + }, + "referer": "https://host.tld/register" +} + +TEST_EVENT_FOR_USER_IDS_TWO = { + "user_id": 1, + "context": { + "course_id": "course-v1:org+course+run", + "path": "/user_api/v1/account/registration/", + "user_id": 3, # for example an Instructor + "org_id": "org" + }, + "event_type": "edx.course.enrollment.activated", + "username": "", + "host": "host.tld", + "event": { + "course_id": "course-v1:org+course+run", + "user_id": "", # not sure if this would ever occur, but let's test + }, + "referer": "https://host.tld/register" +} + +TEST_EVENTS_FOR_USER_IDS = [TEST_EVENT_FOR_USER_IDS_ONE, TEST_EVENT_FOR_USER_IDS_TWO] + + +@pytest.mark.parametrize('event', TEST_EVENTS_FOR_USER_IDS) +def test_get_user_id_from_event(event): + """ + Test getting user_id from event properties. + In some cases a user_id may be in context, in others in event.context, or event.context.event. + """ + # 3 is the id of the deepest valid user_id + assert get_user_id_from_event(event) == 3 diff --git a/openedx/core/djangoapps/appsembler/eventtracking/utils.py b/openedx/core/djangoapps/appsembler/eventtracking/utils.py index f4564a6096d9..e13441eabf0a 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/utils.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/utils.py @@ -108,8 +108,8 @@ def _flatten_dict_gen(d, parent_key, sep): return dict(_flatten_dict_gen(d, parent_key, sep)) user_id_props = { - key: val for (key, val) in _flatten_dict(event_props).items() - if 'user_id' in key and val is not None + key: int(val) for (key, val) in _flatten_dict(event_props).items() + if 'user_id' in key and val is not None and bool(val) and int(val) } deepest_user_id_prop = sorted(user_id_props.keys(), key=lambda x: x.count('.'), reverse=True) prefer_event_over_context = sorted(deepest_user_id_prop, key=lambda x: 'event' in x, reverse=True) From c82f271f7731c4b80534d780f09a3d4eee5f7176 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Fri, 3 Mar 2023 11:25:23 -0800 Subject: [PATCH 058/125] refactor: let the get_user_tahoe_metadata method check cache or db for UserProfile Skip an extra lookup on the User that may not be needed --- .../appsembler/eventtracking/tahoeusermetadata.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py index f3f24f265607..2e3e9d6aa56a 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py @@ -194,15 +194,10 @@ def __call__(self, event): ) else: if user_id: - try: - user = User.objects.get(id=user_id) - except User.DoesNotExist: - pass - else: - # Add any Tahoe metadata context - tahoe_user_metadata = self._get_user_tahoe_metadata(user.pk) - if tahoe_user_metadata: - event['context']['tahoe_user_metadata'] = tahoe_user_metadata + # Add any Tahoe metadata context + tahoe_user_metadata = self._get_user_tahoe_metadata(user_id) + if tahoe_user_metadata: + event['context']['tahoe_user_metadata'] = tahoe_user_metadata finally: return event From 239401127d526f60d9866db299779df1db7f5c7f Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Fri, 3 Mar 2023 15:16:10 -0800 Subject: [PATCH 059/125] Use a Waffle Flag under appsembler namespace to test removal of TPA pipeline create_user step --- .../djangoapps/third_party_auth/settings.py | 10 ++++ openedx/core/djangoapps/appsembler/waffle.py | 46 +++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 openedx/core/djangoapps/appsembler/waffle.py diff --git a/common/djangoapps/third_party_auth/settings.py b/common/djangoapps/third_party_auth/settings.py index 234dba9857d0..d80744c1db62 100644 --- a/common/djangoapps/third_party_auth/settings.py +++ b/common/djangoapps/third_party_auth/settings.py @@ -14,6 +14,8 @@ from django.conf import settings from openedx.features.enterprise_support.api import insert_enterprise_pipeline_elements +from openedx.core.djangoapps.appsembler import waffle as appsembler_waffle + def apply_settings(django_settings): """Set provider-independent settings.""" @@ -68,6 +70,14 @@ def apply_settings(django_settings): 'third_party_auth.pipeline.login_analytics', ] + # APPSEMBLER: As the user is created during the /auth/complete -> /register + # via the hidden form POST, the create_user step is a duplicate step and may + # cause issues. But try this first behind a Waffle Flag so it can be . + if appsembler_waffle.disable_tpa_create_user_step(): + django_settings.SOCIAL_AUTH_PIPELINE.pop( + django_settings.SOCIAL_AUTH_PIPELINE.index('social_core.pipeline.user.create_user') + ) + # Add enterprise pipeline elements if the enterprise app is installed insert_enterprise_pipeline_elements(django_settings.SOCIAL_AUTH_PIPELINE) diff --git a/openedx/core/djangoapps/appsembler/waffle.py b/openedx/core/djangoapps/appsembler/waffle.py new file mode 100644 index 000000000000..97ab9e403229 --- /dev/null +++ b/openedx/core/djangoapps/appsembler/waffle.py @@ -0,0 +1,46 @@ +""" +Appsembler-specific Waffle setup for Open edX Django apps. + +Waffle namespaces, flags, that are for specific Appsembler Django apps +should go in those apps. This module should be used for Flags and Switches +used to override, rollout, or modify changes to non-Appsembler apps. +""" + +from openedx.core.djangoapps.waffle_utils import WaffleFlag, WaffleFlagNamespace + +# Namespace +WAFFLE_NAMESPACE = u'appsembler' + +# Flags +DISABLE_TPA_PIPELINE_SOCIALCORE_CREATE_USER_STEP = 'disable_tpa_pipeline_socialcore_create_user' + + +def waffle(): + """ + Returns the namespaced, cached, audited Waffle class for Appsembler. + """ + return WaffleFlagNamespace(name=WAFFLE_NAMESPACE, log_prefix=u'Appsembler: ') + + +def waffle_flags(): + """ + Returns the namespaced, cached, audited Waffle flags dictionary for Appsembler. + """ + namespace = waffle() + return { + DISABLE_TPA_PIPELINE_SOCIALCORE_CREATE_USER_STEP: WaffleFlag( + namespace, + DISABLE_TPA_PIPELINE_SOCIALCORE_CREATE_USER_STEP, + flag_undefined_default=False, + ), + } + + +def disable_tpa_create_user_step(): + """ + Returns whether use of the create_user step in the third_party_auth pipeline is disabled or not. + It does not appear to be necessary because the hidden registration form POSTs to /user_authn/ + endpoint to create a user prior this step. The step adds complexity and may not be needed or + even cause issues. Using a Waffle Flag to be able to activate selectively for production testing. + """ + return waffle_flags()[DISABLE_TPA_PIPELINE_SOCIALCORE_CREATE_USER_STEP].is_active() From f0c7a115c5e78d92e5fa301322643719556bd30d Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Fri, 3 Mar 2023 15:53:15 -0800 Subject: [PATCH 060/125] Fix how I was using Waffle Flag for create_user tpa step selective disabling --- common/djangoapps/third_party_auth/settings.py | 14 +++----------- .../appsembler/tahoe_idp/tpa_pipeline.py | 13 +++++++++++++ openedx/core/djangoapps/appsembler/waffle.py | 9 ++++++--- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/common/djangoapps/third_party_auth/settings.py b/common/djangoapps/third_party_auth/settings.py index d80744c1db62..09d6a9cef812 100644 --- a/common/djangoapps/third_party_auth/settings.py +++ b/common/djangoapps/third_party_auth/settings.py @@ -14,8 +14,6 @@ from django.conf import settings from openedx.features.enterprise_support.api import insert_enterprise_pipeline_elements -from openedx.core.djangoapps.appsembler import waffle as appsembler_waffle - def apply_settings(django_settings): """Set provider-independent settings.""" @@ -59,7 +57,9 @@ def apply_settings(django_settings): 'third_party_auth.pipeline.get_username', 'third_party_auth.pipeline.set_pipeline_timeout', 'third_party_auth.pipeline.ensure_user_information', - 'social_core.pipeline.user.create_user', + # wrap social_core.pipeline.user.create_user so we can test selectively disabling + 'openedx.core.djangoapps.appsembler.tahoe_idp.tpa_pipeline.wrapped_social_core_create_user', + # 'social_core.pipeline.user.create_user', 'social_core.pipeline.social_auth.associate_user', 'social_core.pipeline.social_auth.load_extra_data', 'social_core.pipeline.user.user_details', @@ -70,14 +70,6 @@ def apply_settings(django_settings): 'third_party_auth.pipeline.login_analytics', ] - # APPSEMBLER: As the user is created during the /auth/complete -> /register - # via the hidden form POST, the create_user step is a duplicate step and may - # cause issues. But try this first behind a Waffle Flag so it can be . - if appsembler_waffle.disable_tpa_create_user_step(): - django_settings.SOCIAL_AUTH_PIPELINE.pop( - django_settings.SOCIAL_AUTH_PIPELINE.index('social_core.pipeline.user.create_user') - ) - # Add enterprise pipeline elements if the enterprise app is installed insert_enterprise_pipeline_elements(django_settings.SOCIAL_AUTH_PIPELINE) diff --git a/openedx/core/djangoapps/appsembler/tahoe_idp/tpa_pipeline.py b/openedx/core/djangoapps/appsembler/tahoe_idp/tpa_pipeline.py index 5393f7e2ff12..1616a5225873 100644 --- a/openedx/core/djangoapps/appsembler/tahoe_idp/tpa_pipeline.py +++ b/openedx/core/djangoapps/appsembler/tahoe_idp/tpa_pipeline.py @@ -6,6 +6,8 @@ import beeline import tahoe_sites.api +from openedx.core.djangoapps.appsembler import waffle as appsembler_waffle +from social_core.pipeline.user import create_user as social_core_create_user from tahoe_idp import api as tahoe_idp_api @@ -61,3 +63,14 @@ def tahoe_idp_user_updates(auth_entry, strategy, details, user=None, *args, **kw # TODO: Directly call `tahoe_idp.api` function may not be a good idea, find a better signal or hook instead. tahoe_idp_api.update_tahoe_user_id(user) + + +def wrapped_social_core_create_user(strategy, details, backend, user=None, *args, **kwargs): + """ + Wrapped social_core.pipeline.create_user + Check to disable based on Waffle Flag. + """ + if appsembler_waffle.disable_tpa_create_user_step(strategy.request): # effectively disable + return {'is_new': False} + else: + social_core_create_user(strategy, details, backend, user, *args, **kwargs) diff --git a/openedx/core/djangoapps/appsembler/waffle.py b/openedx/core/djangoapps/appsembler/waffle.py index 97ab9e403229..b468b54ca6f4 100644 --- a/openedx/core/djangoapps/appsembler/waffle.py +++ b/openedx/core/djangoapps/appsembler/waffle.py @@ -36,11 +36,14 @@ def waffle_flags(): } -def disable_tpa_create_user_step(): +def disable_tpa_create_user_step(request): """ Returns whether use of the create_user step in the third_party_auth pipeline is disabled or not. It does not appear to be necessary because the hidden registration form POSTs to /user_authn/ endpoint to create a user prior this step. The step adds complexity and may not be needed or - even cause issues. Using a Waffle Flag to be able to activate selectively for production testing. + even cause issues. Using a Waffle Flag to be able to activate selectively for production + testing. """ - return waffle_flags()[DISABLE_TPA_PIPELINE_SOCIALCORE_CREATE_USER_STEP].is_active() + return waffle().flag_is_active( + request, waffle_flags()[DISABLE_TPA_PIPELINE_SOCIALCORE_CREATE_USER_STEP] + ) From c94b450cb2370c89a2921951f6a52f611232bdb5 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Fri, 3 Mar 2023 16:41:03 -0800 Subject: [PATCH 061/125] Fix is_flag_active call to check create_user TPA step --- openedx/core/djangoapps/appsembler/waffle.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/waffle.py b/openedx/core/djangoapps/appsembler/waffle.py index b468b54ca6f4..a67fa0fd82c4 100644 --- a/openedx/core/djangoapps/appsembler/waffle.py +++ b/openedx/core/djangoapps/appsembler/waffle.py @@ -44,6 +44,4 @@ def disable_tpa_create_user_step(request): even cause issues. Using a Waffle Flag to be able to activate selectively for production testing. """ - return waffle().flag_is_active( - request, waffle_flags()[DISABLE_TPA_PIPELINE_SOCIALCORE_CREATE_USER_STEP] - ) + return waffle().is_flag_active(DISABLE_TPA_PIPELINE_SOCIALCORE_CREATE_USER_STEP) From 96036d8a0d89969a68cef6e078e837f2505450da Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Sun, 5 Mar 2023 09:37:08 -0800 Subject: [PATCH 062/125] remove Anders, Maxi, Omar, Shadi... add Amir, Sheridan --- .github/CODEOWNERS | 2 +- .github/workflows/push.yml | 4 ++-- .github/workflows/sync_prod_with_main.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f8104b54842f..68beeec8448c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -6,4 +6,4 @@ * @bryanlandia # default set of reviewers -* @melvinsoft @xscrio +* @amirtds @xscrio diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index ccd1c9b49c28..cbad39a52923 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -18,7 +18,7 @@ jobs: PULL_REQUEST_BODY: | This is an automated pull request from branch `hawthorn/main` into `hawthorn/prod` (production). Please review the changes and merge this pull request _before_ running the Tahoe production Cloud Build deployment. - PULL_REQUEST_REVIEWERS: "johnbaldwin amirtds bryanlandia" + PULL_REQUEST_REVIEWERS: "amirtds bryanlandia xscrio" hawthorn-to-juniper-sync: name: PullRequestAction runs-on: ubuntu-latest @@ -35,5 +35,5 @@ jobs: This is meant for making sure all of our Hawthorn changes gets merge into Juniper otherwise Juniper would stall. If tests passes merge this pull request. If there are merge conflicts, it needs to be resolved manually in a seperate pull request. - PULL_REQUEST_REVIEWERS: "melvinsoft shadinaif OmarIthawi thraxil" + PULL_REQUEST_REVIEWERS: "bryanlandia amirtds xscrio" diff --git a/.github/workflows/sync_prod_with_main.yml b/.github/workflows/sync_prod_with_main.yml index 4927eb46f65d..eab2bbcee3df 100644 --- a/.github/workflows/sync_prod_with_main.yml +++ b/.github/workflows/sync_prod_with_main.yml @@ -15,4 +15,4 @@ jobs: BRANCH_PREFIX: "main" PULL_REQUEST_BRANCH: "prod" PULL_REQUEST_TITLE: "Update from `main` (production)" - PULL_REQUEST_REVIEWERS: "melvinsoft OmarIthawi thraxil shadinaif" + PULL_REQUEST_REVIEWERS: "xscrio amirtds bryanlandia" From 7c8f90ce75918017e1342b8bfd1a4a9210f94ed7 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Sun, 5 Mar 2023 10:11:56 -0800 Subject: [PATCH 063/125] reset factory sequence to fix random failures in tahoeusermetadata processor tests --- .../appsembler/eventtracking/tests/test_tahoeusermetadata.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py b/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py index 36c9ce0edf71..af3a178c8caf 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tests/test_tahoeusermetadata.py @@ -53,6 +53,7 @@ class UserWithTahoeMetadataFactory(UserFactory): @pytest.fixture(autouse=True) def users(): + UserProfileFactory.reset_sequence(0) return [UserWithTahoeMetadataFactory() for i in range(2)] From fbd6d0fcf860c1112d5371124ac48c1e13e9c504 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Wed, 15 Mar 2023 19:04:08 -0700 Subject: [PATCH 064/125] Send full event to utils.get_user_id_from_event After changing that method to do a full pass through the event, preferring inner user_id to avoid using request user (instructor, API user, etc.) for id from which to retrieve tahoe user metadata. Fixes RED-3716 --- .../djangoapps/appsembler/eventtracking/tahoeusermetadata.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py index 2e3e9d6aa56a..3686380d912f 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py @@ -183,14 +183,13 @@ def __call__(self, event): # a bulk enrollment or exception certificate triggering the event. Only use the # user from event itself. - event_data = event.get('data') try: - user_id = utils.get_user_id_from_event(event_data) + user_id = utils.get_user_id_from_event(event) except AttributeError: logger.warning( "TahoeUserMetadataProcessor passed invalid type to " "get_user_id_from_event: {}. Likely innocuous. " - "Logging and continuing.".format(event_data) + "Logging and continuing.".format(event) ) else: if user_id: From d84f905f128031b783c9a261f2f39b868b91f28b Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Wed, 15 Mar 2023 19:05:03 -0700 Subject: [PATCH 065/125] less noisy logs for events with invalid event types --- .../djangoapps/appsembler/eventtracking/tahoeusermetadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py index 3686380d912f..e87d3f2fd348 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py @@ -186,7 +186,7 @@ def __call__(self, event): try: user_id = utils.get_user_id_from_event(event) except AttributeError: - logger.warning( + logger.debug( "TahoeUserMetadataProcessor passed invalid type to " "get_user_id_from_event: {}. Likely innocuous. " "Logging and continuing.".format(event) From b494bc3f29b746abd3c9d9bf272d415e2d00a40c Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Thu, 16 Mar 2023 18:31:42 -0700 Subject: [PATCH 066/125] If Tiers app not enabled return all orgs as active --- openedx/core/djangoapps/appsembler/sites/utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/sites/utils.py b/openedx/core/djangoapps/appsembler/sites/utils.py index 308c9e30f63b..cef2bdce7755 100644 --- a/openedx/core/djangoapps/appsembler/sites/utils.py +++ b/openedx/core/djangoapps/appsembler/sites/utils.py @@ -88,9 +88,11 @@ def get_active_organizations(): TODO: This helper should live in a future Tahoe Sites package. """ - active_tiers_uuids = get_active_organizations_uuids() - - return get_organizations_from_uuids(uuids=active_tiers_uuids) + if settings.FEATURES.get('ENABLE_TIERS_APP', False): + active_tiers_uuids = get_active_organizations_uuids() + return get_organizations_from_uuids(uuids=active_tiers_uuids) + else: + return Organization.objects.all() def get_active_sites(order_by='domain'): From b01bfa91c71ea42cd61ff1d181f5ef11b63e5374 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Fri, 17 Mar 2023 18:10:42 +0000 Subject: [PATCH 067/125] Check TPA pipeline for tahoe idp metadata if before added to UserProfile --- .../eventtracking/tahoeusermetadata.py | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py index e87d3f2fd348..1e04453f06cc 100644 --- a/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py +++ b/openedx/core/djangoapps/appsembler/eventtracking/tahoeusermetadata.py @@ -118,6 +118,21 @@ def _get_reg_metadata_from_cache(self, user_id): else: return {} + def _get_idp_metadata_from_tpa_pipeline(self): + """Check ThirdPartyAuth pipeline for details containing IdP metadata.""" + import crum + from third_party_auth import pipeline + try: + request = crum.get_current_request() + tpa_running = pipeline.running(request) + if not tpa_running: + return None + tpa = pipeline.get(request) + details = tpa['kwargs'].get('details') + return details.get('tahoe_idp_metadata', {}) + except: + return None + def _get_custom_registration_metadata(self, user_id): """ Get any custom registration field data for the User. @@ -136,10 +151,18 @@ def _get_custom_registration_metadata(self, user_id): try: profile = UserProfile.objects.get(user__id=user_id) except UserProfile.DoesNotExist: - logger.info("User {user_id} has no UserProfile".format(user_id=user_id)) return {} - meta = profile.get_meta() - idp_metadata = meta.get("tahoe_idp_metadata", {}) + else: + meta = profile.get_meta() + idp_metadata = meta.get("tahoe_idp_metadata", {}) + if not idp_metadata: + logger.info("User {user_id} has no IDP metadata yet".format(user_id=user_id)) + # We could be processing an event (e.g., course enrollment) on very first User.save() + # This can happen before UserProfile has been updated via tahoe_idp TPA step. + # Try getting it direclty from TPA pipeline. + idp_metadata = self._get_idp_metadata_from_tpa_pipeline() + if not idp_metadata: + return {} custom_reg_data = idp_metadata.get("registration_additional") userprofile_metadata_cache.set_by_user_id(user_id, idp_metadata) return custom_reg_data From 483a91d8c617db1f01337c433ee2ff7199a2bbf1 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Mon, 3 Apr 2023 11:44:11 -0700 Subject: [PATCH 068/125] Ensure samesite=none logged_in cookies are deleted Fixes ENG-53 Patch delete_logged_in_cookies to bypass Django < 3.2.7 (Maple+) delete_cookie. It relies on prefix of __SECURE to determine secure cookies, so we have to use set_cookie with an expired date. Note this will still work with Session cookie domain middleware. Taken from https://discuss.overhang.io/t/logged-in-cookies-not-deleted-on-logout-over-https-not-reproducible-on-edx-org/1011/6 --- openedx/core/djangoapps/user_authn/cookies.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/openedx/core/djangoapps/user_authn/cookies.py b/openedx/core/djangoapps/user_authn/cookies.py index 159084b7ee04..ec880504f705 100644 --- a/openedx/core/djangoapps/user_authn/cookies.py +++ b/openedx/core/djangoapps/user_authn/cookies.py @@ -3,6 +3,7 @@ """ +import datetime import json import logging import time @@ -73,10 +74,14 @@ def delete_logged_in_cookies(response): HttpResponse """ for cookie_name in ALL_LOGGED_IN_COOKIE_NAMES: - response.delete_cookie( + response.set_cookie( cookie_name, + '', path='/', - domain=settings.SESSION_COOKIE_DOMAIN + domain=settings.SESSION_COOKIE_DOMAIN, + secure=True if settings.HTTPS == 'on' else False, + max_age=0, + expires=datetime.datetime.utcfromtimestamp(0) ) return response From 3ddb7448d5d63ea3bfb16c9f8356494d2df9f4a2 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Tue, 4 Apr 2023 15:47:15 -0700 Subject: [PATCH 069/125] match func args order from HTTPResponse.set_cookie --- openedx/core/djangoapps/user_authn/cookies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openedx/core/djangoapps/user_authn/cookies.py b/openedx/core/djangoapps/user_authn/cookies.py index ec880504f705..1d97ee4351bd 100644 --- a/openedx/core/djangoapps/user_authn/cookies.py +++ b/openedx/core/djangoapps/user_authn/cookies.py @@ -77,11 +77,11 @@ def delete_logged_in_cookies(response): response.set_cookie( cookie_name, '', + max_age=0, + expires=datetime.datetime.utcfromtimestamp(0), path='/', domain=settings.SESSION_COOKIE_DOMAIN, - secure=True if settings.HTTPS == 'on' else False, - max_age=0, - expires=datetime.datetime.utcfromtimestamp(0) + secure=True if settings.HTTPS == 'on' else False ) return response From e07eade524bafd03a0b01f30aec538af91cdd049 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Tue, 4 Apr 2023 18:35:54 -0700 Subject: [PATCH 070/125] Set explicit 1970 string for delete_cookie expire' --- openedx/core/djangoapps/user_authn/cookies.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openedx/core/djangoapps/user_authn/cookies.py b/openedx/core/djangoapps/user_authn/cookies.py index 1d97ee4351bd..76572284a256 100644 --- a/openedx/core/djangoapps/user_authn/cookies.py +++ b/openedx/core/djangoapps/user_authn/cookies.py @@ -3,7 +3,6 @@ """ -import datetime import json import logging import time @@ -78,7 +77,7 @@ def delete_logged_in_cookies(response): cookie_name, '', max_age=0, - expires=datetime.datetime.utcfromtimestamp(0), + expires='Thu, 01 Jan 1970 00:00:00 GMT', path='/', domain=settings.SESSION_COOKIE_DOMAIN, secure=True if settings.HTTPS == 'on' else False From 7961372ea590dc95f2d7197ceb021fcd0ba2405b Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Fri, 7 Apr 2023 16:37:07 -0700 Subject: [PATCH 071/125] Fix unenroll_by_email for multitenant email settings Fix ENG-57 --- common/djangoapps/student/models.py | 5 +++- .../tests/test_enroll_by_email.py | 25 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/common/djangoapps/student/models.py b/common/djangoapps/student/models.py index 37b208311ae2..9db611db6d7c 100644 --- a/common/djangoapps/student/models.py +++ b/common/djangoapps/student/models.py @@ -1656,7 +1656,10 @@ def unenroll_by_email(cls, email, course_id): RequestCache('get_enrollment').clear() try: - user = User.objects.get(email=email) + if settings.FEATURES.get('APPSEMBLER_MULTI_TENANT_EMAILS', False): + user = CourseEnrollment.get_user_by_email_within_organization(email) + else: + user = User.objects.get(email=email) return cls.unenroll(user, course_id) except User.DoesNotExist: log.error( diff --git a/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_enroll_by_email.py b/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_enroll_by_email.py index 5116b34fe985..3ec6ac0d7a2f 100644 --- a/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_enroll_by_email.py +++ b/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_enroll_by_email.py @@ -12,6 +12,8 @@ create_org_user, with_organization_context, ) +from student.tests.factories import CourseEnrollmentFactory + User = get_user_model() @@ -58,3 +60,26 @@ def test_enroll_by_email_multi_tenant(settings): assert not CourseEnrollment.enroll_by_email(blue_user.email, course_key), 'Should not enroll in other sites' assert not CourseEnrollment.is_enrolled(blue_user, course_key), 'Should not enroll in other sites' + + +@pytest.mark.django_db +def test_unenroll_by_email_multi_tenant(settings): + """ + Ensure `unenroll_by_email` works with APPSEMBLER_MULTI_TENANT_EMAILS is enabled. + """ + settings.FEATURES = {**settings.FEATURES, 'APPSEMBLER_MULTI_TENANT_EMAILS': True} + course = CourseOverviewFactory.create() + course_key = course.id + + with with_organization_context(site_color='blue1') as blue_org: + blue_user = create_org_user(blue_org) + CourseEnrollmentFactory(user=blue_user, course_id=course_key) + CourseEnrollment.enroll_by_email(blue_user.email, course_key) + + with with_organization_context(site_color='red1') as red_org: + red_user = create_org_user(red_org) + CourseEnrollmentFactory(user=red_user, course_id=course_key) + assert CourseEnrollment.unenroll_by_email(red_user.email, course_key), 'Should be able to unenroll in same site' + assert not CourseEnrollment.is_enrolled(red_user, course_key) + + assert CourseEnrollment.is_enrolled(blue_user, course_key), 'Should not unenroll in other sites' From 80957043e3ec8837b7ffde7512e5c6b9342f2024 Mon Sep 17 00:00:00 2001 From: Amir Tadrisi Date: Tue, 11 Apr 2023 20:59:36 -0500 Subject: [PATCH 072/125] fix: sanitize redirect_url parameter for logout --- openedx/core/djangoapps/user_authn/views/logout.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/user_authn/views/logout.py b/openedx/core/djangoapps/user_authn/views/logout.py index decf10928355..d2c8f7127f26 100644 --- a/openedx/core/djangoapps/user_authn/views/logout.py +++ b/openedx/core/djangoapps/user_authn/views/logout.py @@ -2,6 +2,7 @@ import re +import bleach import six.moves.urllib.parse as parse # pylint: disable=import-error from django.conf import settings @@ -60,7 +61,9 @@ def target(self): # >> /courses/course-v1:ARTS+D1+2018_T/course/ # to handle this scenario we need to encode our URL using quote_plus and then unquote it again. if target_url: - target_url = parse.unquote(parse.quote_plus(target_url)) + target_url = bleach.clean( + parse.unquote(parse.quote_plus(target_url)) + ) use_target_url = target_url and is_safe_login_or_logout_redirect( redirect_to=target_url, From 23b4b49e346c694541bc7b9b1050bc05f5f2c314 Mon Sep 17 00:00:00 2001 From: Amir Tadrisi Date: Tue, 11 Apr 2023 21:06:28 -0500 Subject: [PATCH 073/125] chore: add test for the logout redirect_url sanitization --- .../user_authn/views/tests/test_logout.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/openedx/core/djangoapps/user_authn/views/tests/test_logout.py b/openedx/core/djangoapps/user_authn/views/tests/test_logout.py index 2ba360ab8996..40bb34a2f5d5 100644 --- a/openedx/core/djangoapps/user_authn/views/tests/test_logout.py +++ b/openedx/core/djangoapps/user_authn/views/tests/test_logout.py @@ -8,6 +8,8 @@ import ddt import mock import six +import bleach +import urllib from django.conf import settings from django.test import TestCase from django.test.utils import override_settings @@ -194,3 +196,21 @@ def test_learner_portal_logout_having_idp_logout_url(self): 'show_tpa_logout_link': True, } self.assertDictContainsSubset(expected, response.context_data) + + @ddt.data( + ('%22%3E%3Cscript%3Ealert(%27xss%27)%3C/script%3E', 'edx.org'), + ) + @ddt.unpack + def test_logout_redirect_failure_with_xss_vulnerability(self, redirect_url, host): + """ + Verify that it will block the XSS attack on edX’s LMS logout page + """ + url = '{logout_path}?redirect_url={redirect_url}'.format( + logout_path=reverse('logout'), + redirect_url=redirect_url + ) + response = self.client.get(url, HTTP_HOST=host) + expected = { + 'target': bleach.clean(urllib.parse.unquote(redirect_url)), + } + self.assertDictContainsSubset(expected, response.context_data) From 10eea10a06ca7cdb1161138862590edad7cc4406 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Thu, 13 Apr 2023 09:53:55 -0700 Subject: [PATCH 074/125] Fix mte unenrollment_by_email test --- .../multi_tenant_emails/tests/test_enroll_by_email.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_enroll_by_email.py b/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_enroll_by_email.py index 3ec6ac0d7a2f..6c977e5eaff8 100644 --- a/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_enroll_by_email.py +++ b/openedx/core/djangoapps/appsembler/multi_tenant_emails/tests/test_enroll_by_email.py @@ -79,7 +79,8 @@ def test_unenroll_by_email_multi_tenant(settings): with with_organization_context(site_color='red1') as red_org: red_user = create_org_user(red_org) CourseEnrollmentFactory(user=red_user, course_id=course_key) - assert CourseEnrollment.unenroll_by_email(red_user.email, course_key), 'Should be able to unenroll in same site' - assert not CourseEnrollment.is_enrolled(red_user, course_key) + CourseEnrollment.unenroll_by_email(red_user.email, course_key) + assert not CourseEnrollment.is_enrolled(red_user, course_key), 'Should unenroll in same sites' + CourseEnrollment.unenroll_by_email(blue_user.email, course_key) assert CourseEnrollment.is_enrolled(blue_user, course_key), 'Should not unenroll in other sites' From 1837bb5327a35d557adfb75aeb1d6bf91462fe95 Mon Sep 17 00:00:00 2001 From: Amir Tadrisi Date: Fri, 14 Apr 2023 17:31:47 -0500 Subject: [PATCH 075/125] chore: add valid_url_pattern for logout --- openedx/core/djangoapps/user_authn/views/logout.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openedx/core/djangoapps/user_authn/views/logout.py b/openedx/core/djangoapps/user_authn/views/logout.py index d2c8f7127f26..0938be446b1d 100644 --- a/openedx/core/djangoapps/user_authn/views/logout.py +++ b/openedx/core/djangoapps/user_authn/views/logout.py @@ -64,6 +64,11 @@ def target(self): target_url = bleach.clean( parse.unquote(parse.quote_plus(target_url)) ) + # Check if the target_url starts with a valid protocol (http or https) + valid_url_pattern = re.compile(r'^(http|https)://', re.IGNORECASE) + if not valid_url_pattern.match(target_url): + # If the target_url doesn't start with a valid protocol, either use a default URL or raise an error + target_url = self.default_target use_target_url = target_url and is_safe_login_or_logout_redirect( redirect_to=target_url, From e0b4ba1d5401241efd3e57f1daadaa6d291b15bb Mon Sep 17 00:00:00 2001 From: Amir Tadrisi Date: Fri, 14 Apr 2023 17:52:25 -0500 Subject: [PATCH 076/125] chore: cover relative redirect urls for logout --- openedx/core/djangoapps/user_authn/views/logout.py | 10 ++++++---- .../djangoapps/user_authn/views/tests/test_logout.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/openedx/core/djangoapps/user_authn/views/logout.py b/openedx/core/djangoapps/user_authn/views/logout.py index 0938be446b1d..b24d026775c0 100644 --- a/openedx/core/djangoapps/user_authn/views/logout.py +++ b/openedx/core/djangoapps/user_authn/views/logout.py @@ -3,6 +3,7 @@ import re import bleach +from urllib.parse import urlparse import six.moves.urllib.parse as parse # pylint: disable=import-error from django.conf import settings @@ -16,7 +17,6 @@ from openedx.core.djangoapps.user_authn.utils import is_safe_login_or_logout_redirect from third_party_auth import pipeline as tpa_pipeline - from openedx.core.djangoapps.appsembler.tahoe_idp import helpers as tahoe_idp_helpers @@ -64,10 +64,12 @@ def target(self): target_url = bleach.clean( parse.unquote(parse.quote_plus(target_url)) ) - # Check if the target_url starts with a valid protocol (http or https) + parsed_url = urlparse(target_url) valid_url_pattern = re.compile(r'^(http|https)://', re.IGNORECASE) - if not valid_url_pattern.match(target_url): - # If the target_url doesn't start with a valid protocol, either use a default URL or raise an error + + # Allow URLs starting with http or https, as well as relative URLs + if parsed_url.scheme not in ('http', 'https') and not valid_url_pattern.match(target_url) and not parsed_url.path.startswith('/'): + # If the target_url doesn't start with a valid protocol or is not a relative URL, either use a default URL or raise an error target_url = self.default_target use_target_url = target_url and is_safe_login_or_logout_redirect( diff --git a/openedx/core/djangoapps/user_authn/views/tests/test_logout.py b/openedx/core/djangoapps/user_authn/views/tests/test_logout.py index 40bb34a2f5d5..d71830ce2b82 100644 --- a/openedx/core/djangoapps/user_authn/views/tests/test_logout.py +++ b/openedx/core/djangoapps/user_authn/views/tests/test_logout.py @@ -211,6 +211,6 @@ def test_logout_redirect_failure_with_xss_vulnerability(self, redirect_url, host ) response = self.client.get(url, HTTP_HOST=host) expected = { - 'target': bleach.clean(urllib.parse.unquote(redirect_url)), + 'target': '/', } self.assertDictContainsSubset(expected, response.context_data) From b45960cfbf413e57be9cd3e865e21dc026a5be46 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Wed, 19 Apr 2023 13:54:57 -0700 Subject: [PATCH 077/125] Bump tahoe-idp to 2.3.0 --- requirements/edx/appsembler.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/edx/appsembler.txt b/requirements/edx/appsembler.txt index e23b579f6b8e..acbbe794624c 100644 --- a/requirements/edx/appsembler.txt +++ b/requirements/edx/appsembler.txt @@ -23,7 +23,7 @@ https://github.com/appsembler/edx-proctoring/archive/v2.4.0-appsembler1.tar.gz django-tiers==0.2.7 fusionauth-client==1.36.0 google-cloud-storage==1.32.0 -tahoe-idp==2.2.0 +tahoe-idp==2.3.0 tahoe-sites==1.3.2 tahoe-lti==0.3.0 site-configuration-client==0.2.3 From 3b40578737f3de3c3a0fed4351e150f0eac2cfb3 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Mon, 15 May 2023 11:30:06 -0700 Subject: [PATCH 078/125] Remove reference to Harvard, Wharton, etc. in account deletion Trying not to override all of account_settings.html Because of multitenancy we would otherwise have to override in every single SiteConfiguration --- lms/templates/student_account/account_settings.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/templates/student_account/account_settings.html b/lms/templates/student_account/account_settings.html index 83bb822327b3..9a1446d93c83 100644 --- a/lms/templates/student_account/account_settings.html +++ b/lms/templates/student_account/account_settings.html @@ -77,7 +77,7 @@ + return resource; + } + const { fetch: originalFetch } = window; + window.fetch = async (...args) => { + let [resource, config ] = args; + resource = replaceFetchResourceForSegmentSite(resource); + const response = await originalFetch(resource, config); + return response; + }; + }(); + ## Appsembler: end Segment Site ## end Copy % endif From c3da2d92ee1d15b73d4941f625e5741670021902 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Tue, 29 Aug 2023 12:34:39 -0700 Subject: [PATCH 095/125] Update messageId format for faked analytics.js event call --- common/djangoapps/track/views/segmentio.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/djangoapps/track/views/segmentio.py b/common/djangoapps/track/views/segmentio.py index 82aa2923752e..a61c5fdf5a5e 100644 --- a/common/djangoapps/track/views/segmentio.py +++ b/common/djangoapps/track/views/segmentio.py @@ -289,8 +289,8 @@ def send_event(request, method, **params): segment_key = helpers.get_current_site_configuration().get_secret_value('SEGMENT_KEY') if segment_key: data['writeKey'] = segment_key - data['messageId'] = 'ajs-' + uuid.uuid4().hex - site_response = requests.post(url, json=data) + data['messageId'] = 'ajs-next-' + uuid.uuid4().hex + site_response = requests.post(url, json=data) # noqa: F841 return HttpResponse( main_response.content, status=main_response.status_code, From edadd6d5e619eed25eec0eebfb6bc951c9fe247a Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Tue, 29 Aug 2023 15:20:12 -0700 Subject: [PATCH 096/125] Use edx-sga 0.22.0 via appsembler.txt reqs --- requirements/edx/appsembler.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/edx/appsembler.txt b/requirements/edx/appsembler.txt index 701427816a91..197d2a6ceb22 100644 --- a/requirements/edx/appsembler.txt +++ b/requirements/edx/appsembler.txt @@ -14,9 +14,9 @@ django-hijack-admin==2.1.10 honeycomb-beeline==2.12.1 # Patched upstream packages +https://github.com/mitodl/edx-sga/archive/refs/tags/v0.22.0.tar.gz https://github.com/edx-solutions/xblock-google-drive/archive/589d9f51f9b.tar.gz # v0.2.0 but the repo has no tags https://github.com/appsembler/edx-ora2/archive/2.7.6-appsembler.1.tar.gz -https://github.com/appsembler/edx-sga/archive/v0.11.0.appsembler2.tar.gz https://github.com/appsembler/edx-proctoring/archive/v2.4.0-appsembler1.tar.gz # Tahoe plugins and customizations From 8a4dfa4ec5911be16a6668d45e0cadc8d817784f Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Wed, 13 Sep 2023 15:21:02 -0700 Subject: [PATCH 097/125] wrap most of the onYouTubeIframeAPIReady function in try/catch ENG-97 Try to fix continuing cases of window.onYouTubeIframeAPIReady.resolve is not a function and retry the full function --- .../xmodule/js/src/video/01_initialize.js | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/common/lib/xmodule/xmodule/js/src/video/01_initialize.js b/common/lib/xmodule/xmodule/js/src/video/01_initialize.js index 1ff8b591b7f4..92c79c64ef3d 100644 --- a/common/lib/xmodule/xmodule/js/src/video/01_initialize.js +++ b/common/lib/xmodule/xmodule/js/src/video/01_initialize.js @@ -174,14 +174,19 @@ function(VideoPlayer, i18n, moment, _) { throw new Error('Too many OnYouTubeIframeAPIReady retries after TypeError...giving up.'); } - _oldOnYouTubeIframeAPIReady = window.onYouTubeIframeAPIReady || undefined; + try { + _oldOnYouTubeIframeAPIReady = window.onYouTubeIframeAPIReady || undefined; - window.onYouTubeIframeAPIReady = function() { - window.onYouTubeIframeAPIReady.resolve(); - }; + window.onYouTubeIframeAPIReady = function() { + window.onYouTubeIframeAPIReady.resolve(); + }; - try { window.onYouTubeIframeAPIReady.resolve = _youtubeApiDeferred.resolve; + window.onYouTubeIframeAPIReady.done = _youtubeApiDeferred.done; + + if (_oldOnYouTubeIframeAPIReady) { + window.onYouTubeIframeAPIReady.done(_oldOnYouTubeIframeAPIReady); + } } catch (e) { console.error('Error while trying to resolve the Deferred object responsible for calling OnYouTubeIframeAPIReady callbacks.'); console.error('window.onYouTubeIframeAPIReady is ' + window.onYouTubeIframeAPIReady); @@ -193,11 +198,7 @@ function(VideoPlayer, i18n, moment, _) { throw e; } } - window.onYouTubeIframeAPIReady.done = _youtubeApiDeferred.done; - if (_oldOnYouTubeIframeAPIReady) { - window.onYouTubeIframeAPIReady.done(_oldOnYouTubeIframeAPIReady); - } }; // If a Deferred object hasn't been created yet, create one now. It will From d5e0929e13dc3a235484119aa995d588434e266d Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Wed, 13 Sep 2023 15:21:33 -0700 Subject: [PATCH 098/125] Allow up to 5 retries on onYouTubeIframeAPIReady func calls --- common/lib/xmodule/xmodule/js/src/video/01_initialize.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/lib/xmodule/xmodule/js/src/video/01_initialize.js b/common/lib/xmodule/xmodule/js/src/video/01_initialize.js index 92c79c64ef3d..c42bb772b980 100644 --- a/common/lib/xmodule/xmodule/js/src/video/01_initialize.js +++ b/common/lib/xmodule/xmodule/js/src/video/01_initialize.js @@ -90,7 +90,7 @@ function(VideoPlayer, i18n, moment, _) { _youtubeApiDeferred = null, _oldOnYouTubeIframeAPIReady; - const setupOnYouTubeIframeAPIReadyMaxCalls=3; + const setupOnYouTubeIframeAPIReadyMaxCalls=5; Initialize.prototype = methodsDict; From 6bb24a6e8f8475911bd9bd40c53f36acc4613bd5 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Wed, 13 Sep 2023 15:25:04 -0700 Subject: [PATCH 099/125] Add a 500ms delay before retrying onYouTubeIframeAPIReady --- common/lib/xmodule/xmodule/js/src/video/01_initialize.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/lib/xmodule/xmodule/js/src/video/01_initialize.js b/common/lib/xmodule/xmodule/js/src/video/01_initialize.js index c42bb772b980..c43ab4b1693e 100644 --- a/common/lib/xmodule/xmodule/js/src/video/01_initialize.js +++ b/common/lib/xmodule/xmodule/js/src/video/01_initialize.js @@ -192,7 +192,7 @@ function(VideoPlayer, i18n, moment, _) { console.error('window.onYouTubeIframeAPIReady is ' + window.onYouTubeIframeAPIReady); console.error(e); if (e instanceof TypeError) { - setupOnYouTubeIframeAPIReady(); // Try again up to defined max calls. + setTimeout(setupOnYouTubeIframeAPIReady, 500); // Try again up to defined max calls. } else { throw e; From 46a894d25b1b22c574f5126692eeffc98c90ecf6 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Thu, 14 Sep 2023 15:37:35 -0700 Subject: [PATCH 100/125] Refactor try/catch on window.onYouTubeIframeAPIReady.resolve Rename Error instance as seems to have caused a name collision when minified --- .../xmodule/js/src/video/01_initialize.js | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/common/lib/xmodule/xmodule/js/src/video/01_initialize.js b/common/lib/xmodule/xmodule/js/src/video/01_initialize.js index c43ab4b1693e..f386c5bb1ed5 100644 --- a/common/lib/xmodule/xmodule/js/src/video/01_initialize.js +++ b/common/lib/xmodule/xmodule/js/src/video/01_initialize.js @@ -174,11 +174,27 @@ function(VideoPlayer, i18n, moment, _) { throw new Error('Too many OnYouTubeIframeAPIReady retries after TypeError...giving up.'); } + handleYTAPIErr = function(ytapierr) { + console.error('Error while trying to resolve the Deferred object responsible for calling OnYouTubeIframeAPIReady callbacks.'); + console.debug('window.onYouTubeIframeAPIReady is ' + window.onYouTubeIframeAPIReady); + console.error(ytapierr); + if (ytapierr instanceof TypeError) { // expecting TypeError: window.onYouTubeIframeAPIReady.resolve is not a function + setTimeout(setupOnYouTubeIframeAPIReady, 500); // Try again up to defined max calls. + } + else { + throw ytapierr; + } + }; + try { _oldOnYouTubeIframeAPIReady = window.onYouTubeIframeAPIReady || undefined; window.onYouTubeIframeAPIReady = function() { - window.onYouTubeIframeAPIReady.resolve(); + try { // this additional inner try/catch shouldn't really be needed but it's here just in case. + window.onYouTubeIframeAPIReady.resolve(); + } catch (ytapiresolveerr) { + handleYTAPIErr(ytapiresolveerr); + } }; window.onYouTubeIframeAPIReady.resolve = _youtubeApiDeferred.resolve; @@ -187,16 +203,8 @@ function(VideoPlayer, i18n, moment, _) { if (_oldOnYouTubeIframeAPIReady) { window.onYouTubeIframeAPIReady.done(_oldOnYouTubeIframeAPIReady); } - } catch (e) { - console.error('Error while trying to resolve the Deferred object responsible for calling OnYouTubeIframeAPIReady callbacks.'); - console.error('window.onYouTubeIframeAPIReady is ' + window.onYouTubeIframeAPIReady); - console.error(e); - if (e instanceof TypeError) { - setTimeout(setupOnYouTubeIframeAPIReady, 500); // Try again up to defined max calls. - } - else { - throw e; - } + } catch (ytapierr) { + handleYTAPIErr(ytapierr); } }; From 10f8d4887db08ef4881e78636337477a003f4d2b Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Fri, 15 Sep 2023 14:37:23 -0700 Subject: [PATCH 101/125] Revert to edx-sga v0.11.0 Appsembler fork --- requirements/edx/appsembler.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/edx/appsembler.txt b/requirements/edx/appsembler.txt index 197d2a6ceb22..063e1b5221d9 100644 --- a/requirements/edx/appsembler.txt +++ b/requirements/edx/appsembler.txt @@ -14,7 +14,7 @@ django-hijack-admin==2.1.10 honeycomb-beeline==2.12.1 # Patched upstream packages -https://github.com/mitodl/edx-sga/archive/refs/tags/v0.22.0.tar.gz +https://github.com/appsembler/edx-sga/archive/v0.11.0.appsembler2.tar.gz https://github.com/edx-solutions/xblock-google-drive/archive/589d9f51f9b.tar.gz # v0.2.0 but the repo has no tags https://github.com/appsembler/edx-ora2/archive/2.7.6-appsembler.1.tar.gz https://github.com/appsembler/edx-proctoring/archive/v2.4.0-appsembler1.tar.gz From d1f19b3351a665d57528345152041c64ce1fac01 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Fri, 15 Sep 2023 14:44:04 -0700 Subject: [PATCH 102/125] Upgrade edx-sga to v0.12.0 --- requirements/edx/appsembler.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/edx/appsembler.txt b/requirements/edx/appsembler.txt index 197d2a6ceb22..99d255d34214 100644 --- a/requirements/edx/appsembler.txt +++ b/requirements/edx/appsembler.txt @@ -14,7 +14,7 @@ django-hijack-admin==2.1.10 honeycomb-beeline==2.12.1 # Patched upstream packages -https://github.com/mitodl/edx-sga/archive/refs/tags/v0.22.0.tar.gz +https://github.com/mitodl/edx-sga/archive/refs/tags/v0.12.0.tar.gz https://github.com/edx-solutions/xblock-google-drive/archive/589d9f51f9b.tar.gz # v0.2.0 but the repo has no tags https://github.com/appsembler/edx-ora2/archive/2.7.6-appsembler.1.tar.gz https://github.com/appsembler/edx-proctoring/archive/v2.4.0-appsembler1.tar.gz From a42d93fffb4a55d509cbbd05daf4650ac5f0df40 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Tue, 19 Sep 2023 15:21:24 -0700 Subject: [PATCH 103/125] Attempt at small rework of setupOnYouTubeIframeAPIReady and Deferred creation --- .../xmodule/js/src/video/01_initialize.js | 37 +++++++++---------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/common/lib/xmodule/xmodule/js/src/video/01_initialize.js b/common/lib/xmodule/xmodule/js/src/video/01_initialize.js index f386c5bb1ed5..255e621d91f2 100644 --- a/common/lib/xmodule/xmodule/js/src/video/01_initialize.js +++ b/common/lib/xmodule/xmodule/js/src/video/01_initialize.js @@ -164,12 +164,12 @@ function(VideoPlayer, i18n, moment, _) { // so that it resolves our Deferred object, which will call all of the // OnYouTubeIframeAPIReady callbacks. // - // If this global function is already defined, we store it first, and make - // sure that it gets executed when our Deferred object is resolved. let setupOnYouTubeIframeAPIReadyCallsCount = 0; - setupOnYouTubeIframeAPIReady = function() { + let setupOnYouTubeIframeAPIReady = function() { + setupOnYouTubeIframeAPIReadyCallsCount++; + if (setupOnYouTubeIframeAPIReadyCallsCount > setupOnYouTubeIframeAPIReadyMaxCalls) { throw new Error('Too many OnYouTubeIframeAPIReady retries after TypeError...giving up.'); } @@ -186,27 +186,23 @@ function(VideoPlayer, i18n, moment, _) { } }; - try { - _oldOnYouTubeIframeAPIReady = window.onYouTubeIframeAPIReady || undefined; + // If this global function is already defined, we store it first, and make + // sure that it gets executed when our Deferred object is resolved. + _oldOnYouTubeIframeAPIReady = window.onYouTubeIframeAPIReady || undefined; - window.onYouTubeIframeAPIReady = function() { - try { // this additional inner try/catch shouldn't really be needed but it's here just in case. - window.onYouTubeIframeAPIReady.resolve(); - } catch (ytapiresolveerr) { - handleYTAPIErr(ytapiresolveerr); - } - }; + window.onYouTubeIframeAPIReady = function() { + _youtubeApiDeferred.resolve(); + } - window.onYouTubeIframeAPIReady.resolve = _youtubeApiDeferred.resolve; - window.onYouTubeIframeAPIReady.done = _youtubeApiDeferred.done; + window.onYouTubeIframeAPIReady.resolve = _youtubeApiDeferred.resolve; + window.onYouTubeIframeAPIReady.done = _youtubeApiDeferred.done; + _youtubeApiDeferred.catch(function(err) { + handleYTAPIErr(err); + }); - if (_oldOnYouTubeIframeAPIReady) { - window.onYouTubeIframeAPIReady.done(_oldOnYouTubeIframeAPIReady); - } - } catch (ytapierr) { - handleYTAPIErr(ytapierr); + if (_oldOnYouTubeIframeAPIReady) { + window.onYouTubeIframeAPIReady.done(_oldOnYouTubeIframeAPIReady); } - }; // If a Deferred object hasn't been created yet, create one now. It will @@ -230,6 +226,7 @@ function(VideoPlayer, i18n, moment, _) { window.YT.ready(onYTApiReady); }); } + } else { video = VideoPlayer(state); From 88eb438ff29aae7e6534a376c93eea8c57963f57 Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Tue, 19 Sep 2023 16:56:12 -0700 Subject: [PATCH 104/125] Remove newer error handling code and try simple fix --- .../xmodule/js/src/video/01_initialize.js | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/common/lib/xmodule/xmodule/js/src/video/01_initialize.js b/common/lib/xmodule/xmodule/js/src/video/01_initialize.js index 255e621d91f2..98adaf6af182 100644 --- a/common/lib/xmodule/xmodule/js/src/video/01_initialize.js +++ b/common/lib/xmodule/xmodule/js/src/video/01_initialize.js @@ -164,28 +164,9 @@ function(VideoPlayer, i18n, moment, _) { // so that it resolves our Deferred object, which will call all of the // OnYouTubeIframeAPIReady callbacks. // - let setupOnYouTubeIframeAPIReadyCallsCount = 0; let setupOnYouTubeIframeAPIReady = function() { - setupOnYouTubeIframeAPIReadyCallsCount++; - - if (setupOnYouTubeIframeAPIReadyCallsCount > setupOnYouTubeIframeAPIReadyMaxCalls) { - throw new Error('Too many OnYouTubeIframeAPIReady retries after TypeError...giving up.'); - } - - handleYTAPIErr = function(ytapierr) { - console.error('Error while trying to resolve the Deferred object responsible for calling OnYouTubeIframeAPIReady callbacks.'); - console.debug('window.onYouTubeIframeAPIReady is ' + window.onYouTubeIframeAPIReady); - console.error(ytapierr); - if (ytapierr instanceof TypeError) { // expecting TypeError: window.onYouTubeIframeAPIReady.resolve is not a function - setTimeout(setupOnYouTubeIframeAPIReady, 500); // Try again up to defined max calls. - } - else { - throw ytapierr; - } - }; - // If this global function is already defined, we store it first, and make // sure that it gets executed when our Deferred object is resolved. _oldOnYouTubeIframeAPIReady = window.onYouTubeIframeAPIReady || undefined; @@ -196,9 +177,6 @@ function(VideoPlayer, i18n, moment, _) { window.onYouTubeIframeAPIReady.resolve = _youtubeApiDeferred.resolve; window.onYouTubeIframeAPIReady.done = _youtubeApiDeferred.done; - _youtubeApiDeferred.catch(function(err) { - handleYTAPIErr(err); - }); if (_oldOnYouTubeIframeAPIReady) { window.onYouTubeIframeAPIReady.done(_oldOnYouTubeIframeAPIReady); From e98d4aba59cb81744a7c3392f70b952f05d475df Mon Sep 17 00:00:00 2001 From: Amir Tadrisi Date: Tue, 14 Nov 2023 10:20:35 -0500 Subject: [PATCH 105/125] Fix: Handle Request objects in Segment fetch override --- lms/templates/widgets/segment-io.html | 45 +++++++++++++++++++++------ 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/lms/templates/widgets/segment-io.html b/lms/templates/widgets/segment-io.html index 5f8da23000e8..bfde404d9bec 100644 --- a/lms/templates/widgets/segment-io.html +++ b/lms/templates/widgets/segment-io.html @@ -9,18 +9,45 @@ !function(){ var originalAPI = '${settings.SEGMENT_ORIGINAL_API}'; var replicateAPI = '${settings.SEGMENT_REPLICATE_API}'; - function replaceFetchResourceForSegmentSite(resource){ - if (resource.substr(0, originalAPI.length) === originalAPI) { - resource = replicateAPI + resource.substr(originalAPI.length); + function replaceFetchResourceForSegmentSite(resource) { + // Helper function to replace the URL + function replaceUrl(url) { + if (url.substr(0, originalAPI.length) === originalAPI) { + return replicateAPI + url.substr(originalAPI.length); + } + return url; + } + + // Check if resource is a string (a URL) + if (typeof resource === 'string') { + return replaceUrl(resource); + } else if (resource instanceof Request) { + // If resource is a Request object, create a new Request with a replaced URL + const newUrl = replaceUrl(resource.url); + return new Request(newUrl, { + method: resource.method, + headers: resource.headers, + body: resource.body, + mode: resource.mode, + credentials: resource.credentials, + cache: resource.cache, + redirect: resource.redirect, + referrer: resource.referrer, + integrity: resource.integrity, + keepalive: resource.keepalive, + signal: resource.signal + }); + } else { + // If it's neither a string nor a Request object, log an error or handle as needed + console.error('replaceFetchResourceForSegmentSite was called with an unexpected argument type'); + return resource; } - return resource; } - const { fetch: originalFetch } = window; + // Override the fetch function to use the replaceFetchResourceForSegmentSite function + const originalFetch = window.fetch; window.fetch = async (...args) => { - let [resource, config ] = args; - resource = replaceFetchResourceForSegmentSite(resource); - const response = await originalFetch(resource, config); - return response; + args[0] = replaceFetchResourceForSegmentSite(args[0]); + return originalFetch.apply(window, args); }; }(); From 17ec2c0ffa97066784aa7ba9713e2c60f9ce9b95 Mon Sep 17 00:00:00 2001 From: Amir Tadrisi Date: Tue, 14 Nov 2023 16:38:37 -0500 Subject: [PATCH 106/125] chore: log the type of resource passed to replaceFetchResourceForSegmentSite in case of error --- lms/templates/widgets/segment-io.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/templates/widgets/segment-io.html b/lms/templates/widgets/segment-io.html index bfde404d9bec..e794d82cbf1f 100644 --- a/lms/templates/widgets/segment-io.html +++ b/lms/templates/widgets/segment-io.html @@ -39,7 +39,7 @@ }); } else { // If it's neither a string nor a Request object, log an error or handle as needed - console.error('replaceFetchResourceForSegmentSite was called with an unexpected argument type'); + console.error('replaceFetchResourceForSegmentSite was called with an unexpected argument type:', typeof resource, resource); return resource; } } From e5071ecb0f580b974b8f5ef6822dff84fda32574 Mon Sep 17 00:00:00 2001 From: Amir Tadrisi Date: Tue, 14 Nov 2023 16:39:24 -0500 Subject: [PATCH 107/125] chore: log warning instead of error for replaceFetchResourceForSegmentSite --- lms/templates/widgets/segment-io.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lms/templates/widgets/segment-io.html b/lms/templates/widgets/segment-io.html index e794d82cbf1f..ed3f549bfb7d 100644 --- a/lms/templates/widgets/segment-io.html +++ b/lms/templates/widgets/segment-io.html @@ -38,8 +38,8 @@ signal: resource.signal }); } else { - // If it's neither a string nor a Request object, log an error or handle as needed - console.error('replaceFetchResourceForSegmentSite was called with an unexpected argument type:', typeof resource, resource); + // If it's neither a string nor a Request object, log a warning or handle as needed + console.warn('replaceFetchResourceForSegmentSite was called with an unexpected argument type:', typeof resource, resource); return resource; } } From eab140aa57ea8f1dc1fa43b4029c3582b3d5bc63 Mon Sep 17 00:00:00 2001 From: Amir Tadrisi Date: Tue, 14 Nov 2023 16:50:42 -0500 Subject: [PATCH 108/125] chore: handle URL objects for replaceFetchResourceForSegmentSite --- lms/templates/widgets/segment-io.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lms/templates/widgets/segment-io.html b/lms/templates/widgets/segment-io.html index ed3f549bfb7d..49a9394acb44 100644 --- a/lms/templates/widgets/segment-io.html +++ b/lms/templates/widgets/segment-io.html @@ -37,6 +37,9 @@ keepalive: resource.keepalive, signal: resource.signal }); + } else if (resource instanceof URL) { + // If resource is a URL object, convert it to a string and replace the URL + return replaceUrl(resource.href); } else { // If it's neither a string nor a Request object, log a warning or handle as needed console.warn('replaceFetchResourceForSegmentSite was called with an unexpected argument type:', typeof resource, resource); From 1891f4b082e58f20a2a0761e84fa99d24b0b079b Mon Sep 17 00:00:00 2001 From: Amir Tadrisi Date: Thu, 16 Nov 2023 20:32:05 -0500 Subject: [PATCH 109/125] fix: Arabic translation placeholders in progress page --- conf/locale/ar/LC_MESSAGES/django.mo | Bin 816801 -> 816706 bytes conf/locale/ar/LC_MESSAGES/django.po | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/locale/ar/LC_MESSAGES/django.mo b/conf/locale/ar/LC_MESSAGES/django.mo index c05c00998def5cc65fb08e5cbb394f552940d987..a7316394786360e0f77685d883ebc9a449ed515e 100644 GIT binary patch delta 37659 zcmXZlb-Wa18^-Y=&hFXWLw6s#yZg}HAl*oJ=UKX>yAe>j;iaWfT0%e?Nu>lt5Kwr3 z*WCMupYPl=volZK&paC(SG>r7^m+cRXJ-aK#&Db@E5e*Sj^o5%73K`XvREAV;|%;8 zQ*hq+@4}q#@!RjioHy*Rye7;^MO=Jsm{S!StqXH*v47e6FsCH(unl3(PU77g!<2Jfafp^7B~b&=XJSU&7xE6~C60O6URMA+5qHPs_zRZ6{zvSFTd*AQ zNmNCH{-f3u8L%)rs$n-Bz?t|A*W-v|)^yp9hdJqqr=ps03&zK*s2V>(6)@=u`W#DO zF074u-7t)UOOc8Poega00(Vh;`wCNH+#kc7w3r?9V^vIuLopRjMcrUsX#WqGnD_#! zB_E->F6<|2P9Igo3dAi>hB^Pk75JI^JB?3S$-X)r<}BgBV2s2hXRK+9pbF9v`{Fpv zivQp?21(@EFef+hrSsMz&d*`aCgQB9f?q*(f%ig~GaSofTRenSxWAMBqBVIJR8w8S z_89Y0m{SZpVoGYT9BUKryBy{mK;JKR;~%ju@ms8pm9AI;W?&EEGFL7AHlkYY@HMgp zZ(;Be8wGz2a~5%f6xYKXKk?2RHcAhL{1H1|cFPLh05v8?qPlPm zM&eP-fDcev;JF>lRmW97eKX8BOHmtgKQ{&|3jUa`%#z^7pq}I zY=Ro~o3IETMzzfQkVzhgIXQ_7qH5Y2l@&o$foG#yWEZM{Z!ik|Pr{s57_7mDW}@{N ziN`QEJ`BYPf45Ow5Ou+?Lrz3>&4y6?D{63lLUm>JKWswkk1Eh+R0WQr3i1KzqM(!D zsWo9fjO0LT)R-8F1#liJjZTI9gwKd`KeIGU@!W1u4mCqI!fF_ey3R?|Sh<7caN!G^ z=x$gc47`xNu%&1HpJ9e=T95W-BIxbY!Sn8zC#Vx7_Y;e5m*At;2zZc@E(-~ z>E47nn{^x&|AB=t_Fr_l^1nD6YMMrwhMLPOQ=Ek3e|FH9FN-)QTQiu6V!QCV|bjOJj|q6o;Xjq$34GKc+lgV zW5*nJs0QtP9(M+uifW<-sOR|=T!rcV9;YgvK<$qk@VG7240{s~!~xX&2`_~24!&XWJ4|ruTHq>9$G4b{>x@n6ai1~4?~-}k(H@@MaTn_G88;<0jLB0imB+cy{sn0~P6Hg4*5f`4e#U~tiPCwTrC0@(wvSNrNwP?5 zu}av4cqTH#2A!vDXp|35?{T8=6i!Be29KMLb5S$fI%L9dPT>{&9W_H9%;<4`z(kom z?o;w2YQ}t?*}CQnssO37Sc_Fd4f3v-O!+^A4V82zs>zOpeB_?Mb33bbNjeO0{3|R$ zi?qWG#7nc$SH_N4OkxI z=JL3U%zCIPx^u`;sPtTfnh$niCcKR5>wi#d$fUVFjvh93QB(3DjKmeFdE+PsQ?T)j zjjI@z$K$NQTi6!I=Jhy>@G*wtsC;(g$*75CHL4~DP}jK?@*S#) z-z@8K?qbm>i<6b}IM0du;Xux7Q{D=Ct%Ap$2@_YezAcBE7rLTm$XTejUWYIvapKB0 z;RTDap^~*n^=S}m;at>ceSoU@dn}7-t5_elLe+2-sxPOY&iest<3rS-Em+lBwkA%e z04-7Xdr^%pSN{8|dz@bED21A0H=-_3w}v(Ca{Neq598s@noOl!=owZa&RyH%T#V^B z15kskN(lCfCDhHu?@0$sOS1})b)PBa`+0{V}T}Cv3WR-_!0WKztgR$ysV#0W&xExNl5`Vt3*f*cF?#uUCePf#M*jfRsBZVQU3fZ9BHoV5 z`%)b}&SBh-kvP1QRd6XP&3B;&>rb7?f6YiQ*`Yp7(3z4`!?viJU+7}_dJ{Fp#^`Eh z!Scj4P>a-QsQKUsX2WNA1rv9(CVz^W7d~JF7V7SC>S4Jc8*|y1jec&Ny@xeTp`Mm* zwQ)Pgr{P$=pqDlMkEnv*N9B1~Z;!jA%8iqW2Vp#n*~bc$5HAv!!~+9ji9 zR!q6*QZ3zPqO$8eM#ba=tXr;oOI;ow+xF=w1+;}$1gK@V;p=W*Wy?nR|v#_`rA zMNxfQ55sXNDqF^&50|6*b{(o~cA;ACdT75h!O}4f_F#W+oTn1rVM9$jV4}_OV^Q0se!H@z!jQ`=L~!Id`QGd1yN&Z4{A`JL0v!ihK)!zVl1;8XF!eaHh2uX zpvFk<<(8%uP@}yIYLNEB8n_WvqkmA#?eK3cTZ*94wKZ19ZK#>>U*z!}bc(L9r&tix zXTM-?Ot#XN`H|IWH*5auO5jOyZ7AwQw!8_)M@ z5%Mn=8>(Sh)F^I*YSO`&181NH>3+D_2x@A4i0Y%XYwg5JIEc72s?YDCF7y_4qqysAe{t0Nfm)~vbVW_g zqeHGi)%+|*;u{R=#6;_DM^4lzu8C^m&ZrvPLJiJOsFq5%!Okm;xugS)u*w zQRn}RD)=MRYWhQHf98$ke`|J>+-N5(M$J@PP!~Fms?n{`@ei1eIL;hhE zklj&THWu}8`Zl!x6lx6ph6OQvyPa1G8xS|aQW)GAI`9YT#$h|`;ZYP95cfh|ID98f z9)q`HsKHnHe`Xt0O^2eId^r}z8(0nF?=qXAuJax0_&KCSgHEj7w$dqpksRoY8l|&C z@m^F{+(0$;d(0~I-PK=TFp zxBsBW`I-Z-Q0es5A#)VAB;JTE(0|y{vlGT4UV&=*EvTBEL9GGbq9&@eM{Kd#2sL&( zq6YU|RDsuH66OC{HvD)$bixZ%AICguV!QnP&eq0>XP}WuH1o|y6>P`<{7F&o-_8W$$W?nzIQB&|m)F=&{ zwZ1EdihH99vKn=x+o;!nzzUep(|=WxP^LbJLhfUN{bT+%V2Z-G029RF2m2( zmqk!bT@yRt1gweAQSV%eU$9YK9o2+WP&fDvRiLA&_XBtE5qd7#66`tFA})Bzp8KOv zEf}1^hF-WHb>km#FvhrSO*j-aSiVQy@GNR7eTXG7@h>*W>S8J4nWzGs$4dAPOJk`k zwq_iPs>n=ito&ce#xQn#L^b`utM>fgipt~k*F5fbw*ygW?rMX61jJ2ezs$hymhnP=jYa zYP9YTc>~o2A5il}f``@;sZj+lf=auuQOBpFrr<@G6d$4r^eJ>a?IZGEFU-yceZiMa zcEPQv3!Fyv)fLnze~jrc(c{o(JE$Af#P>K4H9t&yV(0&aO7FL*u~F)G%c3Dzop{Ud zK6L)!LP2U&m5l=)l{S8!; zzeiO#@Z7Sd#B=grCp2M)Za4rVa4IS*7GW30!YZLOFAYO>c(fg0bS%H!G0*|7O zZ~b63Nc+)lkP{=>UkR1}9Z@%&hLv$4YD&L{YRQOC?(2e1em2xZHBe1C5TkGws;Mrc zUhobzx>NmYH>`v@-WpYb(Woi<5UMNx#;zFWKbtWJ;~e6B*cj`4R?CrpYuM0<|6qG; z|Hb1jN`Jz##91A$J9TzKHTC~cPd!fzuUmkYSdn-n=E4&=0^fw}9neR$MN_M*J2Y~9^31Dga4pfVr-b#t>HS<+cWYAR>Kn5intRt!XGhP zOx6#6uag5SMR=W&?C%}dZuA-p5_{u$-T9*=>IV(Y<9UPb4~GWE_qv1SPy(;}R630+ z&_j&GSP8xE6rBlG;D)Fg3_*>F={Onp;%BUw$m{IF3W@E8A5l-clu0ZugsNb}Btfs6 zW^LJ_CcKA!jGNT!=5Y#CmsCadZC9*~^RXyC4#jDcdEKk+~O73+Z z=fzTZ-RJ&N)S&$kWTPD$MN)d*sdOQZBYuFbu}dnu&?!{GzMy6@e`>G$cuk91`*lFw za0uqZEvO0V9_q2~OXGE)t`$);X*5>E;BGcFA^eLP-Nn+{Aen?si7%mQnk${x&EpQJ zf^0xF{Vu$Z_fRwGu}H5wZ`?yQ`3F?rCrWRFwhF36TO&{1pfidM%|zQ!eR>3I;x*J$ zFjEE_lvPnR8;@#%-Kc_`LA~xOssPVX`R~hU1xSxtz?8%c*bbEqorUWu z|I4zWrfh|(;Y3uE??v_9>(KsOS*;IS;x-1=eAIfOT6P<}jZib_7;Me)<=BDvOAc$n z4mrKs|HIg*z(#jGgl=Ewwzva+%l?a)C>C$c@>p7p&(DR4OBb+~TUpra zL~-68JcA!FR%~X%B3@?@1vyp1>%54`JX6ZdTbApyKTUb^zY#B(T*2$UQMiXAIIy{* zjpkgHNjqNH4mE10SGBG))KdoT}%3ss#ewNf1mEI4q4JN4N zb>H0ftrhe-?RjCKw%7fVsaGAFYP;6;I=4CT9ezUreyL}3ew&6g8T)tR04&`UXJ6e)B z7}>$=BxIxX*IsuGSGOaBg}7=buk(rZ1DC4mTm| zUp}E2;&tj%qGYH=W7nbfR6Bq}i4zX*71k?<-1q{kRKhp`g=uk=1Mjs;?&ESGWN+_+l=$n&m~!;WI-X zLUq*(jEnD3O&zwxx-=uI>zbola3HFsreOw;Rj_V22Nz&T%)Q)3br)0xCZoFcH&i?n;U|#O;M6;o$-Hf{7dsG$_ zT4AHSGiop`L)~~C>Vj88`#+{jj(qMiMVTwa{L?i9ezWzVdsUaGs$mP++y!nTNelUF2C~H;jjB zskEs5&DN0rJ~rC3L!-I}M&edf8eK%);5Amn5^Jq5gQ#?zk46@QMr~Qt)2tch!11UTY(uro zIaD9NK%F14-OP%rd3jU|)x*@-2~~ixs4iR2*v^gVn^+F0dapMoyxp(#x0)WB<kZ@0C;oIUn#_7LjES@(L~k75g;YTPB{VASyms2k2g4YsYQ3hYG<(x0)2@;_pq z^_)Way(0ULx-*qAsPd*W5pv!c=u zUiT*$i?9&)cN!nG4;ZFkIpP~w8B-s!G0+|>5l=^T$rWsYFR?k+IBYGn1{=^~mr&3Q!%jxGZtT25IB7taLbUAYNd<=N$6~UOP|zKVf6X1zTJ$zQj8lPV8{m z>->UAf3YRiGh9pD`HDRY5?o~&&FlK%VyyKm1;9V9lO34!CKbRUw`_{;a@*_N;q{-f zKujuh$Ls!PH2!ZvuXBw9=YM0FjH~b4Q>wuOultWgKH+WlZ+qx@hEYDcU~t37ycW^&M|YTK~^xV?8_4#SV9WYH=FtQj_^%;ars8 z`Fg_Lv}zI_?(j2jr#rS~fA_fI?mMOP@xtASCSm+=chOlKwFvEr%KKUP0vBU9oRlEk zP1nECr~EIGFx;ImDx$KX4&KFOm=2>8g}Z6F4E6lJhcz*7;&AspTys>5O~!$^7B%yw zOJXfj8TI&WgqoOws2OuICR6_JXCo3XhkT1_sw7G60(nt2DT9Tv3TD6&sBg!;#)WaOa5s{1qEQ(cdA!_FP4O?Q?^x^IbXdLSG2SdisU@hD(LonQ3 zMh|C)ns6VE#k;5#Ooxo&?o9Xzml5a5WHmj9YN^ktuBw&U?1UQiqfk%D`KZ4C4wK<| z)cKE5UFig~*q|zc8XO%^50PG|nvOv={YuoJIu!D8=y;s0))KjJ496Rz(()$i{O33U z6J-l``jIt@u?WV?VJ#i3oHN{+!U+RH#>^G&uI;AdQTBJs9qxXRkS9;L`%%qZ{EhvE z^M*S&Fj>BErz*Zd)v#oK%Z3IRm$(nAD@I}hyn`CNuaNVDPND)f(d0x8q9&*lCSfFQ zM0L$MR7<=;HF3Oxc3x)GjjEz*-WZjRol#vd9;@N(P<$J8{in$I=i~B1*7QkHB}|Wc z%oap-K}~dLF4W)}hxzdgYDxBA$P|Tblov*IWnCBK{H3MEl)$|@}YED(g`n)7QA+CvP z`a)%`1Ce+`D%ZO9?00?kC7zX_E+M^Izr23EnS8rEeKQ8VXSB-?_{dp4A> zzM5uM)CDV{ny4-6#@$hUHWqc^W%wFn*RplPJN%ybSZ!O7bg5(4>5Uq+Q&8FTKU51p z!A#2k|JYEvWvFW>)<=#0VW@(v##r>}QB03N)??7&@Aa)IKQyqe$lcItI3JY_yKy|; zMOC0V==M8D{*-N*s+Eq+4+uUPetQO&f>1Z&VJWjyp~4 z!WmI<0jz|LP!(E?nn8cWR`>ve<=80S)cRyB>bZUy)zlAAE1v&Q6HCr!)*|gt6H7E| zFzv>Y_$;(PYjew%HWb}?*_n;~ezn$fMK^#Qf3{}ue7{E6{Hq?ZlF*6ou zAMUtIdngF62p>bwi67JrCJ(_r9h zyKpAd;HZT2upKtQ52%Mst&VnqA*d#uimKsmQ~__G`t%P}*2L&!>6``?mqZUXMxECR zc?boafoy1WuERX|3s%NhovmQiLw3Vz?4N=f6IbyE`Y?GH-Y+QbY7W(V7*lswO#bC(LrpdiRig!12lt~c5a?l}xe{tDj77b{*o$hh+t?gGqWb=; zp0;AzjN6G@_p%u3qk;qC_utp?lE@-1%R__iV8PA6~0k{vqm26cgK?4ODYPe9pYR+`n`+t7Vp_QSA=D-u#{SgPSzmD9-~u)@mnWHN z6H#q!LOeR;O^hTiG|RfC5o*Tlfkkj2ro&BG8_%KUi`27i(ODHWXosMdbj$H1zQIY# z|NV1pYOORk-2KR8B7VmSx#n4X5jPVLm`_V#rUf=u#$kHm6_^W;VM2U`?hLumjweAq zRg0qXzBOu!?~ien|D)K@;F*E>aXo5=yMK+jMAWBO^RXJ< zT4kd*%lB5GN*Isx2B2odahQPfR^m|NovUpg$h*c`ay~|~f72TBUmNGyp&R~*#WC$# zD@YU6oIMiN^e0fyhIgp>p!7O>ShYbl`Fos!nbz~+68;aXW4R4BSf`-U_ZVvaIKP4X zR#QG?hw?GbMr+Dks1quqo>nce0`?Cb-xJz@3AJi|ipqj`o2+T`q9&jUq4-P{5`7V$FLd3-C|wQ191&GFtihr=x z>PggNHtRv_`!P6|f~>|Z>@RiLX2j<>m$=`NaAyPhj#|3?4~G-qLrp*}k7+SV{$)9C zpZWBcT<`;S;e_ZP!`**Y`#UPVn*S8;ey=wGHIto1wOqQBHrIDY_3a4MnA(GSsQrc` z@jp~oMW3<)Ek^hE|39*kk^{G}4!#eaQ025`L0?pMEJ97yr?5N5JYzTLg|&#k!7_Lq zHRUEhYnDN!aRSH(FVv{6dCn$|KB)7eL#{xz$Z^zMe+L_) z|Gd4fHEOhvMP=0iREyjU#j$_3f@l1h{8v7fWruE99W@qug--YuR}&vVo!9UBj>`|DmL|JC#zLpw%=oR3}EzX>&(lV7qL z^u}bw3sHl08#chim#yzRqq=Yv=D@#DuTT4nt*{1SH{xGWT~;x8#ir0%*q$9nu{376 zYE9k&RpJR4fh#Z^H=}B}8}+=uj4D{iYqqYBM$IGhQBD6p)Whu#swLz9YAq5>#YSIt z^8`f&I5^zDR^R zUJj{1&}qSjM&n?-g;P*X*X6d=a1LtpZ$%B#yXe6fcdQS?@oVC=sHNCg)L?vk*V44) zJzKUHHp_=l0R9Bow4Yr#Yz~@*A z|G~*v=7AkQh1H2~pt>mYL%UvSbpQQ-O*Yh2jd39kMfG{&NA|Wl4=O9VqDJjvQ~|bO zay*E7j9)`P#(iw(rNBtyyr>G)MV;3g8{s^3KmUKuMie_TJ+Zm^Yg8YtLEYdeZo`{c z1*iTV?i|M}sEKCLA69|GsIGi~D(HJuSH^j2tKed&31}8-hTZ#={GZE4#%K2F^81KR=1%_9{IwWE-$JJs-OmEZ`2LGMJ=_?U~#;MYQdB*?Yu%5N!$T7Xvg6eJn@qJ z*Hdo5pY~~Yl~>`;-^Bjc_UX3s#y(`ai7NTIzijZmMRnOH)C`#Rt+h;bR3FbmoxeU5 zUqnq*e_(Bl`?no$8Dyg>JI11Fat5p23U&eYP)YHx6`%=rBA$YN ze2bchzF>Jw^`B)^D^wbOj~X){aRHY99PW(5n@E=hot9s0RQ5&9Q0q{AcN3M4f1{S; z+4#1u8P-QFJT_rvycse>44?aFc)FlgJ_BR=+(9@pme0M;Jk-p33e^(c*gpIHe*rdh zU<_8nBdF(eAk61BQ8p~Y1*1?+8ua*_Nc;}fMJG@#@Jq;7sDeg#eeM+82{jhRpq0*-<>IH*P7g&Pn@fd3GJ;J0I?(?}Vlnyl*tD(ALKxqFS z)O>Ltl|6r=`aZdz??2U-`PfK^3s5)OfO?!>LN)bkOo!QV-0M?{>|7CW5+SOVQb<`aeP5{!BHTt&mAn^U_}nxLl0(-XHA_eWG&R#=oxY% z)*)VrYOyz{2`D_i6*x8KAuffQ(7K_<(s_+gKr?JD^>*A&rnJ7wmdfX@RHmc;MC)yg;`o3x<}TE{@g8;ItZ9Aj)2{|< z|5eo3@}#q_Oo1PXOCVz_=oF6hxq06n^@6#Wj2oRmM>kCGbC+0mGWgs#5(P5)+)uCj zqq5*K>IUyZ`Z8JirACd3L8vh?6?NSesI1zXnF{Oqe}|2t>?oYY=e~*TftqR;p{CCB zs6M`n>eHtfiSe`AV9Jjgl+96NWB}^MQ*jJ8T!mSPvu3yRYoiK27^`!CXE7W470Y8( z2}kF&rd^cF=e~)&i5hIha$8NKu`uyW)L{Jyzr%N^HQ(YqHkS6H2K80c=zoVDF+pCR z`;;7v!Mg0&&PFbbmCpuKK2%MbhU|l}DcB@bOH9jeHC~7+zz$TC-Vepz0`~eMsQIHd z>KQN`l?BI9Ep)p8`L8DZiyeh9MM1l8ZPW`pqq1Tist{Zy+A z4kF%%wXr~9%dSzV^S{M&v6-k)6`Wau{MW|15_aPwSfBVNs%Cjg+QXwI>bx1Kg02n4 zw=pj9-&g`;m$LK9qHfq2m*Y@ufGJD++%KtmVtZONaj=XHzQJWJ3$~(WsQajzMMPO& zEtI>Q&;1y`J?i*J?11shTT2eWTExLgY!qkX8b)G* z3Rcnr_#^RnRC?8_=yU%(PaAARoV2pf{qVUvZlNZhu`T;ISM|BS4~}2W=9TN!ea_pM ztOIKLoX4EEsE*GWKs>6RTR^`5Yv6Mim(v>h+|PQW8rkR{f#td2LDcd(##dJJIJl8G zGfu%v7>=zQTZ?tZvBcw1PrX1BYk}lAn*!v)J?uZ))W=Uc$iHFDe9l^Sq-^eUI&s1= zY)724C0#*Dr=eCZds^FItk;Hgi^W3-r?Nj$2W!d|sLv1n#O>_w-_fSttet%BKS~>p z`V(x!Ix~jwLKnJ3`5)HJ=l({cD|Y6<8?3?p<~^(iYf$;W6P3>&L#FKMbM|t63Do|# zy?pMkToU)T8g~pi9JNlEg-XA5s4;XD-OvATu%U<0->9@p+=tPLT8TcydTTqDO45& z`dbTRNByp^7wW0Ea)90N3=Sp!6*FLiftuTS%%b9Rs2aZt#o>c&$(D0a&{i^4*io7j znxbktA2l&;Kt0tiqSEDW%!d^Q+XefFoPt`8e;0E95S#Na4YeiR2VBVOe8VigSK&(H zb3rzWveA1u&j4JD@v!0u%Yufe7MP3bnjNS$-fyURB2Ki`q-e+ns2Q^trpGC$rP^jx zR~<%O=OXI5!7prRB1uC_Y{G1)rn-t+j=w||C~~CD5BX3{S`Rgt`lA*YQ&Bg(kEJm7 zD63F8Y)O0@b)S->ttBfXPtBk+hYj6mGY-LfsKL@~jMZ=&YI(f})#tyU{&m}b7)ji6 zoXw2UsG9wZ-S9tD4ZDmdU1RbI3Tht7HOc3kATEaSl>eW7R?j;BNo`aJ0V{TC?>= zm3Rx^1wPsp1Ip?24LLUSd~lcETP?CvXOF;76N?mS8U87pUcS%Ab7BDIA5WV2zVD_fN%r z+~3J}%0~M$yiV*nZ7p&S)zsm$;G1Bo8nLLwW^CSHXVII-vxyU?)Txgqfz%*yeKPkqiL>>lzRsv;ww*|XsR zRwGXM+{Q`^RDo8aCaM&{7dFLqK)vufHpfygStjEG)Ke?&pEg6LL`}6(m*F{Szn~t=Ltok0`5BiG$9?T{|G~lr)UzWv>W!tz7F5&!hJ7&YUv}Xcs2T4X*2Ru* z?V+?ARr3UY+n`#EdRSe?VVL`!jfHinL7VrzU3V$&BaZivYtT8(#<%Pk_rdZu%SW35 z$6-JAm-^&${{YEWyiC*A|JNpzET4Vu2Z`-aP5BTrVBRl0ykc@5>Nj3}V))%Dd2~$k z0k+`0in09m`=1qT9An304Df>WvHkA4epi^^ofj&4{BGV(K}{^@F&^jT_WIp-NTtK= zb+LVZcdn1`_q*A%7JG7j{D9w`vV%C1xOjx$d5imSm-2s09J@isxPCVq)`mQSgE)Qz zr(pefcEh`3In?0ToNh_a7P@GgA89sr55zYVDfJ@4n-?f>nuArS`ku*>pfX1Ky%8P%e$% z{WZ)&ET}lG_3;4I=)a2V=&GFQ{O%thxDrWMGd7N8v@V{ICFpm`b3&4=es>COlFcr3 z5({vFe{l=u&F**K$=t>C#Bp<&`7jT0eblHQg(dMnEQ`f+`rU8WhM=A$&#?^V$mMq* zT3v!{j9|xXEQ$$oTgj`T@_!j>O_w;2waf^dO8h&H#DRJJ?z`RhsAs_Ne17+%)#s=Q zs%Cz_n;rd8Psc-84jUElyZ`DYIG+tQRr!K`_Z`lBTt@r|+u_(k*4Iz4JaL7>)-~gC z0`XbgfDMcI-KU*X)bD<5*9ldD&c*!h>i7<7tko&*cPFaZ$b1oW&arG=Jw*D{v8LaM zlZkJmj(4u>chmI|&LM74&+jhVf5%qDaqBaqVGm3}!80}RyT1b})6nn!|G(-sV*X%% zhp+tZXHF{`lmF_=n~nYM8;}A`{O%it(x^GQA1aM5p#F7R{HA_qCeCl>ci&iKZ$T4s zygoAQoDmooC!)sAd`yTZ@h0BHTCw@oyrtj$dt+hk$ba=obUT}wKBCgGO?%7xJE$L- zPU}DyVa<+q-oj3P=Md+8!6kU8v(51>x-iiYC+%uo7`vO_{uw^1LY=z%-5)d@#|FgR zf<0`?JcReyF}tTtz5RRn-G#y?tjGRLz5VWjWFYEa#rgaA-M^YKv@a78aa2Dh8f@2} z!HZW0SPj1#U#+AvHjCO*c%0qfix}`yKhO z!S*LR)Rdo5V#!IgfkT{ut3>KMuXj;I<GRSDHn zt5Mmr56j}6kZCvC4A=^_7~PK*mH$aMS)MmXHPK|$1(u+0a2Qq7=cpUR+-wC&ih5dB z#Pc{A(_qsrcAY_(l6VrP#qTf~9z)GDH_`q7e?1Ewh`rVCR1V{p%c!Pl|34e8LCg_@ zZ!l0#uM4|uri-)3?|e(#9kawFjrZ}8!9oW}KVBdIkY&e8+)4Zb2jR-Ye)ogUlt;)m z{r#UgY~;bSM{Py(=@`#_PON|224^Q!-wwp2xC-OrKD>ZuP&46@6V?JJP%ZZ$WV|1( z<|R?*H9&Rwa16Rl&4vcWVbqNtU>l78lMS9esQ3tKO3rZ73R(~A5wFF<_z^W9l4(LstEQbu5h0G|C)`R>`*~c{%m=<3e{96aUs4&737-> zwn93FC5bBHY#E{9Ei1X4{G#! zFI$skK-H)OY7l)D+J7EZv&X23DgH0Ev`U4lKqHLAXw-zY9M$EgQA@qxOExrxX1`)% zpdPBk{ZWHxF)FQoLcQ=2>O#J&HaH8RrsfK$nXNynpx>d+JB;e{XQ-Bnb8}cH0p*cLh&I~litBX_$hQe->)`7)y7PE9crFfgSyTSsMp;>-S`7$#yHn) z5EsM}dfK#OLo?e_R87ucLHrjZG2ac_Uk}wqqp>d@N2O8Gn|}8*p2nyfufQlgjLMc6 zw`@g}4@(mtLKWyg%)^p+713jCNnHE0 z-#LvNF$xELvGm-7nqczr;hnx0T!DWP?}-s`-y3d>8F2qxU+G}1fcs_eulRryX<%dz@4JI zqb9C#7=_=WZuncs#Hj*q7F9ux2c1c5{J;sPu@X*99dM`C^Qag4(*)fA_giVyVBM9L z8&UJWP=hNXGT?py@i$%~UXecFzS-=b!D=`$W5E3&<02O4c%e)IcSY0_t1JHxvZ1G# zFS8wJj%&HW25is%hFJsd(`pZDtyd;n!2QSO^RX6jFnhrLobViKN==f3F2%a2r|CE> zhTBo|%QGy4sdEPG|Nq&X4b5DGFbbz)20V|NQeUHPoIaPGHx%{on1PyFzr}>O2i3=C zP%V@sw^b-lo`5@1MPq7?FGCI5{pkMwe|+A6JJ+v6UEm6;>C@#4xSwp)M-{M7e(Qn> zs2UzeEmS_Bj>j!vW}8WAoFi;sNLHSo}n)Ou*?F zlQ*5^1I~^x(y<~nAud+k`m$b48=Q+!O?my=4YEX8lZ3W(fBZ;5m zWb9GL2K_&Ef&usKamKoKgMp};AHoj!2K9JuT+g~@5^ByrjT&rE@mu_%zV&IV26p46 zsPy_BHHPvxv@tdpeB0O1I}t zX=36E%>wSCvT5^x`>Xf3Ew~Z;d$$a@e>v%wRsr{e%XO^-?mu2{+J*{oyk%Q!siW-z z?i-U}%=Q6y$<+kQaf9!%ib{xWF>i-}`w7QPtW2Ee>wx=XvjNzbIAzCx`yKIAoJkzB zQ@~xI%)#Zv5uF3>pI+LCGl;+G5^x_vsk^$z`TxJM(SVYDM6La5cegHCj4IGe{2TN2 zuz4X%PwTsO7{TkNVntlkE8u>>c(-@J{ir2lANKRQk*KNq1b&Sf`&vs*!u-nrdu&|e zK*D|jcOCE)l^)6aTlzgfEy)rM2)LgEw!=v}KF|i&O4JSR;v6hJh!&(kmrz}na)=Gq zK6sCKJ?_MBhBD7^e$y<*OICsgSS2&RP!Z@23s*ET93$tTB8(Pht!!cagH^HXL zsS~Zqe!_+v&ppYSel+$c-h|4M43h)ycgPb^c|GQv0DlFYjVU&D|2Q?^etcJUx{dxH zX9S$X9G*9m{MXW}@hltt+PzH#m;lh*K}13E!}BenG(fu=vQrfYXII%_4gkO~7Wv=TPqh zvMmm{UrHTCHEoe4))GJAJmOMI1MVlJmvJ?5xn%+O6O`X^6mi|<0cSm4!a{V(h~T$2 zc($*!2`2t3i|e6oumzPBQQuh$OhtXpSL*wK^M{VF4!A3xg==i4tiLwk{_l6Yt+)L5 zZ3wtOaM+9ac-^v%%q#fwCOa>fYKt}HUM#`|V{f$rl*CoU3sFs9dRxH#U$Q<&jg^nv zt*MLbpl_+k|4=htsa@7}<4_N;Em#5Xq6TNi-4=I77P~=bqir~6P$xV^)j02-fctUV zaMY+jgc^j2_S*c=8FLWt$0&S^ddOtmXU~X=xPy2o-lfKQ_6MB*h~xeca6kQuI2d5{ zO#aPbqbCP8;}cAC$R598hb>*Qp)R}}^|(ENdhY*%8dMpM*mHk0Rww=e^>7M5Y73Nd zsFuiajG9xBW~eEA*>SQ#`M>jo-C*pG0rv~Y71)RUn|>n3bSIgDc|lIp4Nsm5I9}px zr>)8Jow4=75In%~OSm7WpAES0cxs%phgu)h0_8i5!n+t$%_Gm-+HN|2A>N6q@y?$E z?i-DZSey7Stc&F@SRc;DPQ=$xX0T^~C;Qr9y_FWrvYwp?8>Juvei~nY6c^NZueuw)! zPI2OcfViXfQPg0q@Y3=-=bswQy#Ie?GhYrYM_dEd)KhUF9!I^$EcrU% zexDF*#zrM}yhJ}1dSjNw2;%A?8;ACH2-z#-Pz>k1aUrLnUN=AFcOkchJcw~Qeg?T; z(79n7&ZE!?uP{C*{1-CrU)ERYLe4}D!dPzu?q9oZh`M3Tzpda+k%n|SpqjW7w!&ys zL2sizpm>b#-~W&K&Q45;T2|-BOjr)J_Unl0Z~^w9Pxqj1nB^avfC`3eg=*SK7zfv) zZnz7x;~`9sf1n2gA9%g;KOq}Rx6+syo8fpIiRu#X$AJ4WdO}P>97J7sE@s4Km=lkp zX3&?|7z=)~eE$a9690(Gs%-yS{`bb9YPOAyYbLs1~Y+YU=i=3wKBL@d(U? zvruVy2zB03tf~Az!-n!Sx!-P(H)M&Bl~4t$8?t3+f7g%$LXJdT_?wV(P_J7Wa(&3% zs0GP!%%SIhcp$z&>c(RdS{BU0HpB~2 zqx&OH$3}^)V2>~Z@vBgrC~<_lR!obUA^V}aU_Ppa)}ab=A~E@|Isa1Vg!`z``xZ4w z!jjlLPz^Q3PC{Mqf2h}8N1gu%s>UC&7bZ+O&cs!?FJ!Bf5$>mKF;hjjpPU}X znH&!`Ngd&IXX7bqFg8wOUc={#(?+;oDpyPw;r>0)!jTc~GWs9X`R~%(Og1os_3;AK z#I*v|f@@JThCZ4nfc-rxTQaFY?NS?1H^YM!1W}>liB*Kf^1gwrA}cU;@CAT-3!(r|J8&I z*`WeV#|rotR>Gn+txrdwYB&!y(QLx3_zUWF|Kc%hQ!B##05enV2=^^lPt*^Um*JLJ zoL48p{dv#E2A0-28j}ARcwyIu=2R?AycMJHC1%9DjVyf{pl+1lt5DP9pTzG`YriLr zBiuJAo+b=RZrl^K_8Z;Q#>7Tc!LOj!6aNL-P|fl+vl>^yOvF=BCvHa-=mx5w$(vi# zl|;SeYJnOHlkqx!MrFmXEl5WS;AAg(Bi#QBZm>_g2cemi~?hxDu*DOvG2=4Aqf&>i&cY?b^aCZm<*L}bH z_nx!osk(iod*Ax ziL00rW9_!4&w=U4x5vyl0W;z@OoMlWdiI1kIml;0by*{9htqHYe#JsKZ?8S@GL|I& z3RTfSo_*F7l`$U&dSFLfj*~F{{t#y+{*G$8ntz8lsc;jj2`^)8{En(|>;qQ7;uw#7 zE6j?$QP(ZS7BgpsDezu9=HZG zV5;LG&U%8RB4#81`J}Z-rc)u#YVtKu1^pbD@RJ7c?xHvBH2TJG^B zMhk{q4sinaNi-%gn+KG-65{yCU;oFV^ij}sS3{gC9B+xL$y!uXKEbM({hAeg04gR{ zpt|rhM&MISjp5gA6y(4#@(r&u{x!Z^24Cods_|mfIKPT&k&mcRkp2cuhBa{;&O#lp zbJJQNfQ8AQMs->EEsKRLs8LoKb$tLeEDzlZFoBSWc{{|JhJ&#^CcG2kj8h)#>-D=< zgSMy|jzZO79V+U#qK4C>;Oovk%V$6}d2v(=G(ufJ2nXYD0TLR&>F?W#^)M#+9#{u| zK}G#V%#V*zEfev;EQVRgH%3MMNYtoUj(Xq@REyk16)@Su5T^v@!KN7KMM5*tIgG$( zm5{)AdA6Nb^tyx`e90Q&d4xKXJP#;1nXE zCTxfiI1&{TD=;_iMit<7(6moOoTudLp@w0pXZC;&s2Oq)R>Wnf`@BNMis!k_eETpT z`RFfHamHUE5<0Oqs-*4l6Ar{5u+K}2=4q(>I&92nIe`k+%&$V6f!GR*;cZmGB3|2d z6>yDS59Z_k7vkh4pAC~~{I?{brWu54;=k}SUPg`U^KUF_)4jCciR?j^l7V&PD}crVkc$ zA7T#1ZbCKDUetR24VPdgU#L?7U!aZ`@Q1nu8j9V>FTkIvdF=2|XBG~R5$X)Z zh*+WSvb!{PsI!an9-s>PTOdxTJB9AV#T@X*4RvE-4Qjc)k809N@hBiS>V^%;&xs%E zMBrOguq8|o>MqyCP+fHzQ{i<~3w*~3m^@*qlal+aMy)Y{6Ny6IXwQ{6)M?CttVu$h zx;P5!<4x4^Ss*DjjLNE(EY!Kf@x94Io!YoGMX0+Le84>9i=+&7=3zI~u#FiJ>dq%6 zP%YL48{jr%h7CCJQiZxvz6eY3!fPCjc~XbE!*LgCW;=_T(_iBSjFTqRogp9MPVz<4 zhPtcdC)A9YAiZ@>M%08>1|zTsD#)i`B8~sWBvjIEs3vIYi$zm0XkL5J}^O8`L^v9NX9!p_? ztfB5Evma`To)mN?Y8328O-MH}Eq+1ub*gNk?yxP6nkV|ArsRbffk#pE##0O=ArU`& zsB;ms<1!4%5$f(xR^x2)v2upGtKCY}gEydN%u}eYxsSR}NG>xas)-At4=Z3ktc_~1 zQJ51~LY)}otKvv8C>iQJ!@2lB&Kp(A3hI;zb!Wn&sJ`uhnir-7k8el4^?HP9 zI9{}zO?ZJ8Bvi7os6JheRdE+8TEoj*%_Fcl`SPef9D!>3m8cs2g=(P(SPf%Tu%K;> zvC0386DYuN)bkQnqRTb@bCc-Gfi|cq_5$h#eJfkj9!4$4-YTI^ER0^2sgxVV$MWRs zRSR{_M0K3`s37ZBler#$M-A65wX6XBP{VKxDmeG+c$5HrKtd(DTicrQJ1*k^Me2mQ ztKGf2cD!dj>$7!Oo%289&sed(1=()Yx_%gSzpq#llQgiYwh^jgyKxxCY)A|8d}k^N zU9bSPq1cWs@eQgA>i-n#zUl0Uk>pokVT{|zX25FLgZvWggb5pmIvsE*>OL<}U0bt> z)w~@JCjUPSXkvL!LdI@t1<8b^$(Kd-@hCiuQ!zCy)1;Z5H@LY4?_A8m>)S9hKEW=S zpoL91zoKI1H&j6mqlW8^7L5O&NPOTxIjr3>)O`mt8!wZ;j}vfLD{JyZK$&c?_mV~*3K$;5EYC!QNj9eJI23CmMGF5PzbwF!_lake{65# zHF^h|VlxM=j-`0L7ZSzJX4HJ}1U3BPceF*SC@R?Eb+UOOHEMZof;F*2fW$NsJJ832 zYjw7!Y0|}pTW|c6*Ei#@x}d8y{YzBA{oQPwXUB@<>)~izh#Da64S)O~DmI)}qG{$upDCZ39aaltlR!-?JcTi>M|5bE?KUjo%a8&OTSd!ViR zaR%9NI)$w{UY9P_)V&Qgy53`nsC)o1m9Qt zLDjIzaO;xhsJ`uoVYmb}Y*(Qd52O0_EUIg6qFOHM2s@q$TawR@opBaUR|!2Mt!d|D zBl4?J^MH4h?PN+}8S-`UGBucudQi`?wq+ZPO~_wIO+bam*-Y0BRe|NGRdEMutvQ3W z@g)Y-m!-$sve*#wknf8+aWxjhW7q(F6YO{^Oi2D0)F_#Ws__!+Mi(6n9xpJ-VyhJD zyiurScPi@st0po2Rg74ZcFPXo5L*ydtI~-vBi{`vgd+FGiv+xQ0d1nQIpmL)D-v zs>#{~9ftA9FF`fwW>gEF!dCbK>td~W7E8-9F8SN2pnQWWXdu~qYr@Q^2UkW#_bA+t zlTk5JZ-EWdPN-;~j0)1ZSQ#&%YLsfBZMSowMoTl)a2<)2@E_C;DcvG>`3^YENz~!M za#Wvv#co()v2DxOqoVpQ=EuZKY(A)tD!_j1f?rVQbzEv)vk-HTe~jwlB)^$yQCqMa z7+2%JE(z5z5*5XRP))iBGviiNklw@W_zE@WNBnLNsEyhq4h^~%b=_4|mxL~}7EXui z%5tdZG{?3Y|3iZZo?t=pF_v2&l}DY}1$*NpRG)iS*o~5-9#jBzye0mC|3g(^3TkTp zBj{;V&EH}KCR@q)*NH_)=s+D*6#tApaT2NqA%9qKrbV?>1=M*>Q4i=B%r8LQZ+r0g zIn?UgzPjQ?gNT5~`r{EeEaE~9Sr995%`)%JR7)bPoVI4u=kP+f5sHQc@jk7xYT3R(j5aJ(Jryg67KH)9bD_&3-K6;KaujRkNz&cu_b8@Jm? zlSknl3@VC$`^(&oD%cfNlYhf}Sa6eVNV*2yin>p<&Gvd$q(uWxOA^I-VG>5*8B~pn@yPe!DIaD)_RZj+e(0*aQ`{ z^HKLZ5X?WqfL6U&e_Kuaqx${|D*EFbuqMxr>f5TQDR(4Jz`dyDwB|t@X3J4E+kt9{ z8+Zc44%ve*qPp-Ss)C6RGye4+F6UueH2Pp^^1q|{_zAwogh#ANy+>^^DS-;Y$*7vG zLe=aXD$3uXx-#xDo4SjlTBZ`J0Bx`o{(3B68-_z1P%yng^>NPQ7Ns3feYY@}KZWYs zkQ4TxBB%{V1609>VRt-&nhy$}v}L;)YT_D*BXAxnSknbgS=0T3>a%I6rv3w4;{()2 zr1EJCrkPlh{0dYPK1J;dqMfkT#L zI_yKfIjRY-pqe=Pd3#_M)UsL{)nYxd1g=F*DF31gknKXKQx@x@rrbH05U-#{#lKiz z<3Gwp+nY5+HT?zD`XB3(jpM;MS;tXb5qa54J_A+2^+B%%^<1&nbEEpa3aVw>p+?z2 z)ci8imGQTRglcpi^?>OAFmpxa14uj>g$u6PM=(`xggUQyJ?l;0ZgV`vtx#tR`AN6k zPcofKckKPcWz++2qJr`*YBVLjYx#VrMX3R%)A;X1f~IgLqaUB3g6AD7T9e*03!?hG z0jeh5P<_}RRq$!3VfT0N^%q!?{6|#Nm%eWW_zB&Xz<@3sNkRqKjJjd02X;axRA1#q zMR{3N6!*ZyI2rYTKkz-iMa>V-AKLlpAK6DVwNbG#2lL@&tb{QiGyaum_}GGODQYTB z{={lp4z(yXMcsHJYOy(jo$y7_Mo(?idKlI8XRszdL^XZEXV&EPP!;Zg8a1&Uh z>3dY4$9ZqV_6LkbKG1-K`X&-<;cuvSryo##-1bALvj_wD98-O?*JFLM8Vp1|U^GVH zQq=fAfO_C_)CBYaHKiB-Y%SRlxh~*LB%vl+iE7FVSOQ<6hEMJ=juhuekumkx{SQ0PcR7~>Cf_)9v(E16py>=wLukN2bRH`s8NzOs>l64ux8LxsPSGkn#XOS7O3lnU>uGA-$-P`&6o=B z<2&?3_qcDnzu_42^+P;v^xsAGeORc+jp7K@^~F##TPIX-PR4e)9Tn|qJRbK$=aQ(H z8;t=Cr-dXm(Hua{)%Q@*nL5nll#W6$p>EvMYc-sOP01g`x|q(#1EVq#VP;$!?r{cl z{8UVPP>om~cN5wcHGj-T{h(o6EKk7w;n0QH9yeH0#Pzs+n+a8*(inj)QB(9#RDn03 zYH%496EAQyCXVOfHyyky#!a|5zCEyE0=sWtQ~{=-D!3s*z~c_HJseOI7Efpe>5Ll3 zeNbJp4Ar+suo}L@0$4VYRR=`Kt5ObxpnvOw@>_k3FRx3~!4D{!~ViI~GMYdpHqw@Q4 zA?D0ZlSgA&MGdR_xwtX;xw)<7qU85DB{-f4k6{Ch7M+=}fXC@gK{6KhIM1Up&lEAo z7w7&QA5e<%UzZCWm-e`C6pEK&>g0tOWi6V=lw;U&;a*hKzNlbbk)fi;=|#Q_mgD@b zs2?CYm92>@RsrE<>k8_<9 z>eeJr0rJ$cIe&K@nvCNK>w4Vpex~DK^5OM7?iU&3a3T3es8w)OeUEbw*P~i`dIOJh z1>-if`Q=%F#6}Jz`^oz1G?pfxuaS-4zPOV7V${S_wXw(joxogdPri5)f{6;Oz}e)F zH?`r_u9?UExP26^;`Kt!xgH<3u$Ic-ik2iF2(RZQ9%B>Yp(k`Dv(`ZhO$vsPpcl z#`zc2a7@^TMF>k_YV3>=I1zus_1F@_`+6Mqzx;}oL@i46Kh&o2NIzTElJ)nvTdD3C z$?<2H7wZnN@&VL+zM#6O+8~eP!7ZryU=L~n3Mcj^1bA>}5@8(J zHrPtM9~C?gP*ZKpUu-laLDi^w&>pA>X&NTQjrbls1IuFy3--BwYdzcoJ47V;Q zhMJ0-1@ps)Gyc_g%Q&DL??p|OmrygAe}vUM1M0jcsHwSk@c0DOT5te${S~Z)FL5Lm z8X0U+)awsW^T-#}jGAv0O|Pl8?I;^w^HELo996@2sHTlF+HUYe(A=nvLm1E@B)!FxiID zB~;C#PO+frfroj(c2raUI?dyJ$5S{54k)URV+6)pZo?}l>H#&d z49-IJyYsJREnr6!iJIZItTS(-&JSI0&nbqQ2O6Ma zBQiiDI*DH~K8`{~?R?Z~wiOjD_fZ$b{?iJS71hU8Q0I3HIvmxdi%>1J4u8OdsEXc2 zb>Y`wKH%HnaerWt7Juf2OE?6pZnS;;Y19KAqiz`eFS|ifRE%UmO{KX}P1_0;Y-3R^ zu?f{>*HKOW7RzJ2O>Qg%oJJ({b=r)eM{yGQuc)orgw56h|822%vnjUPgNI{&&YOg) z@!_DCg0DY7J@6GO*kWz73M57aX?D!7@!yez`fxF-rsq*H@Df$y9NVqW%A$g)F6#Au zI0m<&g0tKXTa3n_f^Y%W$LrVy^X;^?Vkzz*{}J=@d}q@x`+(somLy+rw+*NMs2JFX zW$^_j#=LuM2CIsV$gf1jNa$XMD=n4_b^fq@_8#&WYOc?+-{bz)WHhRP5q~rOtC46* zLM^Zw)ps{hUE)1p(OU!?lJAAuoSwu6M0?zW9{2OXjz?@_8h|e;zzWpna@H{m(oM(N z>2TczJk9YoCzwAl|0%}*LlSY%*yi%nS>D-j;{J0U=RE#=-nLYgE_mGE0UW{>9Pf6K zZ8X=N#W}d@G6ld2|1di6=c`lzr(Lrt`tWs+bA#)e+~9arzVN*1aep(~^>)DHT;hdn zci1Ll$X#2d{=~n?|8&pZ2gJVbaev4A0`;@I9uIAw|M-!|{W0q3$F?dqf8udJJARZ+tn z5AH)vKu1y6-@`H5|Gy@&k^_UHhq*ts$P^Oh)SxEsuq&4Lgt>dX&0%26U4Qucw ze#Mk{JzkhQEWe`G@8a>p+z%)^V|Mb}P!r8#{2zJ}Sb+wiTI4s>sM?5n&JEOz`4|&v z{3lIlO`AJtZB$eBLfv3IswVR=AO4Q1@fyCy@2H9Cb)qmgh!Z8Y`~8jTg8R4`Kcl*4 zLlSF&{z=1}KmiWyA)&tifNFsR$->;#tu?BTm!V$Yf+z3-Y86}ZgB{<1s?lj|i&2u> z@klH|ei&+IJ&2mnZsT~2ks>VMPCRo{SPY!R?7Yx7WtgMIWO>jtSd;uGbXUEIFn7l5 zh%Goi9n0ef)XY~RRhawL?Qqn7{~qf49{NG1Lxb zKWZlYDQ%cDAIG6;nl+uZR1?&VRs}tXYVupCW%eDa@1v!suQ40y{IaO7Y!)D)pqhpX zj{O)9Poipi2i5dZGFVWh2wE2PdMC_`V{j;LLUmc8jCOuy96`PZ_GHw2!u;4Jv$b^K zw=7}KSWY+}v_;l1cW?Is_i_AawlMdDgmKx!+>dIC zQKR8cRPdh0CU_I$VA0$byw#Bw2{=7SXrdX7YTC`g6CPm%d4C@3nyjdnsEKOgE~ruP zE9ybZP&MC#D!?IB7u?5+_&S&`lGm=Qh{QilNJ35jGpdAxQOoRPR1^P!>a(M$;JSyo zFmt{zcT3hdXdhIRPet8tEtbS>s8R6+t6;1A_MCN?LF0cPi3ofWe8FEJ%>A@GIqE@G zQ8f;rf@m66#?`3d^breTx*zFVY>wKH%s@r|HB`$cC>Z8`E>I2?)Z;LqDRU2r4EPpR z!{mi5s9K?>(($OK{1Y`ScVioT6nwo}VY|Kusw-z>dpv~wFjtW<_w&S6n2vmsqSkW7 zi!%NTbD&Z1zzkHMZopc25%u6q#cT%5kE&^L)YRM$)#tPEA^w4C`YFY&1?Hpv)y!Q~ z3&t#Iv6L3or4>su{`J6C!2`olyVixM3l8B9ypF|icBwG;Q?V9*2Y1EY6plq01!4|0d&O^1p->8B<2?P^wgF5Bx!G0XT38_$T!RBK_e2sb^P^G*z zaU|v-e-;~H)CxA&H^ScJ<5je-8G|a|2F!!|P!$NgBcT~8W+iKyA5l%-1T_(LK)pT) zb;0tWdxBm@73g2o`Qepq^hBUyq#$Z)UV!Sdhgc0gRoqb!aO#oJc#RA?9CgE`s3zKj zdhjt+pWQ{>_$$7`R#k1^P`6r``yMcLb=!~}M&0KWDrldgMpOJ6*23j5t;T<25^BO< zP$#a(%=i!L1|c=W-0yHwVJdR3LHQU z*XyXR@YW;G^PR*b6l`Ts7yg7l;Sf}bucLxAR{b#dyP@2u31tfw$CM51^=7CW4+-Wc zVOjDUQ5E`xnnBYw40GQPl)^ws4lE*}KDmop*K_}5O5=cE2>5MHD>&4SWMYa08CUcbE`=X<;>A4QUZ`O=8r604F%_=CDtH=oe)2YUUJg`&N@HRS zbRZEyVkjy&mg02Wi?y*qTWiWys0Up}HR&@{mn3Xw1uTf_(+a2b#xEA{1~gkWh4nMcR%e50)d}5+iU$(4$z9{8KE6`Pzr^=7Z0Iv4-*;Z2}vPD%c-b z5Kp3lIINSc3HdQT`TlO6@i&n~Dh^ygUHBfWW75ucgAS-@UW$r^yQoz$aTjZ`BG`z0 zLsZ}Yh3haz*D&{!m0hSAuzNSV&sHoz{sWfM_|Ms$cRJV;)e?7cAja&$o(;#M)@!Gy z#Ymo@4RI2O1K1Md^s*N0g`>zX!9keqXKTq-=qG;y6%$u5kd(wz5-l+Ff3|~(MD2!$ zqFQD-#=xT(iI=e%mh4Sm(*=ug8u^`lY_+W2H_TZ}K6bw_rycG=jjD|OEmracEz_Uz zUx62Db3j2e7Zn^^Fa@5*BKQ*3HCYGPR9zhv{Q)e2(@<-}MN~`vi;c1Qz%ciF!kwrB z=N@F+?#ZZ$>hK`Ozxp}~=EIt(8;rnl_#QRRhYb#M|2D*2)Vxse7aI*_umvs90AF%^ z+Yn|r@+*hh#^l4VwrqQcSr@dz_MG=Cs)8>9B${(UjNxXN5%x~#)JVIa->5L>7AMZZ z<2Yb+n6nHkjQyp_8c`jwvP{UKZFXO5axc4&>siz+`xMh zYKmTyY$95X4anaPT4=HrWD2TlHlk+C6XtqBh3BxK0VFhj7vKg?h&n&a{VH|S0<-Hvt8vCf z7X4)~Bj@+R2%Lwiz%EqRTt=;Sw}Z$3Mb+51*v?CYn#yxwKz&#?cwhjksV1UoxCM2i zhp11l-eE;7ywsw1*l$*#r5KCz&ZE|ldl-lFqWo_A|9H!69vF{m$#)om;maBSN@QDZ z53Giovj?IIvKjSG<|e8u(yp+zp)P7Zn2TDhcB7iS-pVld4;Br@G34X_5$1kLwGdU& zr>NnZdX*I<+bYJlnzA$pG#)#lnsQ9=gx^q6za2~Cx!~)GR@?DhsF|`NY7}%qHSKuR z1hhDq|BjmTsydE*y6#rc0>eH^yUCZ_9Hl64%K<0Z4(r>Bs8P@l zRefbKe~ria9MFX;P~-e$@PzO9JNaa% zEsFPHBJw9t!FnHc-FMWD)1I-eD2;kR9n^%=4b=j}QP)ii9$$Ng@vj}wf#88#LEm8q zj)$MMXzq>b(^IGdd_V~pqD?uqLAL#Qr{dfvLQ7HTvM#1?n~J7R$g)@4fqByw=z z6}G}u7wrSdVW=kGk1Fv4R985cY>tnCs$oLZdY>CrumhMFucPLXcc_BLzif+JQB;d} zMYTwvABi3$Mxko(1!Y}3e_U({t0syV6v+=5p733_;b*B*Q`aepej%f z!>~Ov-kl!7*B2rc2sqnGC>k%}HGGPh@$hx4;eV*;k9EU>v>0lcZI0^OcGw07Vhaj- z7Zr?UZ`m-Njd{sG#|TV)JIwtusx%hR_&-NNQ=$KkZL@2kT4EUL!ab;_dW&kx=y$Cv zvZ2O(A@pNq)O*1CI2z{#U(a~YX1;=`miiTSzq#oC`~N>ksHryLEWCp1^PczZZS^?R zs5pYE@h4O)#C~9XogB5g{*Y=|>Z6VNNvE;#X1#{V=D zL!R2F*NLCmJD$1Nh2ycG2cs0#bmLK7@H=YMoI*X|JI2S%FKqu%9Myt-QRhv;2;7fa zJ@4UKO#71YuT}2+OZ&9@_kY8jx8&QuvQM|0y|xdT3jJp#&-%uKuQsa2KcQy8fvA>Q zfhy=LR875aEuRxLQB}Zd*ctWujsS@YB<`YWlJ%XHv_7iO`k-nu0`uS^%!d~+VHAS) zeVEfdDpCExT4w4eJAWtYftOH0`UQ3W7@sY;e?%2D(33<2i7}|AT7_EOuAvH+UUM6#0Ir(X3gi zMfdmrlSt@=J6I7TLcH!e-2v4^Bd{1ZT!3oQ8=+n&0;74nZWpCRO~rYFR!0@IBWemh zi0bQmsO$Y<_Pm^!oB7A-LPCAL2xH+o)bQDZy5J(}2A?q%ruKT>;46a($+ttb&>&PW zE=P66`QY(HK3ax+Nz~}6hN@U^45%+Bkcf-#Q4jL@z3y_F3)R#$FeMH`1<`zrgGW$( zeHBY!>Tt`q!j|M$VqJ_8!|Q%v*(&H!Y{v0iF}(qI!!aqQ*A12@ScVsh$MU*!{I3{6 zeoW9+sMt6W^dVL!A0@W6SWVPg(hgPN{+J!-peD4VsFu2q3f@?80@n0NCpQzM@VfIxJ=Bed z<7iwNJf1J5#a0`P;CLVWgtL&?3OG|EyzaQ)hnj%iU?LurHWlrl2d4JAA4C>S<8>FO zN!XP0&S7fIoz@;uH)tek`1MDPqKl}Qc!s*Klg?Tuae6AO^}i^I0vw-;S@Afk&px82 z&TJX1kBg!Dv?4}eS5z=fLKW6p1Zz2n2vtXNn8m8AVAO4F9*7VuE?k6U7Q4`K5R4gUVVL_b_75#Ow zEq23#cnJeFNW{r$HE)RurU|H;YzcZAqf@{~sAzwl%WC`qRe-p;tw~Fw@@-MqPeaWg zt5K`tRn#a*lgC=9NFK(&nzR-NwAJc^YRc893lE`c@)p&Har0Wu)1#WOF{;J^?2q%X zHzvvFb(h&msP_f8Q0ITg6VaKd@>>D_EflasSYdl`1ZwpwgsRCn)FQM4Ril@vf_jQr zz6i!7Uk3|etKjkZs0VJs1$YH(W8b1)_e-i1*oqeI87OAKcd59If> zPb4vfIXciK8vm_3dfne>9KlFlsM*Que!su1v(+H9i;e$yn3D4w2JMSmIe%6#U%Q*v z{gq2kR7@NQdKI-#c!e5%Vco4IQ=$9$e?bx&CUsE5t|#iv=pxke8q&jprYE59Sx3n*1+RLHD6X(H+c#aeCVIWw9msIz1Wxx?nR06jXm>3{2X~=K74NKJ9>7 zZbzbi9d{BnQAPQmJuox&BVPbhhds~gGqh4=^+Om!A&G^?&=64Ph#Vx4e z@eVaH`TE$ZmJ>Bx>R?V>j0(ncL7$?wN&DHHpR2!Z=^EfH&W}W`8Bqs#-G9cD zH9(>O2TtJ}^bE8MmS73;8&ECq2Guoj2iYF41nNeeP&Jtz^iR}`c@k6MQ&a_GP*HVN zO4NOFqV5}LN5=^Sc<@eK9AlE2t6ZiTAQ zLTrLXhS+mvqgwJeWU3E1|B=wXKgLkokrYP-%T`nkpQCD+=vV9WJgA>kHpU2!?-*t? z<8@TcvJdyVAF(t>)$lMbjmjq|sJH23MtPkBm3iTc-&al?O~mje=R@ zEC#+}CGx4pTY)>FqI*7e!xO03DK^3D{xYf-<|ChNqK%eTsMq_Vf_@yjzyF<0LJvNL zD)B{B@V&u?m~@g2r(US8nu;2J3s5!w3rpZ#jKs8)z3z9v<4`l>KdAA47uB_&P+b^z z3gh4HYZAI~Rn*q00ct+jfW7c6YIQ3=)zJM9#u48lZqgUB{@)T!~&$Qa>zRBH->aq{0e{+^Tu*OQ( zcda$?Y23sK{&n`?eW*QK;`Mf30JoCAi0YDwe>kMuoQ8+&>=G1vju1e#16ca+~dxmZ7%opHMN8dAr#R zN0Oh4g)#XK3)+VGBL(S-#Wnty@3cO7ge5sK(=K~(M{G`hJF4%0*lkf=0~JK`uqGZs z%@fJ?c-_yE%3@RUS5YxgXfMIZ13O^_^1tr$y5E4Dz>FIIMfY1&x5Xu#um*p`T7O#; zjlrShk6=fvdcf;`61o-3l23ck>wf9f0c()|j9IbDA?v!nc!c~dR0UTawz>ZqZsYmR zgd-O1m5zGde;nEdOK`mSF>C6)UAHEjg@w6Ms~gs|3vOBrgx<0nHb4dETwKC=QEz+Q-wFMG$9BKT?|Pk& zygmf0@cMvz0o(f>xzF(6g&q%VBQXM}kdOM18*}1x{2%#$9`Qi(H6MH3uV$7!@j9b8 zek^F+r&f`hsI{TgGy6=bJ1SPTqwW*sxlL4k0wj8IU_Z9RA76OgACt_%pUJ;Rty-O5 z+6>tjHPtS_w73~HzOSLib((*z0!^_F`75Z{$^Odgeyi3QFOc`WwuvZki-d+r%>S(E zOJH~M15r1AiQ_T<8;kA(s6{E^TdR3DR8W0F9nbyFX40`ZjC|O8i;eN9`+mV~*yV$3 zz{&K{>n!BJJ=C}z_Q__zd)SlwoX=kO50J$A;&sl^^y^U*%CK)<_k+Z}s5!s%cUyeM zV+uyadem>cPDk;%^T_R}W~pdC_g(c8ETi%7MEALW>98#NxxgFZbNBTLLVfPL)+Jby z)1IOVn9bvJ|06SF@gVuRVRl_Bug{(9yZU_Y=<)b{?uXc2QB(E}9E>x=ea;(962s?g z((_}+vSrx zSHY^NDS0_6CUV8&@1K??(Tzk=+=#m1HEIToi0^a9aYNJ$xEXch!U=qCkoLj)>Z zm^Go#Ift)MF?Ar3&;2X-?{NY7NlAR}@XVFeuA7mR?>{u7z2tzV)+Wh(?$mk&HC*!k z;B)8femH^reyoDEliLjz;xh6dFb^(FVSRia75({A`kWPX>1fm+Ajq4FuEx}9t&1O| z5BQu?oX{(S&s`KYXS5q-$mDb1_cy||+-N*%JQvICb9cF&gHAvlUyuFq7AiO!XYsjT zG|a>vu`X?KoYM}!>C&}EBSKN`2UKL*fYDe%rzWGzI+Y~rVIEh z`Fc660#~sa`O3L`?gaG*YIK~#X_z9n#o9((Nj~t7gqmtm9-lkMzr*?D%jEUB6VhF5 zM7~@;S_&7Vy5>HPz%2QF?gx(>ur2v!1$^$ub_Y=vIP{~>-5nP#=yT`%HK>W|H8NiW zoNR@xNk*Xh@E~@{Y}(h8iWQi~8JOyUj*5ZIWVE@Y1NL zUw}z5V{x-IMriyuC6R^~0=NQ~;9#s;!smn%8z=EE@)t__+!<_4DJ$StEWqoXO8cA? zxU`JV-Q(>q>vQ)5pHMU6m~s}h_favDyS!C+4QAH(e@;SCow$NEZ5vcg|3K~8QdP7I z@>lY?A4dO-gLr)|&ctMutuHsC`o38e8?L)?Kl##CEw((>eC`INI4VddVqiFl<0J-P z?dsMyr%_GsuiV-eKrhfu?{OiiErFQ5O!bmYs|vYD|nYV&#=lTh%XwSDgIfacZl zx&QsIwRM?4IKIE0&;88FX~6hbUlwZMbKihW!hGcCVq-ju#W6=ipZl-dy5b~m@a`v{ z`^I8qW15iH*CS!)T*H|75LJP9s1L6)H1WCrsHG5Ajn3lL)aU+@yVflO)+g6n+RW6j zl?}(;sBvGkwa@*L>2uVE;g7a<-iLNRXE(1mjr6(8YKrzY$8X2y9RIn4bz!THKKp0* zs0tnIJtbH=n!99g0&pegXBb;@Wrj zxqmg|N)IL^@(X%0(cs=*1TW_OpVjcM-adB+mAwy5&hcOH1ozqA*XJA}9~jloW~iD2 zY##U#LUs#L3abB@e);0IA z5BZmQi}`URX2jj7#pV$zc9M?u zxx3#QIGOxFR4l|EXNy}&Y(u^)CeZl5Kte(EG-$%{HXoG42#&YG)Hob91Fk?#FwZa? z6HajdjJT5q6Odnmdf-;n=y-y=F!n@S%uZo#@`)!g+*I)+xj0-T$5}*55FZW{@ z`4vlThx8|^YmQ<(yoIXxo25jrg00$b)|5?9vCtKDgW0GHmS8sAjGAcfqt5#m8(^~E zeeQ=;y-*dJh}vT9MYW8x%d-2W*cTPQx(|Zowk> z3{|j9f7p%cqk^&zHpiu?mI_&AqbCU#Ctoz^K-3Jl6Mw>_tF`^6w|^#~alQ=|&5u#b z@n_TnQm(O@Rz^Lb1*#xFV_95+C-E^R$1Q8^J{M8z{Uc0)(bickq(;p%h0y))e^m;; z&2KD3PF#-)&V#7Fy?_ZZ>H(ko zS#AKD|qy$)-l=ZH_GlGs-+s7unDI#YS^7ZJvin`o2W9QMoC8u$7$FN7oTMO zw;>Vll+AE`QR6b|X=|zsIE#D@R6(AgcCV?=*aX!C6_m43!*Mlg6zxY%Ft;#07CLLM zw?GxFH!29H1W4o}u>?!v1+0dN&RO)fMfKe;s0(LdZ~QBGJllDzSy|M?)D;V2KU80D z#0b2Ony|j1S~}AOt4N?K2~D9RQ8BO%RpN7~;r9s@ROv6;g=J7TibMtH6x58j7<1q` zR6(O%vhz}+Do_d4QY}%>`=1>TIP*#9hMQ31^fu~&&SlG|Ks9Mm%!@yvUY~$@a5bvS zZlk&&^om{g1M0fMs0TN|bl3^A<76zP)n+dVO(IN5rzQM}mTegU8M#aV+)N_tu1bS~X{)>>vOhPBL!%El# zo8wMgj4AF|OzcH%HXmUG`tA`UnB~6B6ZIdku^_(;vtf=$KIakFH+k%Hzl<*Tgkb0R zb)3%eSkD;$T}k}@jO{WnBz(?07##b;=l*N;XD=<^{FTMXp4Yaie2O!;Zqk3YE6()B z=l(O@ow$YL{odMe&hyUa{waqY*qq~Y-`lYLit56G9|CM9dElxK%>8)xBPU|!&(?=y zz7W0SkE3pI@2l;I%6_v?Mpoldj{Cpc`4>^cGZ7!!X@Z%6`d-k9;&=Yj@u+_Hy`evv z-~Draa|0ywWpIJ$e&;TB3h_I~v4+R*Ms;$p-|gE{xSt#B#xXe5=XVx!UMxT7lm9c^ z@2&}f7=Cw#n;3K+YILkb1?ef&z930Vzx&BbpbCl19GHe0_j@oOJ`CoQ$KpW*M;X+O z9wza-!zw(f-(3STp<-q*YKlIFT6XVY3H*+FV2NaAPb^9PcjWbe^N7Sw4rKbl@2=+$ zu`u~;$?bynsQ>ocTvV_oNXdi9*FptX#|Xds0Ysfte&;NCC$-;wvw03HkbRiO?{02$ zruDls;}q1Qb^WQ5bpQLG+em2Ux`-w48K%Z;S^e&m zS_2!9AB;Nh3dX{hsHyck#>GV0tdBFJTBsMQLgTXg-Sz%DYDWBu3fiPO{Pz3*`y@2i zhvl>zroFroy)r50jh>+a{F!hMZMlR=uiwLKMRxNLeztIVPRgliUY_e zD(DY5ucEV`FX(sPMq|@i%{LTipVt30>6Y?`ESzoTJV!`xCSiKv~@8$5;O>QVq&HesNi-(9Cm*7v*H z;tuHW!U3$zi3b}{Q}S^e+HkA<6HQHiF>d7bEsgx{uiiU1=0O}k)x_`q<)l2#{O$)B zVa@&S&-b@rYhK^c!dfa-E5G~3B+!CH6>hW{OJela7R}|b1^Mwf7yrd_IIfM~{ju43 z>_NV7Tfh4S#xtBmzC}B~I|KfQ3&?kj^t*q0$=}{)z9*pJ5sN*wycTz*wxi-+iMo1j}&#O>BZ`d-&b0 z-VoI0^%3UM_%Ghm@4l_>j@k!Q>}A8FH|8K;>Sw>ZCF_BIk>87>u=oEgxT5s72Nc7p z9G{EjDNwFH_MpCfEm%+EEnfHbv#xuB%Xz-Dpg+OE1r-L^L^FG!-|5Klh(Uh$d%m&Q zntWk`wKguqXq^8BHD!Mo?03J@N%@P__$?0Qc%dQIh{EmTq zocNA}nl|eQF673Ms7>Uvk=A7CNBP~)h{vLa&28+3;iIi!zu;%`4^ZR!&KSRQfcrEX zYg2c+@qYK?yZI9=`qNGFJ9{|q?Igy(ntan_i~f7qf_(8Qes{mV2y2i(hHLPLsdnKp z)G!-6&4$+{)OCfY``u4Q=V64-pJ6Rp6q}LXkM%LlOuze^v;H$_!q*(gHp}mRvJo-c z?|$((0Lyax0qWgq);WIneZUCpNj}wFYuah3mPkL(@9t>kU}y5V=lk8y`4(b3^5qx! z-7l-w;!5(l7BYjjG^~?S4e~9$xN*n)?fB4-WI3!-h;=^@cu`*^~W9J3>t+l31yw2}_tkw!ufZ4bN zKcF5wcfH^JPqJ42(_*FJ25ahRIGhK^-)J-5oK4ns_fVrJ=H}qAL=DFw!Tcd)vm0>y zTik?`8FfNM)Pyu1OW{>i)Th{LLD&Opkw1hA#-!V<<;tQKnc=9QUV9Wvn0mfeKD{| z!rQ2z{9&Js_jafa%0g62jM`62P>`*tDg5i-j0TPWcn9qPcTrzJI*0u3XF=hI$&(#~ zb2&a5^}q~A{f>wH2-G4o;h60YF5?dJxsLnYKe74(^^Rxd30u@oqY4`Bq_tQv45;P- z5^3=Te#dyHtU&Qj``tGhIk6hYYoUhYA}o!sQPG|MjLn?$Q7v#BRr5q={qE25oO3ps z`lFt+7xUwTbBurWZSwOrrFKVcpFd-H?0La95Zh7Feg*Zw{TKc2Ha*KFTgAp+wqbb( zNAh~TE4I$x#0TU%{9_YO&8xP1PIb*%c=fdaK}S=jyv`bct#A0<-)a=OWkDBu+g7We zP@e2t|4DDY9QQwpYBAn3%g@=UYLS;kHwUjiglvJ|1=?pvh-s6MWX3X+b&<2_JaH4(MJScmG86rOO} zf+-kv5i3 zc0&d26jTeXLpAk2)Qyj!`uG}V#a9>=Q~2z>R7h6_oXjLN{CcAvFh1z4pi5B&S{rmn z@c5CS=Y!rv72rwG|4`R`3F`G*J|SxHN`slT{e&p;og&>|0yRPvC=xZSeht39B6xgnFn=9Yz_%EI5wWa-B~in=3cCOO-?k)l;{mAgI|}u{ zwWugRgld_K*aL5)3R)+&9dCk)kx0ym6HwRhK?U<=)cG$^&v}pUFnXNufZH^m<5&q} z#0_`%Y}ru{zJsCo3Y+5xRCG6t7w-NZXd|j%W#U^zKYnjz1ky5JqE<-!tJ zLDD8*{A@{4Hp#Bv$eW)QyW^8LWZ2ei~}xT8J6(UtEGo zlA1elC;1l1!rf0!Q~nU{zFiM&Ceeu(Dkdiwa8uCyDZ-s+%HzMdBxSh!_duscguC15 z`l;;vx~Xj@yMXHB_oyy&(pU?6P%~s@bfX?M;cZ0zR&>BQK|;&r&UDtq$AZ4c%(_o{ z3(9J!8}-7%xEyohJyeS%$Y3|lgld6OsNHi%)ct#-T51?>#)(*3Rj!aRoP~hB7~bcF z8=1r1&uah967If@-jX%keQ#JKn-ydxYS`^UEw6Wj`4rhL2uq_1&=%j|1T0J6|C%G* zU51b540ks$S#yOuN7dv5NvLTmg7lA`Yz< z?!M(Zf%>8HS6my7^J;{IVJ1Q0);}vYu*hWQxCJaXk5Q+LQs(aILXKGaTmdz|Cs>BRu;k=67!kuMNcqi}cZYBP+hXu!7)D9trk4qfLum z?Ynl#R;ER-HiaT`=ggfmTh9F1a^@ykC~vMTIr-m+$VDNWBNv4(4WALGnI}=mzzh)? m`_Fil(DVPvPM1B#$=@!wjtz*}ftUk`If0mKyWBc%l@kD3fs&{I diff --git a/conf/locale/ar/LC_MESSAGES/django.po b/conf/locale/ar/LC_MESSAGES/django.po index 75cc4f5f81b2..1a154de88d00 100644 --- a/conf/locale/ar/LC_MESSAGES/django.po +++ b/conf/locale/ar/LC_MESSAGES/django.po @@ -16825,7 +16825,7 @@ msgstr "استعراض عملية التقييم في Studio" #: lms/templates/courseware/progress.html msgid "Course Progress for '{username}' ({email})" -msgstr "تقدم الدورة ل '{اسم المستخدم}' ({عنوان البريد الالكتروني})" +msgstr "تقدم الدورة ل '{username}' ({email})" #: lms/templates/courseware/progress.html msgid "View Certificate" From e3ef74d7bd9a31edb724d6b3fe26fcf20cf925df Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Mon, 11 Dec 2023 16:04:57 -0800 Subject: [PATCH 110/125] use temp Overhangio fork of py2neo official py2neo is EOL and releases are gone from GitHub and PyPI --- requirements/edx/base.txt | 4 +++- requirements/edx/development.txt | 4 +++- requirements/edx/github.in | 4 +++- requirements/edx/testing.txt | 4 +++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 17f3e0db2b80..b97f0c283174 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -181,7 +181,9 @@ pillow==7.1.2 # via -r requirements/edx/base.in, edx-enterprise, edx pkgconfig==1.5.1 # via xmlsec polib==1.1.0 # via edx-i18n-tools psutil==1.2.1 # via -r requirements/edx/paver.txt, edx-django-utils --e git+https://github.com/technige/py2neo.git@py2neo-3.1.2#egg=py2neo==3.1.2 # via -r requirements/edx/github.in # via -r requirements/edx/base.in +# modified from https://github.com/openedx/edx-platform/pull/33453 +# for earlier pip version in use on Juniper release +py2neo --find-links https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz pycontracts==1.8.12 # via -r requirements/edx/base.in, edx-user-state-client pycountry==19.8.18 # via -r requirements/edx/base.in pycparser==2.20 # via -r requirements/edx/../edx-sandbox/shared.txt, cffi diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 2da53ef6e013..126301a8d36a 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -218,7 +218,9 @@ pkgconfig==1.5.1 # via -r requirements/edx/testing.txt, xmlsec pluggy==0.13.1 # via -r requirements/edx/testing.txt, diff-cover, pytest, tox polib==1.1.0 # via -r requirements/edx/testing.txt, edx-i18n-tools psutil==1.2.1 # via -r requirements/edx/testing.txt, edx-django-utils --e git+https://github.com/technige/py2neo.git@py2neo-3.1.2#egg=py2neo==3.1.2 # via -r requirements/edx/github.in # via -r requirements/edx/testing.txt +# modified from https://github.com/openedx/edx-platform/pull/33453 +# for earlier pip version in use on Juniper release +py2neo --find-links https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz py==1.8.1 # via -r requirements/edx/testing.txt, pytest, tox pycodestyle==2.6.0 # via -r requirements/edx/testing.txt, flake8 pycontracts==1.8.12 # via -r requirements/edx/testing.txt, edx-user-state-client diff --git a/requirements/edx/github.in b/requirements/edx/github.in index a0ae6f9b0ff9..80f7b2f2cc96 100644 --- a/requirements/edx/github.in +++ b/requirements/edx/github.in @@ -59,7 +59,9 @@ git+https://github.com/edx/openedx-chem.git@ff4e3a03d3c7610e47a9af08eb648d8aabe2 git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a#egg=MongoDBProxy==0.1.0+edx.2 -e git+https://github.com/dementrock/pystache_custom.git@776973740bdaad83a3b029f96e415a7d1e8bec2f#egg=pystache_custom-dev -e git+https://github.com/jazkarta/edx-jsme.git@690dbf75441fa91c7c4899df0b83d77f7deb5458#egg=edx-jsme --e git+https://github.com/technige/py2neo.git@py2neo-3.1.2#egg=py2neo==3.1.2 +# modified from https://github.com/openedx/edx-platform/pull/33453 +# for earlier pip version in use on Juniper release +--index-url https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz # The latest 2.0.0 release doesn't yet support Django 2.2, this commit from master does -e git+https://github.com/jsocol/django-ratelimit.git@72edbe8949fbf6699848e5847645a1998f121d46#egg=ratelimit diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 879892db16fe..f3e95dd3404e 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -209,7 +209,9 @@ pkgconfig==1.5.1 # via -r requirements/edx/base.txt, xmlsec pluggy==0.13.1 # via -r requirements/edx/coverage.txt, diff-cover, pytest, tox polib==1.1.0 # via -r requirements/edx/base.txt, -r requirements/edx/testing.in, edx-i18n-tools psutil==1.2.1 # via -r requirements/edx/base.txt, edx-django-utils --e git+https://github.com/technige/py2neo.git@py2neo-3.1.2#egg=py2neo==3.1.2 # via -r requirements/edx/github.in # via -r requirements/edx/base.txt +# modified from https://github.com/openedx/edx-platform/pull/33453 +# for earlier pip version in use on Juniper release +py2neo --find-links https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz py==1.8.1 # via pytest, tox pycodestyle==2.6.0 # via -r requirements/edx/testing.in, flake8 pycontracts==1.8.12 # via -r requirements/edx/base.txt, edx-user-state-client From 8cc191852cf057bcdcde6161505c2ef8f1c8984a Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Wed, 13 Dec 2023 20:52:33 -0800 Subject: [PATCH 111/125] use py2neo-history package on PyPI to get older release overhangio fork is too recent --- requirements/edx/base.txt | 4 +--- requirements/edx/development.txt | 4 +--- requirements/edx/github.in | 3 --- requirements/edx/testing.txt | 2 +- 4 files changed, 3 insertions(+), 10 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index b97f0c283174..775e61c4ed32 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -181,9 +181,7 @@ pillow==7.1.2 # via -r requirements/edx/base.in, edx-enterprise, edx pkgconfig==1.5.1 # via xmlsec polib==1.1.0 # via edx-i18n-tools psutil==1.2.1 # via -r requirements/edx/paver.txt, edx-django-utils -# modified from https://github.com/openedx/edx-platform/pull/33453 -# for earlier pip version in use on Juniper release -py2neo --find-links https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz +py2neo-history==3.1.2 pycontracts==1.8.12 # via -r requirements/edx/base.in, edx-user-state-client pycountry==19.8.18 # via -r requirements/edx/base.in pycparser==2.20 # via -r requirements/edx/../edx-sandbox/shared.txt, cffi diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 126301a8d36a..616984a8598e 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -218,9 +218,7 @@ pkgconfig==1.5.1 # via -r requirements/edx/testing.txt, xmlsec pluggy==0.13.1 # via -r requirements/edx/testing.txt, diff-cover, pytest, tox polib==1.1.0 # via -r requirements/edx/testing.txt, edx-i18n-tools psutil==1.2.1 # via -r requirements/edx/testing.txt, edx-django-utils -# modified from https://github.com/openedx/edx-platform/pull/33453 -# for earlier pip version in use on Juniper release -py2neo --find-links https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz +py2neo-history==3.1.2 py==1.8.1 # via -r requirements/edx/testing.txt, pytest, tox pycodestyle==2.6.0 # via -r requirements/edx/testing.txt, flake8 pycontracts==1.8.12 # via -r requirements/edx/testing.txt, edx-user-state-client diff --git a/requirements/edx/github.in b/requirements/edx/github.in index 80f7b2f2cc96..9a8b2966e725 100644 --- a/requirements/edx/github.in +++ b/requirements/edx/github.in @@ -59,9 +59,6 @@ git+https://github.com/edx/openedx-chem.git@ff4e3a03d3c7610e47a9af08eb648d8aabe2 git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a#egg=MongoDBProxy==0.1.0+edx.2 -e git+https://github.com/dementrock/pystache_custom.git@776973740bdaad83a3b029f96e415a7d1e8bec2f#egg=pystache_custom-dev -e git+https://github.com/jazkarta/edx-jsme.git@690dbf75441fa91c7c4899df0b83d77f7deb5458#egg=edx-jsme -# modified from https://github.com/openedx/edx-platform/pull/33453 -# for earlier pip version in use on Juniper release ---index-url https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz # The latest 2.0.0 release doesn't yet support Django 2.2, this commit from master does -e git+https://github.com/jsocol/django-ratelimit.git@72edbe8949fbf6699848e5847645a1998f121d46#egg=ratelimit diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index f3e95dd3404e..ef360465d765 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -211,7 +211,7 @@ polib==1.1.0 # via -r requirements/edx/base.txt, -r requirements/ed psutil==1.2.1 # via -r requirements/edx/base.txt, edx-django-utils # modified from https://github.com/openedx/edx-platform/pull/33453 # for earlier pip version in use on Juniper release -py2neo --find-links https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz +py2neo-history==3.1.2 py==1.8.1 # via pytest, tox pycodestyle==2.6.0 # via -r requirements/edx/testing.in, flake8 pycontracts==1.8.12 # via -r requirements/edx/base.txt, edx-user-state-client From a1d67c8fe366c24b060d31d6bb4dc051a2d88b4d Mon Sep 17 00:00:00 2001 From: Bryan Wilson Date: Mon, 1 Jan 2024 20:03:13 -0800 Subject: [PATCH 112/125] Sync Dockerfile.tutor with latest server-vars.yml for Tahoe prod --- Dockerfile.tutor | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Dockerfile.tutor b/Dockerfile.tutor index 26a54cd5f3af..fe47d6ea685d 100644 --- a/Dockerfile.tutor +++ b/Dockerfile.tutor @@ -17,23 +17,25 @@ RUN pip install -r ./requirements/edx/base.txt \ # Sync with `edx-configs` `appsembler/tahoe/us/juniper/prod/files/server-vars.yml` RUN echo "Installing pip packages:" \ - && pip install openedx-scorm-xblock==10.4.0 \ - && pip install xblock-launchcontainer==2.3.1 \ + && pip install xblock-launchcontainer==4.0.0 \ && pip install xblock-prismjs==0.1.4 \ && pip install xblock-problem-builder==4.1.9 \ && echo \ + && pip install https://github.com/appsembler/openedx-scorm-xblock/archive/refs/tags/v15.1.0-appsembler-tahoe-compat.tar.gz \ && pip install https://github.com/appsembler/pdfXBlock/archive/v0.3.1.tar.gz \ && pip install https://github.com/edx/xblock-free-text-response/archive/4149cc450.tar.gz \ && pip install https://github.com/pmitros/FeedbackXBlock/archive/v1.1.tar.gz \ && pip install https://github.com/ubc/ubcpi/archive/1.0.0.tar.gz \ && echo \ - && pip install course-access-groups==0.6.0 \ - && pip install figures==0.4.1 \ + && pip install course-access-groups==0.6.1 \ + && pip install figures==0.4.4 \ && pip install tahoe-figures-plugins==0.1.1 \ && pip install tahoe-lti==0.3.0 \ - && pip install tahoe-scorm==0.1.2 \ + && pip install tahoe-scorm==0.1.4 \ + && pip install xblock-grade-fetcher==0.5.0 \ + && pip install django-manage-admins==0.1.0 \ && echo \ - && pip install https://github.com/appsembler/openedx-completion-aggregator/archive/3.0.3-2021-may-18-bug-fixes.tar.gz \ + && pip install https://github.com/appsembler/openedx-completion-aggregator/archive/3.0.3-2023-mar-27-revert-use-of-task-track.tar.gz \ && echo "Finished installing pip packages." EXPOSE 8000 From 3022eb75e62e55442ccd2a87b71ca307d11ac4f7 Mon Sep 17 00:00:00 2001 From: Amir Tadrisi Date: Sat, 27 Jan 2024 16:32:56 -0500 Subject: [PATCH 113/125] feat: Add Mandrill Subaccount support --- .../settings/settings/production_common.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/appsembler/settings/settings/production_common.py b/openedx/core/djangoapps/appsembler/settings/settings/production_common.py index 20f544f0dc83..bdd911e8a286 100644 --- a/openedx/core/djangoapps/appsembler/settings/settings/production_common.py +++ b/openedx/core/djangoapps/appsembler/settings/settings/production_common.py @@ -48,7 +48,22 @@ def plugin_settings(settings): "MANDRILL_API_KEY": settings.MANDRILL_API_KEY, } settings.INSTALLED_APPS += ['anymail'] - + # Mandrill Subaccount Support + settings.MANDRILL_SUBACCOUNT = settings.ENV_TOKENS.get("MANDRILL_SUBACCOUNT") + if settings.MANDRILL_SUBACCOUNT: + subaccount_settings = { + "MANDRILL_SEND_DEFAULTS": { + "esp_extra": { + "message": { + "subaccount": settings.MANDRILL_SUBACCOUNT + } + } + } + } + if settings.ANYMAIL: + settings.ANYMAIL.update(subaccount_settings) + else: + settings.ANYMAIL = subaccount_settings # Sentry settings.SENTRY_DSN = settings.AUTH_TOKENS.get('SENTRY_DSN', False) if settings.SENTRY_DSN: From b131eb054d3a6e4ff6395dc272f8e4d142e4f204 Mon Sep 17 00:00:00 2001 From: Vladyslav Tymofeiev <“vladyslavty@softwareplanetgroup.com”> Date: Tue, 2 Jul 2024 18:56:10 +0300 Subject: [PATCH 114/125] Add sanitize function for redirect parameter next --- common/djangoapps/student/helpers.py | 18 ++++++++++++++ .../djangoapps/student/tests/test_helpers.py | 24 ++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/common/djangoapps/student/helpers.py b/common/djangoapps/student/helpers.py index 3c47510b5af1..4c2fcf7efd6c 100644 --- a/common/djangoapps/student/helpers.py +++ b/common/djangoapps/student/helpers.py @@ -6,6 +6,7 @@ import json import logging import mimetypes +import re import urllib.parse from collections import OrderedDict from datetime import datetime @@ -65,6 +66,8 @@ EMAIL_EXISTS_MSG_FMT = _("An account with the Email '{email}' already exists.") USERNAME_EXISTS_MSG_FMT = _("An account with the Public Username '{username}' already exists.") +COURSE_URL_PATTERN = re.compile(r'courses/course-v1:[^/]+/course') + log = logging.getLogger(__name__) @@ -301,6 +304,7 @@ def _get_redirect_to(request_host, request_headers, request_params, request_is_h redirect url if safe else None """ redirect_to = request_params.get('next') + redirect_to = sanitize_next_parameter(redirect_to) header_accept = request_headers.get('HTTP_ACCEPT', '') accepts_text_html = any( mime_type in header_accept @@ -700,3 +704,17 @@ def get_resume_urls_for_enrollments(user, enrollments): url_to_block = '' resume_course_urls[enrollment.course_id] = url_to_block return resume_course_urls + + +def sanitize_next_parameter(next_param): + """ + Check the next parameter pattern and update the + symbol to its ASCII equivalent. + """ + if not next_param: + return next_param + + if COURSE_URL_PATTERN.match(next_param): + sanitized_next_parameter = re.sub(r'\+', '%2B', next_param) + return sanitized_next_parameter + + return next_param diff --git a/common/djangoapps/student/tests/test_helpers.py b/common/djangoapps/student/tests/test_helpers.py index 655d9b763b36..8231503c8a55 100644 --- a/common/djangoapps/student/tests/test_helpers.py +++ b/common/djangoapps/student/tests/test_helpers.py @@ -14,7 +14,7 @@ from testfixtures import LogCapture from openedx.core.djangoapps.site_configuration.tests.test_util import with_site_configuration_context -from student.helpers import get_next_url_for_login_page +from student.helpers import get_next_url_for_login_page, sanitize_next_parameter LOGGER_NAME = "student.helpers" @@ -156,3 +156,25 @@ def test_custom_tahoe_site_redirect_lms(self): 'LOGIN_REDIRECT_URL': '' # Falsy or empty URLs should not be used }): assert '/dashboard' == get_next_url_for_login_page(request), 'Falsy url should default to dashboard' + + def test_sanitize_next_param(self): + # Valid URL with plus - change the plus symbol to ASCII code + next_param = 'courses/course-v1:abc-sandbox+ACC-PTF+C/course' + expected_result = 'courses/course-v1:abc-sandbox%2BACC-PTF%2BC/course' + self.assertEqual(sanitize_next_parameter(next_param), expected_result) + + # Valid URL without plus - keep the next_param as it is + next_param = 'courses/course-v1:abc-sandbox/course' + self.assertEqual(sanitize_next_parameter(next_param), next_param) + + # Empty string - keep the next_param as it is + next_param = '' + self.assertEqual(sanitize_next_parameter(next_param), next_param) + + # None input - keep the next_param as it is + next_param = None + self.assertEqual(sanitize_next_parameter(next_param), next_param) + + # Invalid pattern - keep the next_param as it is + next_param = 'some/other/path' + self.assertEqual(sanitize_next_parameter(next_param), next_param) From 96c39fa59869e426eb37ecf2a493d68bb24522aa Mon Sep 17 00:00:00 2001 From: Vladyslav Tymofeiev <“vladyslavty@softwareplanetgroup.com”> Date: Mon, 15 Jul 2024 17:36:11 +0300 Subject: [PATCH 115/125] Change upstream_repo variable to actual value --- .github/workflows/report_conflicts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/report_conflicts.yml b/.github/workflows/report_conflicts.yml index 3c1b5e9e8baa..96ff1955f55d 100644 --- a/.github/workflows/report_conflicts.yml +++ b/.github/workflows/report_conflicts.yml @@ -7,7 +7,7 @@ jobs: uses: appsembler/action-conflict-counter/.github/workflows/report-via-comment.yml@main with: local_base_branch: ${{ github.base_ref }} - upstream_repo: 'https://github.com/edx/edx-platform.git' + upstream_repo: 'https://github.com/openedx/edx-platform.git' upstream_branches: 'open-release/nutmeg.master,master' exclude_paths: 'cms/static/js/,conf/locale/,lms/static/js/,package.json,package-lock.json,.github/' secrets: From da69fb92c8aa09169bc48e2b0523fa8da432b561 Mon Sep 17 00:00:00 2001 From: Vladyslav Tymofeiev <“vladyslavty@softwareplanetgroup.com”> Date: Tue, 16 Jul 2024 12:21:58 +0300 Subject: [PATCH 116/125] Add trusted host workaround for tests workflow --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 989a259d4ecc..b96fe22dde49 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,6 +34,8 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + env: + PIP_TRUSTED_HOST: "pypi.python.org pypi.org files.pythonhosted.org" - name: Install dependencies # TODO: Remove tox-pip-version once we upgrade to Koa+, or whenever we have addressed pip 20.3 strict issues. run: | From bc998c9f92d30147cd16be4713fb49516532fb48 Mon Sep 17 00:00:00 2001 From: Vladyslav Tymofeiev <“vladyslavty@softwareplanetgroup.com”> Date: Fri, 19 Jul 2024 17:46:25 +0300 Subject: [PATCH 117/125] Improve the sanitize_next_parameter method with handling new cases --- common/djangoapps/student/helpers.py | 20 ++++++++++++++++ .../djangoapps/student/tests/test_helpers.py | 24 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/common/djangoapps/student/helpers.py b/common/djangoapps/student/helpers.py index 4c2fcf7efd6c..0dd0d1f10d59 100644 --- a/common/djangoapps/student/helpers.py +++ b/common/djangoapps/student/helpers.py @@ -714,7 +714,27 @@ def sanitize_next_parameter(next_param): return next_param if COURSE_URL_PATTERN.match(next_param): + # Sometimes the course id received with incorrect encoding/decoding, we need to + # replace it to the correct pattern: + # course-v1:test-sandbox PREP-CORE C -> course-v1:test-sandbox+PREP-CORE+C + # + # Note: We are not expect to have any spaces in course URL + + if ' ' in next_param: + next_param = next_param.replace(' ', '+') + elif '%20' in next_param: + next_param = next_param.replace('%20', '+') + sanitized_next_parameter = re.sub(r'\+', '%2B', next_param) + + log.info( + u"The course-like next parameter was detected '%(next_param)s'" + u" this will be replaced with sanitized version: '%(sanitized_next_parameter)s'", + { + "next_param": next_param, + "sanitized_next_parameter": sanitized_next_parameter, + } + ) return sanitized_next_parameter return next_param diff --git a/common/djangoapps/student/tests/test_helpers.py b/common/djangoapps/student/tests/test_helpers.py index 8231503c8a55..f069910296f8 100644 --- a/common/djangoapps/student/tests/test_helpers.py +++ b/common/djangoapps/student/tests/test_helpers.py @@ -178,3 +178,27 @@ def test_sanitize_next_param(self): # Invalid pattern - keep the next_param as it is next_param = 'some/other/path' self.assertEqual(sanitize_next_parameter(next_param), next_param) + + # Invalid URL with space - replace the ' ' with '+' and encode it + expected_result = 'courses/course-v1:abc-sandbox%2BACC-PTF%2BC/course' + + next_param = 'courses/course-v1:abc-sandbox ACC-PTF C/course' + self.assertEqual(sanitize_next_parameter(next_param), expected_result) + + next_param = 'courses/course-v1:abc-sandbox ACC-PTF+C/course' + self.assertEqual(sanitize_next_parameter(next_param), expected_result) + + next_param = 'courses/course-v1:abc-sandbox+ACC-PTF C/course' + self.assertEqual(sanitize_next_parameter(next_param), expected_result) + + # Invalid URL with encoded space - replace the '%20' with '+' and encode it + expected_result = 'courses/course-v1:abc-sandbox%2BACC-PTF%2BC/course' + + next_param = 'courses/course-v1:abc-sandbox%20ACC-PTF%20C/course' + self.assertEqual(sanitize_next_parameter(next_param), expected_result) + + next_param = 'courses/course-v1:abc-sandbox%20ACC-PTF+C/course' + self.assertEqual(sanitize_next_parameter(next_param), expected_result) + + next_param = 'courses/course-v1:abc-sandbox+ACC-PTF%20C/course' + self.assertEqual(sanitize_next_parameter(next_param), expected_result) From 7b5c7c1210956f60f8188ac74548fb6afbdfa6aa Mon Sep 17 00:00:00 2001 From: Vladyslav Tymofeiev <“vladyslavty@softwareplanetgroup.com”> Date: Mon, 22 Jul 2024 11:26:57 +0300 Subject: [PATCH 118/125] Fix sync_prod_with_main workflow --- .github/workflows/sync_nutmeg_with_juniper.yml | 2 +- .github/workflows/sync_prod_with_main.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sync_nutmeg_with_juniper.yml b/.github/workflows/sync_nutmeg_with_juniper.yml index f92283f378d1..c1dbb1130b64 100644 --- a/.github/workflows/sync_nutmeg_with_juniper.yml +++ b/.github/workflows/sync_nutmeg_with_juniper.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: pull-request-action - uses: vsoch/pull-request-action@1.0.19 + uses: vsoch/pull-request-action@1.1.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} BRANCH_PREFIX: "main" diff --git a/.github/workflows/sync_prod_with_main.yml b/.github/workflows/sync_prod_with_main.yml index eab2bbcee3df..358f6072bf33 100644 --- a/.github/workflows/sync_prod_with_main.yml +++ b/.github/workflows/sync_prod_with_main.yml @@ -9,10 +9,10 @@ jobs: runs-on: ubuntu-latest steps: - name: pull-request-action - uses: vsoch/pull-request-action@1.0.19 + uses: vsoch/pull-request-action@1.1.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} BRANCH_PREFIX: "main" PULL_REQUEST_BRANCH: "prod" PULL_REQUEST_TITLE: "Update from `main` (production)" - PULL_REQUEST_REVIEWERS: "xscrio amirtds bryanlandia" + PULL_REQUEST_REVIEWERS: "VladyslavTy daniilly" From 878af3c07620c9b86c246ae15cd3116dcfb48112 Mon Sep 17 00:00:00 2001 From: Vladyslav Tymofeiev <“vladyslavty@softwareplanetgroup.com”> Date: Tue, 13 Aug 2024 18:43:05 +0300 Subject: [PATCH 119/125] Add handler of iframe rendering --- common/static/js/src/iframe-render.js | 13 +++++++++++++ lms/static/sass/shared-v2/_base.scss | 5 +++++ lms/templates/main.html | 1 + 3 files changed, 19 insertions(+) create mode 100644 common/static/js/src/iframe-render.js diff --git a/common/static/js/src/iframe-render.js b/common/static/js/src/iframe-render.js new file mode 100644 index 000000000000..a66debfe8b9a --- /dev/null +++ b/common/static/js/src/iframe-render.js @@ -0,0 +1,13 @@ +// List of the classes to hide while rendered in an iframe +const classesToHide = ['.global-header', '.wrapper-course-material', 'a--footer']; + +document.addEventListener('DOMContentLoaded', function () { + // Check if rendered in iframe + if (window.self !== window.top) { + classesToHide.forEach(function (className) { + document.querySelectorAll(className).forEach(function (element) { + element.classList.add('hidden-in-iframe'); + }); + }); + } +}); diff --git a/lms/static/sass/shared-v2/_base.scss b/lms/static/sass/shared-v2/_base.scss index 584cf5799faf..a90a065b38df 100644 --- a/lms/static/sass/shared-v2/_base.scss +++ b/lms/static/sass/shared-v2/_base.scss @@ -24,3 +24,8 @@ @extend .sr-only; } +// Hide element when rendered in iFrame +.hidden-in-iframe { + display: none !important; +} + diff --git a/lms/templates/main.html b/lms/templates/main.html index 48edc767a132..fd826785eefc 100644 --- a/lms/templates/main.html +++ b/lms/templates/main.html @@ -211,6 +211,7 @@ <%static:optional_include_mako file="body-extra.html" is_theming_enabled="True" /> + From 9c2f37476e1713dc1709196dcd1384579ba47cef Mon Sep 17 00:00:00 2001 From: Vladyslav Tymofeiev <“vladyslavty@softwareplanetgroup.com”> Date: Wed, 14 Aug 2024 10:13:34 +0300 Subject: [PATCH 120/125] Fix typo in footer classname --- common/static/js/src/iframe-render.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/static/js/src/iframe-render.js b/common/static/js/src/iframe-render.js index a66debfe8b9a..8264b1ae0783 100644 --- a/common/static/js/src/iframe-render.js +++ b/common/static/js/src/iframe-render.js @@ -1,5 +1,5 @@ // List of the classes to hide while rendered in an iframe -const classesToHide = ['.global-header', '.wrapper-course-material', 'a--footer']; +const classesToHide = ['.global-header', '.wrapper-course-material', '.a--footer']; document.addEventListener('DOMContentLoaded', function () { // Check if rendered in iframe From a7c6b5fa09d8fd1d917a51b3ae689c62817cf6ff Mon Sep 17 00:00:00 2001 From: Vladyslav Tymofeiev <“vladyslavty@softwareplanetgroup.com”> Date: Fri, 16 Aug 2024 17:21:11 +0300 Subject: [PATCH 121/125] Refactor event to hide elements --- common/static/js/src/iframe-render.js | 13 ----------- lms/static/lms/js/iframe-render.js | 31 +++++++++++++++++++++++++++ lms/static/sass/shared-v2/_base.scss | 2 +- lms/templates/main.html | 2 +- 4 files changed, 33 insertions(+), 15 deletions(-) delete mode 100644 common/static/js/src/iframe-render.js create mode 100644 lms/static/lms/js/iframe-render.js diff --git a/common/static/js/src/iframe-render.js b/common/static/js/src/iframe-render.js deleted file mode 100644 index 8264b1ae0783..000000000000 --- a/common/static/js/src/iframe-render.js +++ /dev/null @@ -1,13 +0,0 @@ -// List of the classes to hide while rendered in an iframe -const classesToHide = ['.global-header', '.wrapper-course-material', '.a--footer']; - -document.addEventListener('DOMContentLoaded', function () { - // Check if rendered in iframe - if (window.self !== window.top) { - classesToHide.forEach(function (className) { - document.querySelectorAll(className).forEach(function (element) { - element.classList.add('hidden-in-iframe'); - }); - }); - } -}); diff --git a/lms/static/lms/js/iframe-render.js b/lms/static/lms/js/iframe-render.js new file mode 100644 index 000000000000..f5b5bce6b20a --- /dev/null +++ b/lms/static/lms/js/iframe-render.js @@ -0,0 +1,31 @@ +// List of the classes to hide while rendered in an iframe +const classesToHide = ['.global-header', '.wrapper-course-material', '.a--footer']; + +// Function to get a cookie by name +function getCookieByName(name) { + let cname = name + "="; + let decodedCookie = decodeURIComponent(document.cookie); + let cookies = decodedCookie.split(';'); + for (let i = 0; i < cookies.length; i++) { + let c = cookies[i]; + while (c.charAt(0) == ' ') { + c = c.substring(1); + } + if (c.indexOf(name) == 0) { + return c.substring(name.length, c.length); + } + } + return ""; +} + +document.addEventListener('DOMContentLoaded', function () { + const hideElements = getCookieByName('hideElements'); + + if (hideElements) { + classesToHide.forEach(function (className) { + document.querySelectorAll(className).forEach(function (element) { + element.classList.add('hidden-element'); + }); + }); + } +}); diff --git a/lms/static/sass/shared-v2/_base.scss b/lms/static/sass/shared-v2/_base.scss index a90a065b38df..450a1aa51b75 100644 --- a/lms/static/sass/shared-v2/_base.scss +++ b/lms/static/sass/shared-v2/_base.scss @@ -25,7 +25,7 @@ } // Hide element when rendered in iFrame -.hidden-in-iframe { +.hidden-element { display: none !important; } diff --git a/lms/templates/main.html b/lms/templates/main.html index fd826785eefc..af1d51fe9cbc 100644 --- a/lms/templates/main.html +++ b/lms/templates/main.html @@ -123,6 +123,7 @@ }).call(this, require || RequireJS.require); + <%block name="js_overrides"> ${render_require_js_path_overrides(settings.REQUIRE_JS_PATH_OVERRIDES) | n, decode.utf8} @@ -211,7 +212,6 @@ <%static:optional_include_mako file="body-extra.html" is_theming_enabled="True" /> - From 71f097879ea41ee45510de86cd3b2d236bb28029 Mon Sep 17 00:00:00 2001 From: Vladyslav Tymofeiev <“vladyslavty@softwareplanetgroup.com”> Date: Fri, 16 Aug 2024 18:03:09 +0300 Subject: [PATCH 122/125] Add hide_elements parameter that determine if we need to hide the html elements after finish auth process --- lms/static/js/student_account/views/FinishAuthView.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lms/static/js/student_account/views/FinishAuthView.js b/lms/static/js/student_account/views/FinishAuthView.js index 870c56b61591..ac09152c8289 100644 --- a/lms/static/js/student_account/views/FinishAuthView.js +++ b/lms/static/js/student_account/views/FinishAuthView.js @@ -51,7 +51,8 @@ courseId: $.url('?course_id'), courseMode: $.url('?course_mode'), emailOptIn: $.url('?email_opt_in'), - purchaseWorkflow: $.url('?purchase_workflow') + purchaseWorkflow: $.url('?purchase_workflow'), + hideElements: $.url('?hide_elements') }; for (var key in queryParams) { if (queryParams[key]) { @@ -64,6 +65,7 @@ this.emailOptIn = queryParams.emailOptIn; this.nextUrl = this.urls.defaultNextUrl; this.purchaseWorkflow = queryParams.purchaseWorkflow; + this.hideElements = queryParams.hideElements; if (queryParams.next) { // Ensure that the next URL is internal for security reasons if (! window.isExternal(queryParams.next)) { @@ -76,6 +78,9 @@ try { var next = _.bind(this.enrollment, this); this.checkEmailOptIn(next); + if (this.hideElements) { + document.cookie = 'hideElements=' + this.hideElements + '; path=/'; + } } catch (err) { this.updateTaskDescription(gettext('Error') + ': ' + err.message); this.redirect(this.nextUrl); From a68fc7a3a2558de28670e8865d60e1e49aee7d55 Mon Sep 17 00:00:00 2001 From: Vladyslav Tymofeiev <“vladyslavty@softwareplanetgroup.com”> Date: Wed, 21 Aug 2024 17:44:16 +0300 Subject: [PATCH 123/125] Set cookie value ror hideElements before redirect --- lms/static/js/student_account/views/FinishAuthView.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lms/static/js/student_account/views/FinishAuthView.js b/lms/static/js/student_account/views/FinishAuthView.js index ac09152c8289..461516b6e12e 100644 --- a/lms/static/js/student_account/views/FinishAuthView.js +++ b/lms/static/js/student_account/views/FinishAuthView.js @@ -76,11 +76,11 @@ render: function() { try { - var next = _.bind(this.enrollment, this); - this.checkEmailOptIn(next); if (this.hideElements) { document.cookie = 'hideElements=' + this.hideElements + '; path=/'; } + var next = _.bind(this.enrollment, this); + this.checkEmailOptIn(next); } catch (err) { this.updateTaskDescription(gettext('Error') + ': ' + err.message); this.redirect(this.nextUrl); From 4efbe0971c5742c3cbb66b0c66adbc9a1ae38baa Mon Sep 17 00:00:00 2001 From: Vladyslav Tymofeiev <“vladyslavty@softwareplanetgroup.com”> Date: Fri, 23 Aug 2024 10:53:20 +0300 Subject: [PATCH 124/125] Add hide_elements to allowed post auth parameters --- common/djangoapps/student/helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/djangoapps/student/helpers.py b/common/djangoapps/student/helpers.py index 0dd0d1f10d59..354c9a328d1a 100644 --- a/common/djangoapps/student/helpers.py +++ b/common/djangoapps/student/helpers.py @@ -227,7 +227,8 @@ def check_verify_status_by_course(user, course_enrollments): # Query string parameters that can be passed to the "finish_auth" view to manage # things like auto-enrollment. -POST_AUTH_PARAMS = ('course_id', 'enrollment_action', 'course_mode', 'email_opt_in', 'purchase_workflow') +POST_AUTH_PARAMS = ('course_id', 'enrollment_action', 'course_mode', 'email_opt_in', 'purchase_workflow', + 'hide_elements') def get_next_url_for_login_page(request): From 27d9bd4c04199947fae31c25d6c464d96b4b7ef0 Mon Sep 17 00:00:00 2001 From: Vladyslav Tymofeiev <“vladyslavty@softwareplanetgroup.com”> Date: Tue, 27 Aug 2024 15:12:50 +0300 Subject: [PATCH 125/125] Add function add_hide_elements_cookie_to_redirect to set hideElements cookie on the backend side --- common/djangoapps/student/helpers.py | 14 ++++++++++++++ .../core/djangoapps/user_authn/views/login_form.py | 8 +++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/common/djangoapps/student/helpers.py b/common/djangoapps/student/helpers.py index 354c9a328d1a..a2ff280c7aa3 100644 --- a/common/djangoapps/student/helpers.py +++ b/common/djangoapps/student/helpers.py @@ -19,6 +19,7 @@ from django.core.exceptions import PermissionDenied from django.core.validators import ValidationError from django.db import IntegrityError, transaction +from django.shortcuts import redirect from django.urls import NoReverseMatch, reverse from django.utils.translation import ugettext as _ from pytz import UTC @@ -739,3 +740,16 @@ def sanitize_next_parameter(next_param): return sanitized_next_parameter return next_param + + +def add_hide_elements_cookie_to_redirect(redirect_to): + if 'hide_elements' in redirect_to: + # Perform the redirect and set the cookie only if 'hide_elements' is present + response = redirect(redirect_to) + + # Set a cookie to indicate that elements should be hidden + response.set_cookie('hideElements', 'true', max_age=86400) + + return response + else: + return redirect(redirect_to) diff --git a/openedx/core/djangoapps/user_authn/views/login_form.py b/openedx/core/djangoapps/user_authn/views/login_form.py index 3292f14da641..05e6d0cebb3d 100644 --- a/openedx/core/djangoapps/user_authn/views/login_form.py +++ b/openedx/core/djangoapps/user_authn/views/login_form.py @@ -30,7 +30,7 @@ handle_enterprise_cookies_for_logistration, update_logistration_context_for_enterprise ) -from student.helpers import get_next_url_for_login_page +from student.helpers import get_next_url_for_login_page, add_hide_elements_cookie_to_redirect from third_party_auth import pipeline from third_party_auth.decorators import xframe_allow_whitelisted from util.password_policy_validators import DEFAULT_MAX_PASSWORD_LENGTH @@ -146,12 +146,14 @@ def login_and_registration_form(request, initial_mode="login"): # since Django's SessionAuthentication middleware auto-updates session cookies but not # the other login-related cookies. See ARCH-282. if request.user.is_authenticated and are_logged_in_cookies_set(request): - return redirect(redirect_to) + response = add_hide_elements_cookie_to_redirect(redirect_to) + return response # Tahoe: Disable upstream login/register forms when the Tahoe Identity Provider is enabled. tahoe_idp_redirect_url = tahoe_idp_helpers.get_idp_form_url(request, initial_mode, redirect_to) if tahoe_idp_redirect_url: - return redirect(tahoe_idp_redirect_url) + response = add_hide_elements_cookie_to_redirect(tahoe_idp_redirect_url) + return response # Retrieve the form descriptions from the user API form_descriptions = _get_form_descriptions(request)