From cfa8073a6b21440374669fadc0a62e3e4a10ee35 Mon Sep 17 00:00:00 2001 From: Saleem Latif Date: Mon, 22 Jul 2024 16:32:24 +0500 Subject: [PATCH] feat: Added a new endpoint to get enterprise analytics aggregated data. --- CHANGELOG.rst | 4 + enterprise_data/__init__.py | 2 +- enterprise_data/admin_analytics/__init__.py | 0 enterprise_data/admin_analytics/constants.py | 15 ++ .../admin_analytics/data_loaders.py | 134 ++++++++++++++++++ enterprise_data/admin_analytics/database.py | 31 ++++ enterprise_data/admin_analytics/utils.py | 81 +++++++++++ enterprise_data/api/v1/serializers.py | 20 +++ enterprise_data/api/v1/urls.py | 19 ++- enterprise_data/api/v1/views/__init__.py | 0 enterprise_data/api/v1/views/base.py | 26 ++++ .../api/v1/views/enterprise_admin.py | 109 ++++++++++++++ .../{views.py => views/enterprise_learner.py} | 119 +--------------- .../api/v1/views/enterprise_offers.py | 41 ++++++ enterprise_data/tests/test_filters.py | 2 +- enterprise_data/utils.py | 49 ++++++- enterprise_reporting/clients/__init__.py | 5 +- .../external_resource_link_report.py | 6 +- enterprise_reporting/tests/test_clients.py | 2 +- .../tests/test_enterprise_client.py | 7 +- .../tests/test_external_link_report.py | 4 +- enterprise_reporting/tests/test_utils.py | 6 +- enterprise_reporting/utils.py | 2 +- requirements/base.in | 3 + requirements/base.txt | 32 ++++- requirements/ci.txt | 6 +- requirements/common_constraints.txt | 1 - requirements/constraints.txt | 3 + requirements/dev.txt | 49 +++++-- requirements/pip.txt | 2 +- requirements/pip_tools.txt | 12 +- requirements/quality.txt | 55 +++++-- requirements/test-master.txt | 40 ++++-- requirements/test-reporting.txt | 25 +++- requirements/test.txt | 40 ++++-- 35 files changed, 748 insertions(+), 204 deletions(-) create mode 100644 enterprise_data/admin_analytics/__init__.py create mode 100644 enterprise_data/admin_analytics/constants.py create mode 100644 enterprise_data/admin_analytics/data_loaders.py create mode 100644 enterprise_data/admin_analytics/database.py create mode 100644 enterprise_data/admin_analytics/utils.py create mode 100644 enterprise_data/api/v1/views/__init__.py create mode 100644 enterprise_data/api/v1/views/base.py create mode 100644 enterprise_data/api/v1/views/enterprise_admin.py rename enterprise_data/api/v1/{views.py => views/enterprise_learner.py} (81%) create mode 100644 enterprise_data/api/v1/views/enterprise_offers.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9c34b944..c5f93997 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,6 +15,10 @@ Unreleased ---------- ========================= +[8.2.0] - 2024-07-25 +--------------------- + * Added a new API endpoint to get admin analytics aggregated data on user enrollment and engagement. + [8.1.0] - 2024-07-22 --------------------- * Upgrade python requirements diff --git a/enterprise_data/__init__.py b/enterprise_data/__init__.py index 5a89ee96..2eb9938b 100644 --- a/enterprise_data/__init__.py +++ b/enterprise_data/__init__.py @@ -2,4 +2,4 @@ Enterprise data api application. This Django app exposes API endpoints used by enterprises. """ -__version__ = "8.1.0" +__version__ = "8.2.0" diff --git a/enterprise_data/admin_analytics/__init__.py b/enterprise_data/admin_analytics/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/enterprise_data/admin_analytics/constants.py b/enterprise_data/admin_analytics/constants.py new file mode 100644 index 00000000..b5036cde --- /dev/null +++ b/enterprise_data/admin_analytics/constants.py @@ -0,0 +1,15 @@ +""" +Constants for admin analytics. +""" +import mysql.connector + +from django.conf import settings + +DATABASE_CONNECTION_CONFIG = { + 'host': settings.DATABASES[settings.ENTERPRISE_REPORTING_DB_ALIAS]['HOST'], + 'port': settings.DATABASES[settings.ENTERPRISE_REPORTING_DB_ALIAS]['PORT'], + 'database': settings.DATABASES[settings.ENTERPRISE_REPORTING_DB_ALIAS]['NAME'], + 'user': settings.DATABASES[settings.ENTERPRISE_REPORTING_DB_ALIAS]['USER'], + 'password': settings.DATABASES[settings.ENTERPRISE_REPORTING_DB_ALIAS]['PASSWORD'], +} +DATABASE_CONNECTOR = mysql.connector.connect diff --git a/enterprise_data/admin_analytics/data_loaders.py b/enterprise_data/admin_analytics/data_loaders.py new file mode 100644 index 00000000..817bbd17 --- /dev/null +++ b/enterprise_data/admin_analytics/data_loaders.py @@ -0,0 +1,134 @@ +""" +Utility functions for fetching data from the database. +""" +import numpy +import pandas + +from django.http import Http404 + +from enterprise_data.admin_analytics.database import run_query + + +def get_select_query(table: str, columns: list, enterprise_uuid: str) -> str: + """ + Generate a SELECT query for the given table and columns. + + Arguments: + table (str): The table to query. + columns (list): The columns to select. + enterprise_uuid (str): The UUID of the enterprise customer. + + Returns: + (str): The SELECT query. + """ + return f'SELECT {", ".join(columns)} FROM {table} WHERE enterprise_customer_uuid = "{enterprise_uuid}"' + + +def fetch_enrollment_data(enterprise_uuid: str): + """ + Fetch enrollment data from the database for the given enterprise customer. + + Arguments: + enterprise_uuid (str): The UUID of the enterprise customer. + + Returns: + (pandas.DataFrame): The enrollment data. + """ + enterprise_uuid = enterprise_uuid.replace('-', '') + + columns = [ + 'enterprise_customer_name', + 'enterprise_customer_uuid', + 'lms_enrollment_id', + 'user_id', + 'email', + 'course_key', + 'courserun_key', + 'course_id', + 'course_subject', + 'course_title', + 'enterprise_enrollment_date', + 'lms_enrollment_mode', + 'enroll_type', + 'program_title', + 'date_certificate_awarded', + 'grade_percent', + 'cert_awarded', + 'date_certificate_created_raw', + 'passed_date_raw', + 'passed_date', + 'has_passed', + ] + query = get_select_query( + table='fact_enrollment_admin_dash', + columns=columns, + enterprise_uuid=enterprise_uuid, + ) + + results = run_query(query=query) + if not results: + raise Http404(f'No enrollment data found for enterprise {enterprise_uuid}') + + enrollments = pandas.DataFrame(numpy.array(results), columns=columns) + + # Convert date columns to datetime. + enrollments['enterprise_enrollment_date'] = enrollments['enterprise_enrollment_date'].astype('datetime64[ns]') + enrollments['date_certificate_awarded'] = enrollments['date_certificate_awarded'].astype('datetime64[ns]') + enrollments['date_certificate_created_raw'] = enrollments['date_certificate_created_raw'].astype('datetime64[ns]') + enrollments['passed_date_raw'] = enrollments['passed_date_raw'].astype('datetime64[ns]') + enrollments['passed_date'] = enrollments['passed_date'].astype('datetime64[ns]') + + return enrollments + + +def fetch_engagement_data(enterprise_uuid: str): + """ + Fetch engagement data from the database for the given enterprise customer. + + Arguments: + enterprise_uuid (str): The UUID of the enterprise customer. + + Returns: + (pandas.DataFrame): The engagement data. + """ + enterprise_uuid = enterprise_uuid.replace('-', '') + + columns = [ + 'user_id', + 'email', + 'enterprise_customer_uuid', + 'course_key', + 'enroll_type', + 'activity_date', + 'course_title', + 'course_subject', + 'is_engaged', + 'is_engaged_video', + 'is_engaged_forum', + 'is_engaged_problem', + 'is_active', + 'learning_time_seconds', + ] + query = get_select_query( + table='fact_enrollment_engagement_day_admin_dash', columns=columns, enterprise_uuid=enterprise_uuid + ) + + results = run_query(query=query) + if not results: + raise Http404(f'No engagement data found for enterprise {enterprise_uuid}') + + engagement = pandas.DataFrame(numpy.array(results), columns=columns) + engagement['activity_date'] = engagement['activity_date'].astype('datetime64[ns]') + + return engagement + + +def fetch_max_enrollment_datetime(): + """ + Fetch the latest created date from the enterprise_learner_enrollment table. + """ + query = "SELECT MAX(created) FROM enterprise_learner_enrollment" + results = run_query(query) + if not results: + return None + return pandas.to_datetime(results[0][0]) diff --git a/enterprise_data/admin_analytics/database.py b/enterprise_data/admin_analytics/database.py new file mode 100644 index 00000000..ccb17c94 --- /dev/null +++ b/enterprise_data/admin_analytics/database.py @@ -0,0 +1,31 @@ +""" +Utility functions for interacting with the database. +""" +from contextlib import closing +from logging import getLogger + +from enterprise_data.admin_analytics.constants import DATABASE_CONNECTION_CONFIG, DATABASE_CONNECTOR +from enterprise_data.utils import timeit + +LOGGER = getLogger(__name__) + + +@timeit +def run_query(query): + """ + Run a query on the database and return the results. + + Arguments: + query (str): The query to run. + + Returns: + (list): The results of the query. + """ + try: + with closing(DATABASE_CONNECTOR(**DATABASE_CONNECTION_CONFIG)) as connection: + with closing(connection.cursor()) as cursor: + cursor.execute(query) + return cursor.fetchall() + except Exception: + LOGGER.exception(f'[run_query]: run_query failed for query "{query}".') + raise diff --git a/enterprise_data/admin_analytics/utils.py b/enterprise_data/admin_analytics/utils.py new file mode 100644 index 00000000..44ff3176 --- /dev/null +++ b/enterprise_data/admin_analytics/utils.py @@ -0,0 +1,81 @@ +""" +Utility functions for fetching data from the database. +""" +from datetime import datetime + +from edx_django_utils.cache import TieredCache, get_cache_key + +from enterprise_data.admin_analytics.data_loaders import fetch_engagement_data, fetch_enrollment_data + + +def get_cache_timeout(cache_expiry): + """ + Helper method to calculate cache timeout in seconds. + + Arguments: + cache_expiry (datetime): Datetime object denoting the cache expiry. + + Returns: + (int): Cache timeout in seconds. + """ + now = datetime.now() + cache_timeout = 0 + if cache_expiry > now: + # Calculate cache expiry in seconds from now. + cache_timeout = (cache_expiry - now).seconds + + return cache_timeout + + +def fetch_and_cache_enrollments_data(enterprise_id, cache_expiry): + """ + Helper method to fetch and cache enrollments data. + + Arguments: + enterprise_id (str): UUID of the enterprise customer in string format. + cache_expiry (datetime): Datetime object denoting the cache expiry. + + Returns: + (pandas.DataFrame): The enrollments data. + """ + cache_key = get_cache_key( + resource='enterprise-admin-analytics-aggregates-enrollments', + enterprise_customer=enterprise_id, + ) + cached_response = TieredCache.get_cached_response(cache_key) + + if cached_response.is_found: + return cached_response.value + else: + enrollments = fetch_enrollment_data(enterprise_id) + TieredCache.set_all_tiers( + cache_key, enrollments, get_cache_timeout(cache_expiry) + ) + return enrollments + + +def fetch_and_cache_engagements_data(enterprise_id, cache_expiry): + """ + Helper method to fetch and cache engagements data. + + Arguments: + enterprise_id (str): UUID of the enterprise customer in string format. + cache_expiry (datetime): Datetime object denoting the cache expiry. + + Returns: + (pandas.DataFrame): The engagements data. + """ + cache_key = get_cache_key( + resource='enterprise-admin-analytics-aggregates-engagements', + enterprise_customer=enterprise_id, + ) + cached_response = TieredCache.get_cached_response(cache_key) + + if cached_response.is_found: + return cached_response.value + else: + engagements = fetch_engagement_data(enterprise_id) + TieredCache.set_all_tiers( + cache_key, engagements, get_cache_timeout(cache_expiry) + ) + return engagements diff --git a/enterprise_data/api/v1/serializers.py b/enterprise_data/api/v1/serializers.py index c88ef868..0031d0a9 100644 --- a/enterprise_data/api/v1/serializers.py +++ b/enterprise_data/api/v1/serializers.py @@ -196,3 +196,23 @@ class EnterpriseAdminSummarizeInsightsSerializer(serializers.ModelSerializer): class Meta: model = EnterpriseAdminSummarizeInsights fields = '__all__' + + +class AdminAnalyticsAggregatesQueryParamsSerializer(serializers.Serializer): # pylint: disable=abstract-method + """ + Serializer for validating admin analytics query params. + """ + start_date = serializers.DateField(required=False) + end_date = serializers.DateField(required=False) + + def validate(self, attrs): + """ + Validate the query params. + + Raises: + serializers.ValidationError: If start_date is greater than end_date. + """ + if 'start_date' in attrs and 'end_date' in attrs: + if attrs['start_date'] > attrs['end_date']: + raise serializers.ValidationError("start_date should be less than or equal to end_date.") + return attrs diff --git a/enterprise_data/api/v1/urls.py b/enterprise_data/api/v1/urls.py index eed35fb2..168a6610 100644 --- a/enterprise_data/api/v1/urls.py +++ b/enterprise_data/api/v1/urls.py @@ -7,7 +7,9 @@ from django.urls import re_path -from enterprise_data.api.v1 import views +from enterprise_data.api.v1.views import enterprise_admin as enterprise_admin_views +from enterprise_data.api.v1.views import enterprise_learner as enterprise_learner_views +from enterprise_data.api.v1.views import enterprise_offers as enterprise_offers_views from enterprise_data.constants import UUID4_REGEX app_name = 'enterprise_data_api_v1' @@ -15,31 +17,36 @@ router = DefaultRouter() router.register( r'enterprise/(?P.+)/enrollments', - views.EnterpriseLearnerEnrollmentViewSet, + enterprise_learner_views.EnterpriseLearnerEnrollmentViewSet, 'enterprise-learner-enrollment', ) router.register( r'enterprise/(?P.+)/offers', - views.EnterpriseOfferViewSet, + enterprise_offers_views.EnterpriseOfferViewSet, 'enterprise-offers', ) router.register( r'enterprise/(?P.+)/users', - views.EnterpriseLearnerViewSet, + enterprise_learner_views.EnterpriseLearnerViewSet, 'enterprise-learner', ) router.register( r'enterprise/(?P.+)/learner_completed_courses', - views.EnterpriseLearnerCompletedCoursesViewSet, + enterprise_learner_views.EnterpriseLearnerCompletedCoursesViewSet, 'enterprise-learner-completed-courses', ) urlpatterns = [ re_path( fr'^admin/insights/(?P{UUID4_REGEX})$', - views.EnterpriseAdminInsightsView.as_view(), + enterprise_admin_views.EnterpriseAdminInsightsView.as_view(), name='enterprise-admin-insights' ), + re_path( + fr'^admin/anlaytics/(?P{UUID4_REGEX})$', + enterprise_admin_views.EnterpriseAdminAnalyticsAggregatesView.as_view(), + name='enterprise-admin-analytics-aggregates' + ), ] urlpatterns += router.urls diff --git a/enterprise_data/api/v1/views/__init__.py b/enterprise_data/api/v1/views/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/enterprise_data/api/v1/views/base.py b/enterprise_data/api/v1/views/base.py new file mode 100644 index 00000000..bb52f184 --- /dev/null +++ b/enterprise_data/api/v1/views/base.py @@ -0,0 +1,26 @@ +""" +Base views for enterprise data api v1. +""" +from edx_rbac.mixins import PermissionRequiredMixin +from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication +from edx_rest_framework_extensions.paginators import DefaultPagination + +from enterprise_data.constants import ANALYTICS_API_VERSION_1 + + +class EnterpriseViewSetMixin(PermissionRequiredMixin): + """ + Base class for all Enterprise view sets. + """ + authentication_classes = (JwtAuthentication,) + pagination_class = DefaultPagination + permission_required = 'can_access_enterprise' + API_VERSION = ANALYTICS_API_VERSION_1 + + def paginate_queryset(self, queryset): + """ + Allows no_page query param to skip pagination + """ + if 'no_page' in self.request.query_params: + return None + return super().paginate_queryset(queryset) diff --git a/enterprise_data/api/v1/views/enterprise_admin.py b/enterprise_data/api/v1/views/enterprise_admin.py new file mode 100644 index 00000000..cb6566ca --- /dev/null +++ b/enterprise_data/api/v1/views/enterprise_admin.py @@ -0,0 +1,109 @@ +""" +Views for enterprise admin api v1. +""" +from datetime import datetime, timedelta + +from edx_rbac.decorators import permission_required +from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication +from rest_framework.response import Response +from rest_framework.status import HTTP_200_OK, HTTP_404_NOT_FOUND +from rest_framework.views import APIView + +from enterprise_data.admin_analytics.data_loaders import fetch_max_enrollment_datetime +from enterprise_data.admin_analytics.utils import fetch_and_cache_engagements_data, fetch_and_cache_enrollments_data +from enterprise_data.api.v1 import serializers +from enterprise_data.models import EnterpriseAdminLearnerProgress, EnterpriseAdminSummarizeInsights +from enterprise_data.utils import date_filter + + +class EnterpriseAdminInsightsView(APIView): + """ + API for getting the enterprise admin insights. + """ + authentication_classes = (JwtAuthentication,) + http_method_names = ['get'] + + @permission_required('can_access_enterprise', fn=lambda request, enterprise_id: enterprise_id) + def get(self, request, enterprise_id): + """ + HTTP GET endpoint to retrieve the enterprise admin insights + """ + response_data = {} + learner_progress = {} + learner_engagement = {} + + try: + learner_progress = EnterpriseAdminLearnerProgress.objects.get(enterprise_customer_uuid=enterprise_id) + learner_progress = serializers.EnterpriseAdminLearnerProgressSerializer(learner_progress).data + response_data['learner_progress'] = learner_progress + except EnterpriseAdminLearnerProgress.DoesNotExist: + pass + + try: + learner_engagement = EnterpriseAdminSummarizeInsights.objects.get(enterprise_customer_uuid=enterprise_id) + learner_engagement = serializers.EnterpriseAdminSummarizeInsightsSerializer(learner_engagement).data + response_data['learner_engagement'] = learner_engagement + except EnterpriseAdminSummarizeInsights.DoesNotExist: + pass + + status = HTTP_200_OK + if learner_progress == {} and learner_engagement == {}: + status = HTTP_404_NOT_FOUND + + return Response(data=response_data, status=status) + + +class EnterpriseAdminAnalyticsAggregatesView(APIView): + """ + API for getting the enterprise admin analytics aggregates. + """ + authentication_classes = (JwtAuthentication,) + http_method_names = ['get'] + + @permission_required('can_access_enterprise', fn=lambda request, enterprise_id: enterprise_id) + def get(self, request, enterprise_id): + """ + HTTP GET endpoint to retrieve the enterprise admin aggregate data. + """ + serializer = serializers.AdminAnalyticsAggregatesQueryParamsSerializer(data=request.GET) + serializer.is_valid(raise_exception=True) + + last_updated_at = fetch_max_enrollment_datetime() + cache_expiry = last_updated_at + timedelta(days=1) if last_updated_at else datetime.now() + + enrollment = fetch_and_cache_enrollments_data(enterprise_id, cache_expiry).copy() + engagement = fetch_and_cache_engagements_data(enterprise_id, cache_expiry).copy() + # Use start and end date if provided by the client, if client has not provided then use + # 1. minimum enrollment date from the data as the start_date + # 2. today's date as the end_date + start_date = serializer.data.get('start_date', enrollment.enterprise_enrollment_date.min()) + end_date = serializer.data.get('end_date', datetime.now()) + + # Date filtering. + dff = date_filter( + start=start_date, end=end_date, data_frame=enrollment.copy(), date_column='enterprise_enrollment_date' + ) + + enrolls = len(dff) + courses = len(dff.course_key.unique()) + + dff = date_filter(start=start_date, end=end_date, data_frame=enrollment.copy(), date_column='passed_date') + + completions = dff.has_passed.sum() + + # Date filtering. + dff = date_filter(start=start_date, end=end_date, data_frame=engagement.copy(), date_column='activity_date') + + hours = round(dff.learning_time_seconds.sum() / 60 / 60, 1) + sessions = dff.is_engaged.sum() + + return Response(data={ + 'enrolls': enrolls, + 'courses': courses, + 'completions': completions, + 'hours': hours, + 'sessions': sessions, + 'last_updated_at': last_updated_at.date() if last_updated_at else None, + 'min_enrollment_date': enrollment.enterprise_enrollment_date.min().date(), + 'max_enrollment_date': enrollment.enterprise_enrollment_date.max().date(), + }, status=HTTP_200_OK) diff --git a/enterprise_data/api/v1/views.py b/enterprise_data/api/v1/views/enterprise_learner.py similarity index 81% rename from enterprise_data/api/v1/views.py rename to enterprise_data/api/v1/views/enterprise_learner.py index 36e1bc52..1e24d48b 100644 --- a/enterprise_data/api/v1/views.py +++ b/enterprise_data/api/v1/views/enterprise_learner.py @@ -1,22 +1,15 @@ """ -Views for enterprise api v1. +Views for the enterprise learner. """ from datetime import date, timedelta from logging import getLogger from uuid import UUID -from django_filters.rest_framework import DjangoFilterBackend from edx_django_utils.cache import TieredCache -from edx_rbac.decorators import permission_required -from edx_rbac.mixins import PermissionRequiredMixin -from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication -from edx_rest_framework_extensions.paginators import DefaultPagination from rest_framework import filters, viewsets from rest_framework.decorators import action from rest_framework.response import Response -from rest_framework.status import HTTP_200_OK, HTTP_404_NOT_FOUND -from rest_framework.views import APIView from django.conf import settings from django.core.paginator import Paginator @@ -27,52 +20,18 @@ from django.utils import timezone from enterprise_data.api.v1 import serializers -from enterprise_data.constants import ANALYTICS_API_VERSION_1 from enterprise_data.filters import AuditEnrollmentsFilterBackend, AuditUsersEnrollmentFilterBackend -from enterprise_data.models import ( - EnterpriseAdminLearnerProgress, - EnterpriseAdminSummarizeInsights, - EnterpriseLearner, - EnterpriseLearnerEnrollment, - EnterpriseOffer, -) +from enterprise_data.models import EnterpriseLearner, EnterpriseLearnerEnrollment from enterprise_data.paginators import EnterpriseEnrollmentsPagination from enterprise_data.renderers import EnrollmentsCSVRenderer -from enterprise_data.utils import get_cache_key +from enterprise_data.utils import get_cache_key, subtract_one_month + +from .base import EnterpriseViewSetMixin LOGGER = getLogger(__name__) DEFAULT_LEARNER_CACHE_TIMEOUT = 60 * 10 -def subtract_one_month(original_date): - """ - Returns a date exactly one month prior to the passed in date. - """ - one_day = timedelta(days=1) - one_month_earlier = original_date - one_day - while one_month_earlier.month == original_date.month or one_month_earlier.day > original_date.day: - one_month_earlier -= one_day - return one_month_earlier - - -class EnterpriseViewSetMixin(PermissionRequiredMixin): - """ - Base class for all Enterprise view sets. - """ - authentication_classes = (JwtAuthentication,) - pagination_class = DefaultPagination - permission_required = 'can_access_enterprise' - API_VERSION = ANALYTICS_API_VERSION_1 - - def paginate_queryset(self, queryset): - """ - Allows no_page query param to skip pagination - """ - if 'no_page' in self.request.query_params: - return None - return super().paginate_queryset(queryset) - - class EnterpriseLearnerEnrollmentViewSet(EnterpriseViewSetMixin, viewsets.ReadOnlyModelViewSet): """ Viewset for routes related to Enterprise course enrollments. @@ -337,37 +296,6 @@ def overview(self, request, **kwargs): return Response(content) -class EnterpriseOfferViewSet(EnterpriseViewSetMixin, viewsets.ReadOnlyModelViewSet): - """ - Viewset for enterprise offers. - """ - serializer_class = serializers.EnterpriseOfferSerializer - filter_backends = (filters.OrderingFilter, DjangoFilterBackend,) - ordering_fields = '__all__' - - lookup_field = 'offer_id' - - filterset_fields = ( - 'offer_id', - 'status' - ) - - def get_object(self): - """ - This ensures that UUIDs with dashes are properly handled when requesting info about offers. - - Related to the work in EnterpriseOfferSerializer with `to_internal_value` and `to_representation` - """ - self.kwargs['offer_id'] = self.kwargs['offer_id'].replace('-', '') - return super().get_object() - - def get_queryset(self): - enterprise_customer_uuid = self.kwargs['enterprise_id'] - return EnterpriseOffer.objects.filter( - enterprise_customer_uuid=enterprise_customer_uuid, - ) - - class EnterpriseLearnerViewSet(EnterpriseViewSetMixin, viewsets.ReadOnlyModelViewSet): """ Viewset for routes related to Enterprise Learners. @@ -498,40 +426,3 @@ def get_queryset(self): is_consent_granted=True, # DSC check required ).values('user_email').annotate(completed_courses=Count('courserun_key')).order_by('user_email') return enrollments - - -class EnterpriseAdminInsightsView(APIView): - """ - API for getting the enterprise admin insights. - """ - authentication_classes = (JwtAuthentication,) - http_method_names = ['get'] - - @permission_required('can_access_enterprise', fn=lambda request, enterprise_id: enterprise_id) - def get(self, request, enterprise_id): - """ - HTTP GET endpoint to retrieve the enterprise admin insights - """ - response_data = {} - learner_progress = {} - learner_engagement = {} - - try: - learner_progress = EnterpriseAdminLearnerProgress.objects.get(enterprise_customer_uuid=enterprise_id) - learner_progress = serializers.EnterpriseAdminLearnerProgressSerializer(learner_progress).data - response_data['learner_progress'] = learner_progress - except EnterpriseAdminLearnerProgress.DoesNotExist: - pass - - try: - learner_engagement = EnterpriseAdminSummarizeInsights.objects.get(enterprise_customer_uuid=enterprise_id) - learner_engagement = serializers.EnterpriseAdminSummarizeInsightsSerializer(learner_engagement).data - response_data['learner_engagement'] = learner_engagement - except EnterpriseAdminSummarizeInsights.DoesNotExist: - pass - - status = HTTP_200_OK - if learner_progress == {} and learner_engagement == {}: - status = HTTP_404_NOT_FOUND - - return Response(data=response_data, status=status) diff --git a/enterprise_data/api/v1/views/enterprise_offers.py b/enterprise_data/api/v1/views/enterprise_offers.py new file mode 100644 index 00000000..affea38a --- /dev/null +++ b/enterprise_data/api/v1/views/enterprise_offers.py @@ -0,0 +1,41 @@ +""" +Views for enterprise offers +""" +from django_filters.rest_framework import DjangoFilterBackend +from rest_framework import filters, viewsets + +from enterprise_data.api.v1 import serializers +from enterprise_data.models import EnterpriseOffer + +from .base import EnterpriseViewSetMixin + + +class EnterpriseOfferViewSet(EnterpriseViewSetMixin, viewsets.ReadOnlyModelViewSet): + """ + Viewset for enterprise offers. + """ + serializer_class = serializers.EnterpriseOfferSerializer + filter_backends = (filters.OrderingFilter, DjangoFilterBackend,) + ordering_fields = '__all__' + + lookup_field = 'offer_id' + + filterset_fields = ( + 'offer_id', + 'status' + ) + + def get_object(self): + """ + This ensures that UUIDs with dashes are properly handled when requesting info about offers. + + Related to the work in EnterpriseOfferSerializer with `to_internal_value` and `to_representation` + """ + self.kwargs['offer_id'] = self.kwargs['offer_id'].replace('-', '') + return super().get_object() + + def get_queryset(self): + enterprise_customer_uuid = self.kwargs['enterprise_id'] + return EnterpriseOffer.objects.filter( + enterprise_customer_uuid=enterprise_customer_uuid, + ) diff --git a/enterprise_data/tests/test_filters.py b/enterprise_data/tests/test_filters.py index a7de0cb3..c6f2fabb 100644 --- a/enterprise_data/tests/test_filters.py +++ b/enterprise_data/tests/test_filters.py @@ -12,7 +12,7 @@ from django.conf import settings -from enterprise_data.api.v1.views import EnterpriseLearnerViewSet +from enterprise_data.api.v1.views.enterprise_learner import EnterpriseLearnerViewSet from enterprise_data.filters import AuditUsersEnrollmentFilterBackend from enterprise_data.models import EnterpriseEnrollment, EnterpriseLearnerEnrollment from enterprise_data.tests.mixins import JWTTestMixin diff --git a/enterprise_data/utils.py b/enterprise_data/utils.py index 1ff3cd74..a98bbc1f 100644 --- a/enterprise_data/utils.py +++ b/enterprise_data/utils.py @@ -1,9 +1,14 @@ """ Utility functions for Enterprise Data app. """ - import hashlib import random +import time +from datetime import timedelta +from functools import wraps +from logging import getLogger + +LOGGER = getLogger(__name__) def get_cache_key(**kwargs): @@ -35,3 +40,45 @@ def get_unique_id(): Return a unique 32 bit integer. """ return random.getrandbits(32) + + +def subtract_one_month(original_date): + """ + Return a date exactly one month prior to the passed in date. + """ + one_day = timedelta(days=1) + one_month_earlier = original_date - one_day + while one_month_earlier.month == original_date.month or one_month_earlier.day > original_date.day: + one_month_earlier -= one_day + return one_month_earlier + + +def timeit(func): + """ + Measure time taken by a function. + """ + @wraps(func) + def wrapper(*args, **kwargs): + start = time.time() + result = func(*args, **kwargs) + end = time.time() + LOGGER.info(f'Time taken by {func.__name__}: {end - start} seconds') + return result + + return wrapper + + +def date_filter(start, end, data_frame, date_column): + """ + Filter a pandas DataFrame by date range. + + Arguments: + start (DatetimeScalar | NaTType | None): The start date. + end (DatetimeScalar | NaTType | None): The end date. + data_frame (pandas.DataFrame): The DataFrame to filter. + date_column (str): The name of the date column. + + Returns: + (pandas.DataFrame): The filtered DataFrame. + """ + return data_frame[(start <= data_frame[date_column]) & (data_frame[date_column] <= end)] diff --git a/enterprise_reporting/clients/__init__.py b/enterprise_reporting/clients/__init__.py index 1734a1c9..7e8f4203 100644 --- a/enterprise_reporting/clients/__init__.py +++ b/enterprise_reporting/clients/__init__.py @@ -2,18 +2,17 @@ Clients used to access third party systems. """ +import logging import os from datetime import datetime, timedelta from functools import wraps from urllib.parse import parse_qs, urljoin, urlparse -from edx_rest_api_client.client import get_oauth_access_token -import logging import requests +from edx_rest_api_client.client import get_oauth_access_token from enterprise_reporting.utils import retry_on_exception - LOGGER = logging.getLogger(__name__) diff --git a/enterprise_reporting/external_resource_link_report.py b/enterprise_reporting/external_resource_link_report.py index c8f51d61..619c441b 100644 --- a/enterprise_reporting/external_resource_link_report.py +++ b/enterprise_reporting/external_resource_link_report.py @@ -3,13 +3,13 @@ """ -from collections import Counter -from datetime import date -import operator import logging +import operator import os import re import sys +from collections import Counter +from datetime import date from urllib.parse import urlparse from py2neo import Graph diff --git a/enterprise_reporting/tests/test_clients.py b/enterprise_reporting/tests/test_clients.py index 3dd3a6c0..2338b9ef 100644 --- a/enterprise_reporting/tests/test_clients.py +++ b/enterprise_reporting/tests/test_clients.py @@ -1,9 +1,9 @@ """ Tests for clients in enterprise_reporting. """ +from datetime import datetime, timedelta from unittest.mock import Mock, patch from urllib.parse import urljoin -from datetime import datetime, timedelta import responses diff --git a/enterprise_reporting/tests/test_enterprise_client.py b/enterprise_reporting/tests/test_enterprise_client.py index a912c730..155462e0 100644 --- a/enterprise_reporting/tests/test_enterprise_client.py +++ b/enterprise_reporting/tests/test_enterprise_client.py @@ -2,14 +2,11 @@ Test Enterprise client. """ +import json import os import unittest -import json - -from enterprise_reporting.utils import ( - extract_catalog_uuids_from_reporting_config, -) +from enterprise_reporting.utils import extract_catalog_uuids_from_reporting_config REPO_DIR = os.getcwd() FIXTURE_DIR = os.path.join(REPO_DIR, 'enterprise_reporting/fixtures') diff --git a/enterprise_reporting/tests/test_external_link_report.py b/enterprise_reporting/tests/test_external_link_report.py index f1914550..14c61e19 100644 --- a/enterprise_reporting/tests/test_external_link_report.py +++ b/enterprise_reporting/tests/test_external_link_report.py @@ -2,13 +2,13 @@ Test utils for external link reports. """ -from collections import OrderedDict import unittest +from collections import OrderedDict from enterprise_reporting.external_resource_link_report import ( AGGREGATE_REPORT_CSV_HEADER_ROW, - create_csv_string, create_columns_for_aggregate_report, + create_csv_string, process_coursegraph_results, split_up_results, ) diff --git a/enterprise_reporting/tests/test_utils.py b/enterprise_reporting/tests/test_utils.py index d3df3307..06b17724 100644 --- a/enterprise_reporting/tests/test_utils.py +++ b/enterprise_reporting/tests/test_utils.py @@ -3,12 +3,15 @@ """ +import datetime import os import tempfile import unittest from collections import OrderedDict + import ddt import pgpy +import pytz from pgpy.constants import CompressionAlgorithm, HashAlgorithm, KeyFlags, PubKeyAlgorithm, SymmetricKeyAlgorithm from pgpy.errors import PGPError @@ -16,9 +19,6 @@ from .utils import create_files, verify_compressed -import pytz -import datetime - @ddt.ddt class TestUtilities(unittest.TestCase): diff --git a/enterprise_reporting/utils.py b/enterprise_reporting/utils.py index 573c824a..718cd750 100644 --- a/enterprise_reporting/utils.py +++ b/enterprise_reporting/utils.py @@ -11,6 +11,7 @@ from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText +from urllib.parse import parse_qs, urlparse import boto3 import pgpy @@ -18,7 +19,6 @@ import pytz from cryptography.fernet import Fernet from fernet_fields.hkdf import derive_fernet_key -from urllib.parse import parse_qs, urlparse from django.utils.encoding import force_str diff --git a/requirements/base.in b/requirements/base.in index 3453a660..f232713d 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -13,3 +13,6 @@ django-model-utils edx-rbac rules factory_boy +numpy +pandas +mysql-connector-python diff --git a/requirements/base.txt b/requirements/base.txt index 44a9d702..49b5f1cd 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # make upgrade @@ -10,15 +10,15 @@ asgiref==3.8.1 # via django asn1crypto==1.5.1 # via snowflake-connector-python -awscli==1.33.27 +awscli==1.33.31 # via -r requirements/reporting.in -bcrypt==4.1.3 +bcrypt==4.2.0 # via paramiko billiard==4.2.0 # via celery -boto3==1.34.145 +boto3==1.34.149 # via -r requirements/reporting.in -botocore==1.34.145 +botocore==1.34.149 # via # awscli # boto3 @@ -143,12 +143,23 @@ kombu==5.3.7 # via celery monotonic==1.6 # via py2neo +mysql-connector-python==9.0.0 + # via -r requirements/base.in newrelic==9.12.0 # via edx-django-utils +numpy==1.24.4 + # via + # -c requirements/constraints.txt + # -r requirements/base.in + # pandas packaging==24.1 # via # py2neo # snowflake-connector-python +pandas==2.0.3 + # via + # -c requirements/constraints.txt + # -r requirements/base.in pansi==2020.7.3 # via py2neo paramiko==3.4.0 @@ -194,10 +205,12 @@ python-dateutil==2.9.0.post0 # botocore # celery # faker + # pandas # vertica-python pytz==2024.1 # via # interchange + # pandas # snowflake-connector-python pyyaml==6.0.1 # via awscli @@ -228,7 +241,7 @@ six==1.16.0 # vertica-python slumber==0.7.1 # via edx-rest-api-client -snowflake-connector-python==3.11.0 +snowflake-connector-python==3.12.0 # via -r requirements/reporting.in sortedcontainers==2.4.0 # via snowflake-connector-python @@ -242,10 +255,14 @@ tomlkit==0.13.0 # via snowflake-connector-python typing-extensions==4.12.2 # via + # asgiref # edx-opaque-keys + # kombu # snowflake-connector-python tzdata==2024.1 - # via celery + # via + # celery + # pandas unicodecsv==0.14.1 # via -r requirements/reporting.in urllib3==1.26.19 @@ -254,6 +271,7 @@ urllib3==1.26.19 # botocore # py2neo # requests + # snowflake-connector-python vertica-python==1.4.0 # via -r requirements/reporting.in vine==5.1.0 diff --git a/requirements/ci.txt b/requirements/ci.txt index 38c9e484..c6907d0f 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # make upgrade @@ -30,6 +30,10 @@ pluggy==1.5.0 # via tox pyproject-api==1.7.1 # via tox +tomli==2.0.1 + # via + # pyproject-api + # tox tox==4.16.0 # via -r requirements/ci.in virtualenv==20.26.3 diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt index 94b56b99..b957ec4c 100644 --- a/requirements/common_constraints.txt +++ b/requirements/common_constraints.txt @@ -1,4 +1,3 @@ - # A central location for most common version constraints # (across edx repos) for pip-installation. # diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 9edcdccd..13de12fc 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -18,3 +18,6 @@ backports.zoneinfo; python_version<"3.9" # botocore 1.34.145 depends on urllib3<1.27 and >=1.25.4; python_version < "3.10" urllib3<2.0.0 + +numpy<=1.24.4 +pandas<=2.0.3 \ No newline at end of file diff --git a/requirements/dev.txt b/requirements/dev.txt index 5cbb21e7..3f7a4926 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # make upgrade @@ -14,17 +14,17 @@ astroid==3.2.4 # via # pylint # pylint-celery -awscli==1.33.27 +awscli==1.33.31 # via -r requirements/reporting.in backports-tarfile==1.2.0 # via jaraco-context -bcrypt==4.1.3 +bcrypt==4.2.0 # via paramiko billiard==4.2.0 # via celery -boto3==1.34.145 +boto3==1.34.149 # via -r requirements/reporting.in -botocore==1.34.145 +botocore==1.34.149 # via # awscli # boto3 @@ -86,7 +86,6 @@ cryptography==42.0.8 # pgpy # pyjwt # pyopenssl - # secretstorage # snowflake-connector-python diff-cover==9.1.1 # via -r requirements/dev-enterprise_data.in @@ -179,6 +178,7 @@ idna==3.7 importlib-metadata==6.11.0 # via # -c requirements/common_constraints.txt + # build # keyring # twine interchange==2021.0.4 @@ -193,10 +193,6 @@ jaraco-context==5.3.0 # via keyring jaraco-functools==4.0.1 # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage jinja2==3.1.4 # via # code-annotations @@ -229,10 +225,17 @@ more-itertools==10.3.0 # via # jaraco-classes # jaraco-functools +mysql-connector-python==9.0.0 + # via -r requirements/base.in newrelic==9.12.0 # via edx-django-utils nh3==0.2.18 # via readme-renderer +numpy==1.24.4 + # via + # -c requirements/constraints.txt + # -r requirements/base.in + # pandas packaging==24.1 # via # build @@ -240,6 +243,10 @@ packaging==24.1 # pyproject-api # snowflake-connector-python # tox +pandas==2.0.3 + # via + # -c requirements/constraints.txt + # -r requirements/base.in pansi==2020.7.3 # via py2neo paramiko==3.4.0 @@ -329,12 +336,14 @@ python-dateutil==2.9.0.post0 # botocore # celery # faker + # pandas # vertica-python python-slugify==8.0.4 # via code-annotations pytz==2024.1 # via # interchange + # pandas # snowflake-connector-python pyyaml==6.0.1 # via @@ -366,8 +375,6 @@ s3transfer==0.10.2 # via # awscli # boto3 -secretstorage==3.3.3 - # via keyring semantic-version==2.10.0 # via edx-drf-extensions six==1.16.0 @@ -383,7 +390,7 @@ slumber==0.7.1 # via edx-rest-api-client snowballstemmer==2.2.0 # via pydocstyle -snowflake-connector-python==3.11.0 +snowflake-connector-python==3.12.0 # via -r requirements/reporting.in sortedcontainers==2.4.0 # via snowflake-connector-python @@ -398,6 +405,13 @@ testfixtures==8.3.0 # via -r requirements/quality.in text-unidecode==1.3 # via python-slugify +tomli==2.0.1 + # via + # build + # pip-tools + # pylint + # pyproject-api + # tox tomlkit==0.13.0 # via # pylint @@ -408,10 +422,16 @@ twine==5.1.1 # via -r requirements/dev-enterprise_data.in typing-extensions==4.12.2 # via + # asgiref + # astroid # edx-opaque-keys + # kombu + # pylint # snowflake-connector-python tzdata==2024.1 - # via celery + # via + # celery + # pandas unicodecsv==0.14.1 # via -r requirements/reporting.in urllib3==1.26.19 @@ -420,6 +440,7 @@ urllib3==1.26.19 # botocore # py2neo # requests + # snowflake-connector-python # twine vertica-python==1.4.0 # via -r requirements/reporting.in diff --git a/requirements/pip.txt b/requirements/pip.txt index 854334df..9ef0a77b 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # make upgrade diff --git a/requirements/pip_tools.txt b/requirements/pip_tools.txt index 0b0b25e9..22d70da3 100644 --- a/requirements/pip_tools.txt +++ b/requirements/pip_tools.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # make upgrade @@ -8,6 +8,10 @@ build==1.2.1 # via pip-tools click==8.1.7 # via pip-tools +importlib-metadata==6.11.0 + # via + # -c requirements/common_constraints.txt + # build packaging==24.1 # via build pip-tools==7.4.1 @@ -16,8 +20,14 @@ pyproject-hooks==1.1.0 # via # build # pip-tools +tomli==2.0.1 + # via + # build + # pip-tools wheel==0.43.0 # via pip-tools +zipp==3.19.2 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/quality.txt b/requirements/quality.txt index ba0f202b..615f1791 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # make upgrade @@ -14,17 +14,17 @@ astroid==3.2.4 # via # pylint # pylint-celery -awscli==1.33.27 +awscli==1.33.31 # via -r requirements/reporting.in backports-tarfile==1.2.0 # via jaraco-context -bcrypt==4.1.3 +bcrypt==4.2.0 # via paramiko billiard==4.2.0 # via celery -boto3==1.34.145 +boto3==1.34.149 # via -r requirements/reporting.in -botocore==1.34.145 +botocore==1.34.149 # via # awscli # boto3 @@ -88,7 +88,6 @@ cryptography==42.0.8 # pgpy # pyjwt # pyopenssl - # secretstorage # snowflake-connector-python ddt==1.7.2 # via -r requirements/test.in @@ -168,6 +167,8 @@ edx-rbac==1.9.0 # via -r requirements/base.in edx-rest-api-client==5.7.1 # via -r requirements/base.in +exceptiongroup==1.2.2 + # via pytest factory-boy==3.3.0 # via # -r requirements/base.in @@ -190,6 +191,7 @@ idna==3.7 importlib-metadata==6.11.0 # via # -c requirements/common_constraints.txt + # build # keyring # twine iniconfig==2.0.0 @@ -206,10 +208,6 @@ jaraco-context==5.3.0 # via keyring jaraco-functools==4.0.1 # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage jinja2==3.1.4 # via # code-annotations @@ -244,10 +242,17 @@ more-itertools==10.3.0 # via # jaraco-classes # jaraco-functools +mysql-connector-python==9.0.0 + # via -r requirements/base.in newrelic==9.12.0 # via edx-django-utils nh3==0.2.18 # via readme-renderer +numpy==1.24.4 + # via + # -c requirements/constraints.txt + # -r requirements/base.in + # pandas packaging==24.1 # via # build @@ -256,6 +261,10 @@ packaging==24.1 # pytest # snowflake-connector-python # tox +pandas==2.0.3 + # via + # -c requirements/constraints.txt + # -r requirements/base.in pansi==2020.7.3 # via py2neo paramiko==3.4.0 @@ -341,7 +350,7 @@ pyproject-hooks==1.1.0 # via # build # pip-tools -pytest==8.3.1 +pytest==8.3.2 # via # pytest-cov # pytest-django @@ -355,12 +364,14 @@ python-dateutil==2.9.0.post0 # celery # faker # freezegun + # pandas # vertica-python python-slugify==8.0.4 # via code-annotations pytz==2024.1 # via # interchange + # pandas # snowflake-connector-python pyyaml==6.0.1 # via @@ -396,8 +407,6 @@ s3transfer==0.10.2 # via # awscli # boto3 -secretstorage==3.3.3 - # via keyring semantic-version==2.10.0 # via edx-drf-extensions six==1.16.0 @@ -413,7 +422,7 @@ slumber==0.7.1 # via edx-rest-api-client snowballstemmer==2.2.0 # via pydocstyle -snowflake-connector-python==3.11.0 +snowflake-connector-python==3.12.0 # via -r requirements/reporting.in sortedcontainers==2.4.0 # via snowflake-connector-python @@ -430,6 +439,15 @@ testfixtures==8.3.0 # -r requirements/test.in text-unidecode==1.3 # via python-slugify +tomli==2.0.1 + # via + # build + # coverage + # pip-tools + # pylint + # pyproject-api + # pytest + # tox tomlkit==0.13.0 # via # pylint @@ -440,10 +458,16 @@ twine==5.1.1 # via -r requirements/dev-enterprise_data.in typing-extensions==4.12.2 # via + # asgiref + # astroid # edx-opaque-keys + # kombu + # pylint # snowflake-connector-python tzdata==2024.1 - # via celery + # via + # celery + # pandas unicodecsv==0.14.1 # via -r requirements/reporting.in urllib3==1.26.19 @@ -453,6 +477,7 @@ urllib3==1.26.19 # py2neo # requests # responses + # snowflake-connector-python # twine vertica-python==1.4.0 # via -r requirements/reporting.in diff --git a/requirements/test-master.txt b/requirements/test-master.txt index ed3b2db9..19c78018 100644 --- a/requirements/test-master.txt +++ b/requirements/test-master.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # make upgrade @@ -10,15 +10,15 @@ asgiref==3.8.1 # via django asn1crypto==1.5.1 # via snowflake-connector-python -awscli==1.33.27 +awscli==1.33.31 # via -r requirements/reporting.in -bcrypt==4.1.3 +bcrypt==4.2.0 # via paramiko billiard==4.2.0 # via celery -boto3==1.34.145 +boto3==1.34.149 # via -r requirements/reporting.in -botocore==1.34.145 +botocore==1.34.149 # via # awscli # boto3 @@ -130,6 +130,8 @@ edx-rbac==1.9.0 # via -r requirements/base.in edx-rest-api-client==5.7.1 # via -r requirements/base.in +exceptiongroup==1.2.2 + # via pytest factory-boy==3.3.0 # via # -r requirements/base.in @@ -160,13 +162,24 @@ mock==5.1.0 # via -r requirements/test.in monotonic==1.6 # via py2neo +mysql-connector-python==9.0.0 + # via -r requirements/base.in newrelic==9.12.0 # via edx-django-utils +numpy==1.24.4 + # via + # -c requirements/constraints.txt + # -r requirements/base.in + # pandas packaging==24.1 # via # py2neo # pytest # snowflake-connector-python +pandas==2.0.3 + # via + # -c requirements/constraints.txt + # -r requirements/base.in pansi==2020.7.3 # via py2neo paramiko==3.4.0 @@ -209,7 +222,7 @@ pynacl==1.5.0 # paramiko pyopenssl==24.2.1 # via snowflake-connector-python -pytest==8.3.1 +pytest==8.3.2 # via # pytest-cov # pytest-django @@ -223,10 +236,12 @@ python-dateutil==2.9.0.post0 # celery # faker # freezegun + # pandas # vertica-python pytz==2024.1 # via # interchange + # pandas # snowflake-connector-python pyyaml==6.0.1 # via @@ -262,7 +277,7 @@ six==1.16.0 # vertica-python slumber==0.7.1 # via edx-rest-api-client -snowflake-connector-python==3.11.0 +snowflake-connector-python==3.12.0 # via -r requirements/reporting.in sortedcontainers==2.4.0 # via snowflake-connector-python @@ -274,14 +289,22 @@ stevedore==5.2.0 # edx-opaque-keys testfixtures==8.3.0 # via -r requirements/test.in +tomli==2.0.1 + # via + # coverage + # pytest tomlkit==0.13.0 # via snowflake-connector-python typing-extensions==4.12.2 # via + # asgiref # edx-opaque-keys + # kombu # snowflake-connector-python tzdata==2024.1 - # via celery + # via + # celery + # pandas unicodecsv==0.14.1 # via -r requirements/reporting.in urllib3==1.26.19 @@ -291,6 +314,7 @@ urllib3==1.26.19 # py2neo # requests # responses + # snowflake-connector-python vertica-python==1.4.0 # via -r requirements/reporting.in vine==5.1.0 diff --git a/requirements/test-reporting.txt b/requirements/test-reporting.txt index 4aa73e38..aaa31c4d 100644 --- a/requirements/test-reporting.txt +++ b/requirements/test-reporting.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # make upgrade @@ -8,15 +8,15 @@ amqp==5.2.0 # via kombu asn1crypto==1.5.1 # via snowflake-connector-python -awscli==1.33.27 +awscli==1.33.31 # via -r requirements/reporting.in -bcrypt==4.1.3 +bcrypt==4.2.0 # via paramiko billiard==4.2.0 # via celery -boto3==1.34.145 +boto3==1.34.149 # via -r requirements/reporting.in -botocore==1.34.145 +botocore==1.34.149 # via # awscli # boto3 @@ -72,6 +72,8 @@ distlib==0.3.8 # via virtualenv docutils==0.16 # via awscli +exceptiongroup==1.2.2 + # via pytest filelock==3.15.4 # via # snowflake-connector-python @@ -180,16 +182,24 @@ six==1.16.0 # py2neo # python-dateutil # vertica-python -snowflake-connector-python==3.11.0 +snowflake-connector-python==3.12.0 # via -r requirements/reporting.in sortedcontainers==2.4.0 # via snowflake-connector-python +tomli==2.0.1 + # via + # coverage + # pyproject-api + # pytest + # tox tomlkit==0.13.0 # via snowflake-connector-python tox==4.16.0 # via -r requirements/test-reporting.in typing-extensions==4.12.2 - # via snowflake-connector-python + # via + # kombu + # snowflake-connector-python tzdata==2024.1 # via celery unicodecsv==0.14.1 @@ -201,6 +211,7 @@ urllib3==1.26.19 # py2neo # requests # responses + # snowflake-connector-python vertica-python==1.4.0 # via -r requirements/reporting.in vine==5.1.0 diff --git a/requirements/test.txt b/requirements/test.txt index d095fba4..33ff4dec 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # make upgrade @@ -10,15 +10,15 @@ asgiref==3.8.1 # via django asn1crypto==1.5.1 # via snowflake-connector-python -awscli==1.33.27 +awscli==1.33.31 # via -r requirements/reporting.in -bcrypt==4.1.3 +bcrypt==4.2.0 # via paramiko billiard==4.2.0 # via celery -boto3==1.34.145 +boto3==1.34.149 # via -r requirements/reporting.in -botocore==1.34.145 +botocore==1.34.149 # via # awscli # boto3 @@ -128,6 +128,8 @@ edx-rbac==1.9.0 # via -r requirements/base.in edx-rest-api-client==5.7.1 # via -r requirements/base.in +exceptiongroup==1.2.2 + # via pytest factory-boy==3.3.0 # via # -r requirements/base.in @@ -158,13 +160,24 @@ mock==5.1.0 # via -r requirements/test.in monotonic==1.6 # via py2neo +mysql-connector-python==9.0.0 + # via -r requirements/base.in newrelic==9.12.0 # via edx-django-utils +numpy==1.24.4 + # via + # -c requirements/constraints.txt + # -r requirements/base.in + # pandas packaging==24.1 # via # py2neo # pytest # snowflake-connector-python +pandas==2.0.3 + # via + # -c requirements/constraints.txt + # -r requirements/base.in pansi==2020.7.3 # via py2neo paramiko==3.4.0 @@ -207,7 +220,7 @@ pynacl==1.5.0 # paramiko pyopenssl==24.2.1 # via snowflake-connector-python -pytest==8.3.1 +pytest==8.3.2 # via # pytest-cov # pytest-django @@ -221,10 +234,12 @@ python-dateutil==2.9.0.post0 # celery # faker # freezegun + # pandas # vertica-python pytz==2024.1 # via # interchange + # pandas # snowflake-connector-python pyyaml==6.0.1 # via @@ -260,7 +275,7 @@ six==1.16.0 # vertica-python slumber==0.7.1 # via edx-rest-api-client -snowflake-connector-python==3.11.0 +snowflake-connector-python==3.12.0 # via -r requirements/reporting.in sortedcontainers==2.4.0 # via snowflake-connector-python @@ -272,14 +287,22 @@ stevedore==5.2.0 # edx-opaque-keys testfixtures==8.3.0 # via -r requirements/test.in +tomli==2.0.1 + # via + # coverage + # pytest tomlkit==0.13.0 # via snowflake-connector-python typing-extensions==4.12.2 # via + # asgiref # edx-opaque-keys + # kombu # snowflake-connector-python tzdata==2024.1 - # via celery + # via + # celery + # pandas unicodecsv==0.14.1 # via -r requirements/reporting.in urllib3==1.26.19 @@ -289,6 +312,7 @@ urllib3==1.26.19 # py2neo # requests # responses + # snowflake-connector-python vertica-python==1.4.0 # via -r requirements/reporting.in vine==5.1.0