diff --git a/enterprise_subsidy/apps/content_metadata/api.py b/enterprise_subsidy/apps/content_metadata/api.py index b07f4066..adee857a 100644 --- a/enterprise_subsidy/apps/content_metadata/api.py +++ b/enterprise_subsidy/apps/content_metadata/api.py @@ -6,6 +6,7 @@ from decimal import Decimal from django.conf import settings +from django.utils import functional, timezone from edx_django_utils.cache import TieredCache from enterprise_subsidy.apps.api_client.enterprise_catalog import EnterpriseCatalogApiClient @@ -13,6 +14,7 @@ from enterprise_subsidy.apps.subsidy.constants import CENTS_PER_DOLLAR from .constants import CourseModes, ProductSources +from .models import ReplicatedContentMetadata logger = logging.getLogger(__name__) @@ -34,6 +36,112 @@ def content_metadata_cache_key(enterprise_customer_uuid, content_key): return versioned_cache_key(CACHE_NAMESPACE, enterprise_customer_uuid, content_key) +def _price_for_content(content_data, course_run_data): + """ + Helper to return the "official" price for content. + The endpoint at ``self.content_metadata_url`` will always return price fields + as USD (dollars), possibly as a string or a float. This method converts + those values to USD cents as an integer. + """ + content_price = None + if course_run_data.get('first_enrollable_paid_seat_price'): + content_price = course_run_data['first_enrollable_paid_seat_price'] + + if not content_price: + enrollment_mode_for_content = _mode_for_content(content_data) + for entitlement in content_data.get('entitlements', []): + if entitlement.get('mode') == enrollment_mode_for_content: + content_price = entitlement.get('price') + + if content_price: + return int(Decimal(content_price) * CENTS_PER_DOLLAR) + else: + logger.info( + f"Could not determine price for content key {content_data.get('key')} " + f"and course run key {course_run_data.get('key')}" + ) + return None + + +def _mode_for_content(content_data): + """ + Helper to extract the relevant enrollment mode for a piece of content metadata. + """ + product_source = _product_source_for_content(content_data) + return CONTENT_MODES_BY_PRODUCT_SOURCE.get(product_source, CourseModes.EDX_VERIFIED.value) + + +def _product_source_for_content(content_data): + """ + Helps get the product source string, given a dict of ``content_data``. + """ + if product_source := content_data.get('product_source'): + source_name = product_source.get('slug') + if source_name in CONTENT_MODES_BY_PRODUCT_SOURCE: + return source_name + return ProductSources.EDX.value + + +def _get_geag_variant_id_for_content(content_data): + """ + Returns the GEAG ``variant_id`` or ``None``, given a dict of ``content_data``. + In the GEAG system a ``variant_id`` is aka a ``product_id``. + """ + variant_id = None + if additional_metadata := content_data.get('additional_metadata'): + variant_id = additional_metadata.get('variant_id') + return variant_id + + +def _get_course_run(content_identifier, content_data): + """ + Given a content_identifier (key, run key, uuid) extract the appropriate course_run. + When given a run key or uuid for a run, extract that. When given a course key or + course uuid, extract the advertised course_run. + """ + if content_data.get('content_type') == 'courserun': + return content_data + + course_run_identifier = content_identifier + # if the supplied content_identifer refers to the course, look for an advertised run + if content_identifier == content_data.get('key') or content_identifier == content_data.get('uuid'): + course_run_identifier = content_data.get('advertised_course_run_uuid') + for course_run in content_data.get('course_runs', []): + if course_run_identifier == course_run.get('key') or course_run_identifier == course_run.get('uuid'): + return course_run + return {} + + +def _get_and_cache_content_metadata(enterprise_customer_uuid, content_identifier): + """ + Fetches details about the given content from a tiered (request + django) cache; + or it fetches from the enterprise-catalog API if not present in the cache, + and then caches that result. + """ + cache_key = content_metadata_cache_key(enterprise_customer_uuid, content_identifier) + cached_response = TieredCache.get_cached_response(cache_key) + if cached_response.is_found: + return cached_response.value + + course_details = EnterpriseCatalogApiClient().get_content_metadata_for_customer( + enterprise_customer_uuid, + content_identifier + ) + if course_details: + TieredCache.set_all_tiers( + cache_key, + course_details, + django_cache_timeout=CONTENT_METADATA_CACHE_TIMEOUT, + ) + logger.info( + 'Fetched course details %s for customer %s and content_identifier %s', + course_details, + enterprise_customer_uuid, + content_identifier, + ) + return course_details + + class ContentMetadataApi: """ An API for interacting with enterprise catalog content metadata. @@ -54,51 +162,26 @@ def price_for_content(self, content_data, course_run_data): as USD (dollars), possibly as a string or a float. This method converts those values to USD cents as an integer. """ - content_price = None - if course_run_data.get('first_enrollable_paid_seat_price'): - content_price = course_run_data['first_enrollable_paid_seat_price'] - - if not content_price: - enrollment_mode_for_content = self.mode_for_content(content_data) - for entitlement in content_data.get('entitlements', []): - if entitlement.get('mode') == enrollment_mode_for_content: - content_price = entitlement.get('price') - - if content_price: - return int(Decimal(content_price) * CENTS_PER_DOLLAR) - else: - logger.info( - f"Could not determine price for content key {content_data.get('key')} " - f"and course run key {course_run_data.get('key')}" - ) - return None + return _price_for_content(content_data, course_run_data) def mode_for_content(self, content_data): """ Helper to extract the relevant enrollment mode for a piece of content metadata. """ - product_source = self.product_source_for_content(content_data) - return CONTENT_MODES_BY_PRODUCT_SOURCE.get(product_source, CourseModes.EDX_VERIFIED.value) + return _mode_for_content(content_data) def product_source_for_content(self, content_data): """ Helps get the product source string, given a dict of ``content_data``. """ - if product_source := content_data.get('product_source'): - source_name = product_source.get('slug') - if source_name in CONTENT_MODES_BY_PRODUCT_SOURCE: - return source_name - return ProductSources.EDX.value + return _product_source_for_content(content_data) def get_geag_variant_id_for_content(self, content_data): """ Returns the GEAG ``variant_id`` or ``None``, given a dict of ``content_data``. In the GEAG system a ``variant_id`` is aka a ``product_id``. """ - variant_id = None - if additional_metadata := content_data.get('additional_metadata'): - variant_id = additional_metadata.get('variant_id') - return variant_id + return _get_geag_variant_id_for_content(content_data) def summary_data_for_content(self, content_identifier, content_data): """ @@ -123,17 +206,7 @@ def get_course_run(self, content_identifier, content_data): When given a run key or uuid for a run, extract that. When given a course key or course uuid, extract the advertised course_run. """ - if content_data.get('content_type') == 'courserun': - return content_data - - course_run_identifier = content_identifier - # if the supplied content_identifer refers to the course, look for an advertised run - if content_identifier == content_data.get('key') or content_identifier == content_data.get('uuid'): - course_run_identifier = content_data.get('advertised_course_run_uuid') - for course_run in content_data.get('course_runs', []): - if course_run_identifier == course_run.get('key') or course_run_identifier == course_run.get('uuid'): - return course_run - return {} + return _get_course_run(content_identifier, content_data) def get_content_summary(self, enterprise_customer_uuid, content_identifier): """ @@ -206,25 +279,125 @@ def get_content_metadata(enterprise_customer_uuid, content_identifier): or it fetches from the enterprise-catalog API if not present in the cache, and then caches that result. """ - cache_key = content_metadata_cache_key(enterprise_customer_uuid, content_identifier) - cached_response = TieredCache.get_cached_response(cache_key) - if cached_response.is_found: - return cached_response.value + return _get_and_cache_content_metadata(enterprise_customer_uuid, content_identifier) - course_details = EnterpriseCatalogApiClient().get_content_metadata_for_customer( - enterprise_customer_uuid, - content_identifier + +class UpstreamContentMetadata: + """ + Expected interactions with this class: + # given a customer and desired course/run key + enterprise_customer_uuid = 'some-uuid' + content_identifier = 'whatever-course-or-run-key' + + # fetches course/run metadata from upstream or cache + metadata_wrapper = UpstreamContentMetadata(enterprise_customer_uuid, content_identifier) + """ + def __init__(self, enterprise_customer_uuid, content_identifier): + self.enterprise_customer_uuid = enterprise_customer_uuid + self.content_identifier = content_identifier + self.raw_metadata = None + self.course_run_data = None + self.fetch() + + def fetch(self): + self.raw_metadata = _get_and_cache_content_metadata( + self.enterprise_customer_uuid, + self.content_identifier, ) - if course_details: - TieredCache.set_all_tiers( - cache_key, - course_details, - django_cache_timeout=CONTENT_METADATA_CACHE_TIMEOUT, - ) - logger.info( - 'Fetched course details %s for customer %s and content_identifier %s', - course_details, - enterprise_customer_uuid, - content_identifier, + self.course_run_data = _get_course_run(self.content_identifier, self.raw_metadata) + + @functional.cached_property + def title(self): + self.raw_metadata.get('title') + + @functional.cached_property + def price(self): + return _price_for_content(self.raw_metadata, self.course_run_data) + + @functional.cached_property + def course_mode(self): + return _mode_for_content(self.raw_metadata) + + @functional.cached_property + def product_source(self): + return _product_source_for_content(self.raw_metadata) + + @functional.cached_property + def geag_variant_id(self): + return _get_geag_variant_id_for_content(self.raw_metadata) + + @functional.cached_property + def content_key(self): + return self.raw_metadata.get('key') + + @functional.cached_property + def course_run_key(self): + return self.course_run_data.get('key') + + @functional.cached_property + def content_type(self): + return self.raw_metadata.get('content_type') + + +def test_upstream(): + return UpstreamContentMetadata( + enterprise_customer_uuid='378d5bf0-f67d-4bf7-8b2a-cbbc53d0f772', + content_identifier='course-v1:edX+DemoX+Demo_Course', + ) + + +class ContentMetadataReplicator: + """ + """ + def __init__(self, enterprise_customer_uuid, content_identifier): + self.enterprise_customer_uuid = enterprise_customer_uuid + self.content_identifier = content_identifier + self.upstream_data = None + + def fetch_upstream(self): + self.upstream_data = UpstreamContentMetadata( + enterprise_customer_uuid=self.enterprise_customer_uuid, + content_identifier=self.content_identifier, + ) + + def get_recent_metadata_record(self): + return ReplicatedContentMetadata.get_recent_record_or_null( + enterprise_customer_uuid=self.enterprise_customer_uuid, + content_key=self.content_identifier, + ) + + def _replicate_metadata_record(self): + self.fetch_upstream() + new_values = { + 'content_type': self.upstream_data.content_type, + 'raw_metadata': self.upstream_data.raw_metadata, + 'raw_fetched_at': timezone.now(), + 'title': self.upstream_data.title, + 'price': self.upstream_data.price, + 'product_source': self.upstream_data.product_source, + 'course_mode': self.upstream_data.course_mode, + } + record, _ = ReplicatedContentMetadata.objects.update_or_create( + enterprise_customer_uuid=self.enterprise_customer_uuid, + content_key=self.content_identifier, + defaults=new_values, ) - return course_details + return record + + def replicate(self): + # Check for a recent replicated record in the DB. + # If we have a good one, return it. + record = self.get_recent_metadata_record() + if record: + return record + + # Otherwise, fetch from upstream, replicate, and return + return self._replicate_metadata_record() + + +def test_replicate(): + replicator = ContentMetadataReplicator( + enterprise_customer_uuid='378d5bf0-f67d-4bf7-8b2a-cbbc53d0f772', + content_identifier='course-v1:edX+DemoX+Demo_Course', + ) + return replicator.replicate() diff --git a/enterprise_subsidy/apps/content_metadata/migrations/0001_create_replicated_metadata_model.py b/enterprise_subsidy/apps/content_metadata/migrations/0001_create_replicated_metadata_model.py new file mode 100644 index 00000000..7c1c7f4d --- /dev/null +++ b/enterprise_subsidy/apps/content_metadata/migrations/0001_create_replicated_metadata_model.py @@ -0,0 +1,71 @@ +# Generated by Django 3.2.19 on 2023-09-15 14:24 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django.db.models.manager +import django_extensions.db.fields +import simple_history.models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='ReplicatedContentMetadata', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')), + ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')), + ('enterprise_customer_uuid', models.UUIDField(db_index=True, editable=False)), + ('content_key', models.CharField(db_index=True, editable=False, help_text='The globally unique content identifier for this course. Joinable with ContentMetadata.content_key in enterprise-catalog.', max_length=255)), + ('content_type', models.CharField(db_index=True, help_text='The type of content (e.g. course or courserun).', max_length=255)), + ('raw_metadata', models.JSONField(help_text='The raw JSON metadata fetched from the enterprise-catalog customer metadata API.')), + ('raw_fetched_at', models.DateTimeField(editable=False, help_text='Time at which raw_metadata was last fetched.')), + ('title', models.CharField(help_text='The title of the course', max_length=2047, null=True)), + ('price', models.BigIntegerField(help_text='Cost of this course run in USD Cents.')), + ('product_source', models.CharField(db_index=True, help_text='The product source for this course.', max_length=255, null=True)), + ('course_mode', models.CharField(db_index=True, help_text='The enrollment mode supported for this course.', max_length=255, null=True)), + ], + options={ + 'unique_together': {('enterprise_customer_uuid', 'content_key')}, + }, + managers=[ + ('recent_objects', django.db.models.manager.Manager()), + ], + ), + migrations.CreateModel( + name='HistoricalReplicatedContentMetadata', + fields=[ + ('id', models.BigIntegerField(auto_created=True, blank=True, db_index=True, verbose_name='ID')), + ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')), + ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')), + ('enterprise_customer_uuid', models.UUIDField(db_index=True, editable=False)), + ('content_key', models.CharField(db_index=True, editable=False, help_text='The globally unique content identifier for this course. Joinable with ContentMetadata.content_key in enterprise-catalog.', max_length=255)), + ('content_type', models.CharField(db_index=True, help_text='The type of content (e.g. course or courserun).', max_length=255)), + ('raw_metadata', models.JSONField(help_text='The raw JSON metadata fetched from the enterprise-catalog customer metadata API.')), + ('raw_fetched_at', models.DateTimeField(editable=False, help_text='Time at which raw_metadata was last fetched.')), + ('title', models.CharField(help_text='The title of the course', max_length=2047, null=True)), + ('price', models.BigIntegerField(help_text='Cost of this course run in USD Cents.')), + ('product_source', models.CharField(db_index=True, help_text='The product source for this course.', max_length=255, null=True)), + ('course_mode', models.CharField(db_index=True, help_text='The enrollment mode supported for this course.', max_length=255, null=True)), + ('history_id', models.AutoField(primary_key=True, serialize=False)), + ('history_date', models.DateTimeField()), + ('history_change_reason', models.CharField(max_length=100, null=True)), + ('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)), + ('history_user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'verbose_name': 'historical replicated content metadata', + 'ordering': ('-history_date', '-history_id'), + 'get_latest_by': 'history_date', + }, + bases=(simple_history.models.HistoricalChanges, models.Model), + ), + ] diff --git a/enterprise_subsidy/apps/content_metadata/models.py b/enterprise_subsidy/apps/content_metadata/models.py index 77d24e27..ef09854c 100644 --- a/enterprise_subsidy/apps/content_metadata/models.py +++ b/enterprise_subsidy/apps/content_metadata/models.py @@ -1,6 +1,105 @@ """ Models for the content_metadata app. """ -from django.db import models # pylint: disable=unused-import +from datetime import timedelta -# Create your models here. +from django.db import models +from django.utils import timezone +from django_extensions.db.models import TimeStampedModel +from simple_history.models import HistoricalRecords + + +FRESHNESS_THRESHOLD = timedelta(days=7) + + +class RecentlyModifiedManager(models.Manager): + def get_queryset(self): + threshold_point = timezone.now() - FRESHNESS_THRESHOLD + return super().get_queryset().filter( + modified__gte=threshold_point, + ) + + +class ReplicatedContentMetadata(TimeStampedModel): + """ + Let this thing have a BigAutoField PK - ``uuid`` would be too overloaded. + """ + class Meta: + # Let's have at most one record per (customer, content key) combination. + # side-note: modeling it like this means we're replicating not just + # content metadtata, but also catalog inclusion, which I think is a + # nice side effect to have going forward. + unique_together = [ + ('enterprise_customer_uuid', 'content_key'), + ] + + enterprise_customer_uuid = models.UUIDField( + null=False, + blank=False, + editable=False, + db_index=True, + ) + content_key = models.CharField( + max_length=255, + editable=False, + null=False, + blank=False, + db_index=True, + help_text=( + "The globally unique content identifier for this course. Joinable with " + "ContentMetadata.content_key in enterprise-catalog." + ), + ) + content_type = models.CharField( + max_length=255, + null=False, + db_index=True, + help_text="The type of content (e.g. course or courserun).", + ) + raw_metadata = models.JSONField( + null=False, + blank=False, + help_text="The raw JSON metadata fetched from the enterprise-catalog customer metadata API.", + ) + raw_fetched_at = models.DateTimeField( + null=False, + blank=False, + editable=False, + help_text="Time at which raw_metadata was last fetched.", + ) + title = models.CharField( + max_length=2047, + null=True, + help_text="The title of the course", + ) + price = models.BigIntegerField( + null=False, + blank=False, + help_text="Cost of this course run in USD Cents.", + ) + product_source = models.CharField( + max_length=255, + null=True, + db_index=True, + help_text="The product source for this course.", + ) + course_mode = models.CharField( + max_length=255, + null=True, + db_index=True, + help_text="The enrollment mode supported for this course.", + ) + history = HistoricalRecords() + + objects = models.Manager() + recent_objects = RecentlyModifiedManager() + + @classmethod + def get_recent_record_or_null(cls, enterprise_customer_uuid, content_key): + try: + return cls.recent_objects.get( + enterprise_customer_uuid=enterprise_customer_uuid, + content_key=content_key, + ) + except cls.DoesNotExist: + return None diff --git a/enterprise_subsidy/settings/base.py b/enterprise_subsidy/settings/base.py index 5c33e50d..8e32838b 100644 --- a/enterprise_subsidy/settings/base.py +++ b/enterprise_subsidy/settings/base.py @@ -149,7 +149,7 @@ def root(*path_fragments): } # New DB primary keys default to an IntegerField. -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' # Django Rest Framework REST_FRAMEWORK = {