From c751a90a61c673a7bbea4054409e6c512fb5a3a6 Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Sat, 8 May 2021 18:36:51 +0300 Subject: [PATCH 1/3] RED-1961 Disable the upstream prefix override to refresh cache on every deploy and use the random prefix Related to https://github.com/appsembler/configuration/pull/348 --- cms/envs/production.py | 7 +++++-- lms/envs/production.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/cms/envs/production.py b/cms/envs/production.py index 542db6fa4d7a..24762fb6789b 100644 --- a/cms/envs/production.py +++ b/cms/envs/production.py @@ -209,8 +209,11 @@ def get_env_setting(setting): 'LOCATION': 'edx_location_mem_cache', } -if 'staticfiles' in CACHES: - CACHES['staticfiles']['KEY_PREFIX'] = EDX_PLATFORM_REVISION +# Tahoe: RED-1961 Disable the upstream prefix override to refresh cache on every deploy and use the random prefix: +# https://github.com/appsembler/configuration/pull/348 +# +# if 'staticfiles' in CACHES: +# CACHES['staticfiles']['KEY_PREFIX'] = EDX_PLATFORM_REVISION # In order to transition from local disk asset storage to S3 backed asset storage, # we need to run asset collection twice, once for local disk and once for S3. diff --git a/lms/envs/production.py b/lms/envs/production.py index a98a5764c55c..2961092d3a19 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -233,8 +233,11 @@ def get_env_setting(setting): 'LOCATION': 'edx_location_mem_cache', } -if 'staticfiles' in CACHES: - CACHES['staticfiles']['KEY_PREFIX'] = EDX_PLATFORM_REVISION +# Tahoe: RED-1961 Disable the upstream prefix override to refresh cache on every deploy and use the random prefix: +# https://github.com/appsembler/configuration/pull/348 +# +# if 'staticfiles' in CACHES: +# CACHES['staticfiles']['KEY_PREFIX'] = EDX_PLATFORM_REVISION # In order to transition from local disk asset storage to S3 backed asset storage, # we need to run asset collection twice, once for local disk and once for S3. From c5164f65855dedc244d75f5a25041bec641c6e07 Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Tue, 11 May 2021 19:35:24 +0300 Subject: [PATCH 2/3] honeycomb tracing for static files and mako rendering this mostly to debug slowness in static files look up as reported in RED-1961. --- common/djangoapps/edxmako/shortcuts.py | 4 ++++ .../pipeline_mako/templates/static_content.html | 14 ++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/common/djangoapps/edxmako/shortcuts.py b/common/djangoapps/edxmako/shortcuts.py index 6c1160ae4c68..e3142313d8d0 100644 --- a/common/djangoapps/edxmako/shortcuts.py +++ b/common/djangoapps/edxmako/shortcuts.py @@ -15,6 +15,7 @@ import logging +import beeline import six from django.conf import settings from django.http import HttpResponse @@ -149,6 +150,7 @@ def marketing_link_context_processor(request): ) +@beeline.traced(name='common.djangoapps.edxmako.shortcuts.render_to_string') def render_to_string(template_name, dictionary, namespace='main', request=None): """ Render a Mako template to as a string. @@ -169,6 +171,8 @@ def render_to_string(template_name, dictionary, namespace='main', request=None): request: The request to use to construct the RequestContext for rendering this template. If not supplied, the current request will be used. """ + beeline.add_context_field('render_to_string.template_name', template_name) + beeline.add_context(dictionary) if namespace == 'lms.main': engine = engines[Engines.PREVIEW] else: diff --git a/common/djangoapps/pipeline_mako/templates/static_content.html b/common/djangoapps/pipeline_mako/templates/static_content.html index d80e3336d13d..160ce85dab9b 100644 --- a/common/djangoapps/pipeline_mako/templates/static_content.html +++ b/common/djangoapps/pipeline_mako/templates/static_content.html @@ -2,6 +2,7 @@ <%! import logging import json +import beeline from django.contrib.staticfiles.storage import staticfiles_storage from pipeline_mako import compressed_css, compressed_js from pipeline_mako.helpers.studiofrontend import load_sfe_i18n_messages @@ -31,10 +32,15 @@ %> <%def name='url(file, raw=False)'><% -try: - url = staticfiles_storage.url(file) -except: - url = file +with beeline.tracer(name='static_content.html.url'): + beeline.add_context_field('url.file', file) + try: + url = staticfiles_storage.url(file) + beeline.add_context_field('url.error', False) + except: + beeline.add_context_field('url.error', True) + url = file + beeline.add_context_field('url.url', url) ## HTML-escaping must be handled by caller %>${url | n, decode.utf8}${"?raw" if raw else ""} From 9e1e8d07adec6de3c7e9532f829b940f0b3612da Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Tue, 11 May 2021 19:36:10 +0300 Subject: [PATCH 3/3] RED-1961: Enforce static file look up for ProductionS3Storage backend settings.CACHES['staticfiles'] wasn't getting hit when ProductionS3Storage is used in HTTP request context. This change enforces cache by overriding the ProductionS3Storage.url method. --- openedx/core/storage.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/openedx/core/storage.py b/openedx/core/storage.py index fefad2ee4a31..660b10455ad9 100644 --- a/openedx/core/storage.py +++ b/openedx/core/storage.py @@ -2,10 +2,12 @@ Django storage backends for Open edX. """ +import beeline from django.conf import settings from django.contrib.staticfiles.storage import StaticFilesStorage from django.core.files.storage import get_storage_class, FileSystemStorage +from django.core.cache import caches from django.utils.deconstruct import deconstructible from django.utils.lru_cache import lru_cache from pipeline.storage import NonPackagingMixin @@ -62,7 +64,31 @@ class ProductionStorage(ProductionMixin, StaticFilesStorage): class ProductionS3Storage(ProductionMixin, S3Boto3Storage): # pylint: disable=abstract-method - pass + + @beeline.traced('ProductionS3Storage.url') + def url(self, name, force=False): + """ + Return the non-hashed URL in DEBUG mode with cache support. + + Tahoe: RED-1961 This method is created for Tahoe to address mysteriously missing cache. + """ + beeline.add_context_field('ProductionS3Storage.file_name', name) + beeline.add_context_field('ProductionS3Storage.force', force) + + static_files_cache = caches['staticfiles'] + cache_entry_name = 'ProductionS3Storage.staticfiles_cache.{}'.format(name) + beeline.add_context_field('ProductionS3Storage.staticfiles_cache', cache_entry_name) + + url = static_files_cache.get(cache_entry_name) + if not url: + beeline.add_context_field('ProductionS3Storage.cached', False) + url = super().url(name, force) + static_files_cache.set(cache_entry_name, url) + else: + beeline.add_context_field('ProductionS3Storage.cached', True) + + beeline.add_context_field('ProductionS3Storage.hashed_url', url) + return url class DevelopmentStorage(