From 531211d355c5883a5fe947548593783b5c4a7e6f Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Thu, 9 Jan 2025 23:50:55 +0000 Subject: [PATCH] temp: Enable Celery distributed tracing in stage edxapp Datadog Support suggested that we try enabling this to see if it changes anything about the orphaned spans on the edxapp workers (unexpected top-level spans that are not `operation_name:celery.run`, but instead are other spans that have lost their parent association). Just going to enable this on stage so we don't mess with traces in prod, for now. See https://github.com/edx/edx-arch-experiments/issues/822 --- .../roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 | 10 +++++++++- .../roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 | 7 +++++++ .../roles/edxapp/templates/edx/app/edxapp/worker.sh.j2 | 8 ++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 b/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 index 76d81677697..5a1728fc5be 100644 --- a/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 +++ b/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 @@ -21,6 +21,14 @@ export DD_TRACE_LOG_STREAM_HANDLER=false # Suppress middleware spans because there are about 100 for each request. # Remove (or set to true) for debugging. export DD_DJANGO_INSTRUMENT_MIDDLEWARE=false + +{% if COMMON_ENVIRONMENT == "stage" %} +# Temporary 2025-01-09: Enable Celery distributed tracing to see if it +# has an effect on the remaining orphaned spans. +# https://github.com/edx/edx-arch-experiments/issues/822 +export DD_CELERY_DISTRIBUTED_TRACING=true +{% endif %} + {% endif -%} # We want to be able to toggle this on separately from DD in general. @@ -54,4 +62,4 @@ source {{ edxapp_app_dir }}/edxapp_env exec {{ executable }} -c {{ edxapp_app_dir }}/cms_gunicorn.py {{ EDXAPP_CMS_GUNICORN_EXTRA }} cms.wsgi --log-file {{ edxapp_gunicorn_log_dir[1] }}/edx.log {% else %} exec {{ executable }} -c {{ edxapp_app_dir }}/cms_gunicorn.py {{ EDXAPP_CMS_GUNICORN_EXTRA }} cms.wsgi -{% endif %} \ No newline at end of file +{% endif %} diff --git a/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 b/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 index d439c30eda6..90ab215988b 100644 --- a/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 +++ b/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 @@ -22,6 +22,13 @@ export DD_TRACE_LOG_STREAM_HANDLER=false # Remove (or set to true) for debugging. export DD_DJANGO_INSTRUMENT_MIDDLEWARE=false +{% if COMMON_ENVIRONMENT == "stage" %} +# Temporary 2025-01-09: Enable Celery distributed tracing to see if it +# has an effect on the remaining orphaned spans. +# https://github.com/edx/edx-arch-experiments/issues/822 +export DD_CELERY_DISTRIBUTED_TRACING=true +{% endif %} + {% endif -%} # We want to be able to toggle this on separately from DD in general. diff --git a/playbooks/roles/edxapp/templates/edx/app/edxapp/worker.sh.j2 b/playbooks/roles/edxapp/templates/edx/app/edxapp/worker.sh.j2 index 457c0120445..954735a1b8a 100644 --- a/playbooks/roles/edxapp/templates/edx/app/edxapp/worker.sh.j2 +++ b/playbooks/roles/edxapp/templates/edx/app/edxapp/worker.sh.j2 @@ -21,6 +21,14 @@ export DD_TRACE_LOG_STREAM_HANDLER=false # Suppress middleware spans because there are about 100 for each request. # Remove (or set to true) for debugging. export DD_DJANGO_INSTRUMENT_MIDDLEWARE=false + +{% if COMMON_ENVIRONMENT == "stage" %} +# Temporary 2025-01-09: Enable Celery distributed tracing to see if it +# has an effect on the remaining orphaned spans. +# https://github.com/edx/edx-arch-experiments/issues/822 +export DD_CELERY_DISTRIBUTED_TRACING=true +{% endif %} + {% endif -%} # We want to be able to toggle this on separately from DD in general.