From 80f94368b19b479706436a07b28a22f046068eb0 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 4 Jan 2025 10:49:49 +1000 Subject: [PATCH 01/19] Run manage.py migrate in labs-engine cli --- app/app/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/app/cli.py b/app/app/cli.py index 703bd98..ad32396 100644 --- a/app/app/cli.py +++ b/app/app/cli.py @@ -27,6 +27,7 @@ def main(): sys.path.insert(0, str(BASE_DIR)) if len(sys.argv) > 2: os.environ["LAB_CONTENT_ENTRYPOINT"] = sys.argv[2] + execute_from_command_line(["manage.py", "migrate"]) execute_from_command_line(["manage.py", "runserver"]) else: print(f"Unknown command: {sys.argv[1]}") From c7d0064fb3484adcb9c1fc82a5b0eff8ed6435aa Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 4 Jan 2025 10:56:20 +1000 Subject: [PATCH 02/19] Use in-mem database for cli server --- app/app/settings/cli.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/app/settings/cli.py b/app/app/settings/cli.py index 4a0f10a..2e49c89 100644 --- a/app/app/settings/cli.py +++ b/app/app/settings/cli.py @@ -30,3 +30,10 @@ f"http://{HOSTNAME}/static/local/{LAB_CONTENT_ENTRYPOINT}") INSTALLED_APPS.remove('django_light') + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': ':memory:', + } +} From c0ffaffeabeda41a0bb81025cf99af4572a6aa98 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 4 Jan 2025 11:04:41 +1000 Subject: [PATCH 03/19] Refresh db connections before runserver --- app/app/cli.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/app/cli.py b/app/app/cli.py index ad32396..f7ba95d 100644 --- a/app/app/cli.py +++ b/app/app/cli.py @@ -12,6 +12,8 @@ def main(): """CLI entry point for running the development server.""" try: from django.core.management import execute_from_command_line + from django.db import connections + from django.apps import apps except ImportError as exc: raise ImportError( "Couldn't import Django. Make sure it's installed and " @@ -28,6 +30,9 @@ def main(): if len(sys.argv) > 2: os.environ["LAB_CONTENT_ENTRYPOINT"] = sys.argv[2] execute_from_command_line(["manage.py", "migrate"]) + for conn in connections.all(): + conn.close() + apps.clear_cache() execute_from_command_line(["manage.py", "runserver"]) else: print(f"Unknown command: {sys.argv[1]}") From e268a3f6ab9ab5afe848204cb9e79bd191ce54f6 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 4 Jan 2025 11:12:53 +1000 Subject: [PATCH 04/19] CLI runs custom runserver_migrate command --- app/app/cli.py | 8 +------- app/app/management/commands/__init__.py | 0 app/app/management/commands/runserver_migrate.py | 12 ++++++++++++ 3 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 app/app/management/commands/__init__.py create mode 100644 app/app/management/commands/runserver_migrate.py diff --git a/app/app/cli.py b/app/app/cli.py index f7ba95d..cb19802 100644 --- a/app/app/cli.py +++ b/app/app/cli.py @@ -12,8 +12,6 @@ def main(): """CLI entry point for running the development server.""" try: from django.core.management import execute_from_command_line - from django.db import connections - from django.apps import apps except ImportError as exc: raise ImportError( "Couldn't import Django. Make sure it's installed and " @@ -29,11 +27,7 @@ def main(): sys.path.insert(0, str(BASE_DIR)) if len(sys.argv) > 2: os.environ["LAB_CONTENT_ENTRYPOINT"] = sys.argv[2] - execute_from_command_line(["manage.py", "migrate"]) - for conn in connections.all(): - conn.close() - apps.clear_cache() - execute_from_command_line(["manage.py", "runserver"]) + execute_from_command_line(["manage.py", "runserver_migrate"]) else: print(f"Unknown command: {sys.argv[1]}") diff --git a/app/app/management/commands/__init__.py b/app/app/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/app/management/commands/runserver_migrate.py b/app/app/management/commands/runserver_migrate.py new file mode 100644 index 0000000..4bfa7ad --- /dev/null +++ b/app/app/management/commands/runserver_migrate.py @@ -0,0 +1,12 @@ +from django.core.management.commands.runserver import ( + Command as RunserverCommand, +) +from django.core.management import call_command + + +class Command(RunserverCommand): + def handle(self, *args, **options): + self.stdout.write("Applying migrations...") + call_command("migrate") + self.stdout.write("Migrations applied successfully.") + super().handle(*args, **options) From 8179ad8af53c7382c0291f0fd43e4b5bcc8f84db Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 4 Jan 2025 11:15:00 +1000 Subject: [PATCH 05/19] Fix management command folder --- app/{app => labs}/management/commands/__init__.py | 0 app/{app => labs}/management/commands/runserver_migrate.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename app/{app => labs}/management/commands/__init__.py (100%) rename app/{app => labs}/management/commands/runserver_migrate.py (100%) diff --git a/app/app/management/commands/__init__.py b/app/labs/management/commands/__init__.py similarity index 100% rename from app/app/management/commands/__init__.py rename to app/labs/management/commands/__init__.py diff --git a/app/app/management/commands/runserver_migrate.py b/app/labs/management/commands/runserver_migrate.py similarity index 100% rename from app/app/management/commands/runserver_migrate.py rename to app/labs/management/commands/runserver_migrate.py From a09fbf281b4e88357338b2e661a621d78aaace82 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 4 Jan 2025 11:20:15 +1000 Subject: [PATCH 06/19] Fix management commands folder --- app/labs/management/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 app/labs/management/__init__.py diff --git a/app/labs/management/__init__.py b/app/labs/management/__init__.py new file mode 100644 index 0000000..e69de29 From 63bb082a94c5af16221d94b3927ea0bb3a50de6e Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 4 Jan 2025 11:28:20 +1000 Subject: [PATCH 07/19] Add MANIFEST.in for pip install --- MANIFEST.in | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..1cb64ea --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +recursive-include app * +recursive-include labs * +recursive-include utils * From 935e37a06189b6c5b396b7ff6a3a452e4bcb446f Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 4 Jan 2025 13:13:35 +1000 Subject: [PATCH 08/19] Skip cache in CLI_DEV mode --- app/labs/cache.py | 7 ++++++- app/labs/views.py | 6 ++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/app/labs/cache.py b/app/labs/cache.py index 99dc4a4..f304434 100644 --- a/app/labs/cache.py +++ b/app/labs/cache.py @@ -17,7 +17,10 @@ class LabCache: @classmethod def get(cls, request): - if request.GET.get('cache', '').lower() == 'false': + if ( + request.GET.get('cache', '').lower() == 'false' + or settings.CLI_DEV + ): return cache_key = cls._generate_cache_key(request) @@ -63,6 +66,8 @@ class WebCache: @classmethod def get(cls, url): + if settings.CLI_DEV: + return cache_key = cls._generate_cache_key(url) data = cache.get(cache_key) if data: diff --git a/app/labs/views.py b/app/labs/views.py index 71af77b..2b27ac6 100644 --- a/app/labs/views.py +++ b/app/labs/views.py @@ -30,10 +30,8 @@ def export_lab(request): an ad hoc basis, where the content would typically be hosted in a GitHub repo with a YAML file root which is specified as a GET parameter. """ - - if not settings.CLI_DEV: - if response := LabCache.get(request): - return response + if response := LabCache.get(request): + return response template = 'labs/exported.html' From 90e48f18c87993862de47a12b2f8de5bfebbdda0 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 4 Jan 2025 13:15:30 +1000 Subject: [PATCH 09/19] CLI use sqlite db --- app/app/settings/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/app/settings/cli.py b/app/app/settings/cli.py index 2e49c89..f2f7a2c 100644 --- a/app/app/settings/cli.py +++ b/app/app/settings/cli.py @@ -34,6 +34,6 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': ':memory:', + 'NAME': BASE_DIR / 'db.sqlite3', } } From fb13d938eaf26c6576eb1ce76f677d781b7d0d7f Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 4 Jan 2025 13:21:23 +1000 Subject: [PATCH 10/19] Collectstatic before runserver --- app/labs/management/commands/runserver_migrate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/labs/management/commands/runserver_migrate.py b/app/labs/management/commands/runserver_migrate.py index 4bfa7ad..1481e88 100644 --- a/app/labs/management/commands/runserver_migrate.py +++ b/app/labs/management/commands/runserver_migrate.py @@ -9,4 +9,5 @@ def handle(self, *args, **options): self.stdout.write("Applying migrations...") call_command("migrate") self.stdout.write("Migrations applied successfully.") + call_command("collectstatic", "--noinput") super().handle(*args, **options) From fc305108849c0512ce201d3591972bc064e98204 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 4 Jan 2025 13:25:31 +1000 Subject: [PATCH 11/19] Explicitly add static to urlpatterns --- app/app/urls.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/app/urls.py b/app/app/urls.py index 7174ef5..e3705d2 100644 --- a/app/app/urls.py +++ b/app/app/urls.py @@ -15,6 +15,8 @@ 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ from django.contrib import admin +from django.conf import settings +from django.conf.urls.static import static from django.urls import path, include urlpatterns = [ @@ -22,4 +24,10 @@ path('', include('labs.urls')), ] +if settings.CLI_DEV: + urlpatterns += static( + settings.STATIC_URL, + document_root=settings.STATIC_ROOT, + ) + handler400 = 'labs.views.custom_400' From edf7c31254cb319978f4e84a31e0b2b7c430279c Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 4 Jan 2025 13:29:50 +1000 Subject: [PATCH 12/19] Skip all cache methods when CLI dev --- app/labs/cache.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/labs/cache.py b/app/labs/cache.py index f304434..10d5393 100644 --- a/app/labs/cache.py +++ b/app/labs/cache.py @@ -36,12 +36,15 @@ def get(cls, request): @classmethod def put(cls, request, body): + if settings.CLI_DEV: + return HttpResponse(body) logger.debug( f"Cache PUT for {request.GET.get('content_root', 'root')}") cache_key = cls._generate_cache_key(request) timeout = (settings.CACHE_TIMEOUT if request.GET.get('content_root') else None) # No timeout for default "Docs Lab" page + cache.set(cache_key, body, timeout=timeout) response = HttpResponse(body) response['X-Cache-Status'] = 'MISS' @@ -75,6 +78,8 @@ def get(cls, url): @classmethod def put(cls, url, data, timeout=3600): + if settings.CLI_DEV: + return cache_key = cls._generate_cache_key(url) cache.set(cache_key, data, timeout=timeout) From 5eca8cde930a3e6fe8022aef00675ac43b3a7b6c Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 4 Jan 2025 13:46:15 +1000 Subject: [PATCH 13/19] runserver_quit manage.py command to ignore migration warnings --- app/app/cli.py | 3 ++- app/app/settings/cli.py | 2 +- .../commands/{runserver_migrate.py => runserver_quiet.py} | 8 +++++--- 3 files changed, 8 insertions(+), 5 deletions(-) rename app/labs/management/commands/{runserver_migrate.py => runserver_quiet.py} (67%) diff --git a/app/app/cli.py b/app/app/cli.py index cb19802..9de5e36 100644 --- a/app/app/cli.py +++ b/app/app/cli.py @@ -27,7 +27,8 @@ def main(): sys.path.insert(0, str(BASE_DIR)) if len(sys.argv) > 2: os.environ["LAB_CONTENT_ENTRYPOINT"] = sys.argv[2] - execute_from_command_line(["manage.py", "runserver_migrate"]) + execute_from_command_line(["manage.py", "collectstatic", "--noinput"]) + execute_from_command_line(["manage.py", "runserver_quiet"]) else: print(f"Unknown command: {sys.argv[1]}") diff --git a/app/app/settings/cli.py b/app/app/settings/cli.py index f2f7a2c..2e49c89 100644 --- a/app/app/settings/cli.py +++ b/app/app/settings/cli.py @@ -34,6 +34,6 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': BASE_DIR / 'db.sqlite3', + 'NAME': ':memory:', } } diff --git a/app/labs/management/commands/runserver_migrate.py b/app/labs/management/commands/runserver_quiet.py similarity index 67% rename from app/labs/management/commands/runserver_migrate.py rename to app/labs/management/commands/runserver_quiet.py index 1481e88..0009e92 100644 --- a/app/labs/management/commands/runserver_migrate.py +++ b/app/labs/management/commands/runserver_quiet.py @@ -1,3 +1,4 @@ +import warnings from django.core.management.commands.runserver import ( Command as RunserverCommand, ) @@ -6,8 +7,9 @@ class Command(RunserverCommand): def handle(self, *args, **options): - self.stdout.write("Applying migrations...") - call_command("migrate") - self.stdout.write("Migrations applied successfully.") + warnings.filterwarnings( + "ignore", + message="apply the migration", + ) call_command("collectstatic", "--noinput") super().handle(*args, **options) From 5826ea2db9f0e7b12acfdbf7af190248755d86bc Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 4 Jan 2025 13:52:38 +1000 Subject: [PATCH 14/19] Replace management command with DisableMigrations MIGRATION_MODULES --- app/app/settings/cli.py | 9 +++++++++ app/labs/management/__init__.py | 0 app/labs/management/commands/__init__.py | 0 app/labs/management/commands/runserver_quiet.py | 15 --------------- 4 files changed, 9 insertions(+), 15 deletions(-) delete mode 100644 app/labs/management/__init__.py delete mode 100644 app/labs/management/commands/__init__.py delete mode 100644 app/labs/management/commands/runserver_quiet.py diff --git a/app/app/settings/cli.py b/app/app/settings/cli.py index 2e49c89..ebeb4c7 100644 --- a/app/app/settings/cli.py +++ b/app/app/settings/cli.py @@ -37,3 +37,12 @@ 'NAME': ':memory:', } } + +class DisableMigrations(object): + def __contains__(self, _): + return True + + def __getitem__(self, _): + return "notmigrations" + +MIGRATION_MODULES = DisableMigrations() diff --git a/app/labs/management/__init__.py b/app/labs/management/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/app/labs/management/commands/__init__.py b/app/labs/management/commands/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/app/labs/management/commands/runserver_quiet.py b/app/labs/management/commands/runserver_quiet.py deleted file mode 100644 index 0009e92..0000000 --- a/app/labs/management/commands/runserver_quiet.py +++ /dev/null @@ -1,15 +0,0 @@ -import warnings -from django.core.management.commands.runserver import ( - Command as RunserverCommand, -) -from django.core.management import call_command - - -class Command(RunserverCommand): - def handle(self, *args, **options): - warnings.filterwarnings( - "ignore", - message="apply the migration", - ) - call_command("collectstatic", "--noinput") - super().handle(*args, **options) From fb287e6f023bd700bc6cb001f41ed072e4344268 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 4 Jan 2025 13:54:01 +1000 Subject: [PATCH 15/19] Fix CLI runserver command --- app/app/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/app/cli.py b/app/app/cli.py index 9de5e36..3894604 100644 --- a/app/app/cli.py +++ b/app/app/cli.py @@ -28,7 +28,7 @@ def main(): if len(sys.argv) > 2: os.environ["LAB_CONTENT_ENTRYPOINT"] = sys.argv[2] execute_from_command_line(["manage.py", "collectstatic", "--noinput"]) - execute_from_command_line(["manage.py", "runserver_quiet"]) + execute_from_command_line(["manage.py", "runserver"]) else: print(f"Unknown command: {sys.argv[1]}") From b3a465e23507953f87e7ea0ad6e40d6f849e398f Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 4 Jan 2025 13:59:38 +1000 Subject: [PATCH 16/19] Disable django.db.backends.schema logging --- app/app/settings/cli.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/app/app/settings/cli.py b/app/app/settings/cli.py index ebeb4c7..3d1108a 100644 --- a/app/app/settings/cli.py +++ b/app/app/settings/cli.py @@ -38,11 +38,24 @@ } } -class DisableMigrations(object): - def __contains__(self, _): - return True - - def __getitem__(self, _): - return "notmigrations" - -MIGRATION_MODULES = DisableMigrations() +LOGGING = { + 'version': 1, + 'handlers': { + 'devnull': { + 'level': 'DEBUG', + 'class': 'logging.FileHandler', + 'filename': os.devnull, + }, + }, + 'loggers': { + 'django.db.backends.schema': { + 'handlers': ['devnull'], + 'propagate': True, + 'level': 'INFO', + }, + '': { + 'handlers': ['file'], + 'level': 'DEBUG', + } + } +} From 774fc6d2dfd6d790c39135c18d9c9fb96d985319 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 4 Jan 2025 14:01:30 +1000 Subject: [PATCH 17/19] Fix LOGGING --- app/app/settings/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/app/settings/cli.py b/app/app/settings/cli.py index 3d1108a..a0967f2 100644 --- a/app/app/settings/cli.py +++ b/app/app/settings/cli.py @@ -54,7 +54,7 @@ 'level': 'INFO', }, '': { - 'handlers': ['file'], + 'handlers': ['devnull'], 'level': 'DEBUG', } } From ab163305bb08619052c6654f25f83130358aef0c Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 4 Jan 2025 14:05:54 +1000 Subject: [PATCH 18/19] Try disable migrations with MIGRATION_MODULES --- app/app/settings/cli.py | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/app/app/settings/cli.py b/app/app/settings/cli.py index a0967f2..2cd7a05 100644 --- a/app/app/settings/cli.py +++ b/app/app/settings/cli.py @@ -38,24 +38,11 @@ } } -LOGGING = { - 'version': 1, - 'handlers': { - 'devnull': { - 'level': 'DEBUG', - 'class': 'logging.FileHandler', - 'filename': os.devnull, - }, - }, - 'loggers': { - 'django.db.backends.schema': { - 'handlers': ['devnull'], - 'propagate': True, - 'level': 'INFO', - }, - '': { - 'handlers': ['devnull'], - 'level': 'DEBUG', - } - } -} +class DisableMigrations: + def __contains__(self, item): + return True + + def __getitem__(self, item): + return None + +MIGRATION_MODULES = DisableMigrations() From 4f79290ae0539edccfb3290690d457de4ee7d1e9 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 4 Jan 2025 14:18:53 +1000 Subject: [PATCH 19/19] Fix html_tags validate function --- app/labs/lab_schema.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/labs/lab_schema.py b/app/labs/lab_schema.py index 1de43ae..f0e932f 100644 --- a/app/labs/lab_schema.py +++ b/app/labs/lab_schema.py @@ -131,14 +131,16 @@ def html_tags(value: str) -> str: if "<" not in value: return value # Remove self closing tags - value = ( + value_stripped = ( re.sub(r'(<.*?/>)|()', '', value, flags=re.MULTILINE) .replace('
', '') .replace('
', '') ) # Enumerate open/close tags - open_tags = re.findall(r'<[^/][\s\S]*?>', value, flags=re.MULTILINE) - close_tags = re.findall(r'', value, flags=re.MULTILINE) + open_tags = re.findall( + r'<[^/][\s\S]*?>', value_stripped, flags=re.MULTILINE) + close_tags = re.findall( + r'', value_stripped, flags=re.MULTILINE) assert len(open_tags) == len(close_tags), ( f'Unclosed HTML tag in section content:\n{value}') return value