Migrate update_assets and compile_locales to python scripts + add syn…

…c_host_files.py script to run tasks before starting containers

Dockerfile

@@ -155,7 +155,7 @@ COPY --chown=olympia:olympia locale ${LOCALE_DIR}
 RUN \
     --mount=type=bind,source=requirements/locale.txt,target=${HOME}/requirements/locale.txt \
     --mount=type=bind,source=Makefile-docker,target=${HOME}/Makefile-docker \
-    --mount=type=bind,source=locale/compile-mo.sh,target=${HOME}/compile-mo.sh \
+    --mount=type=bind,source=scripts/compile_locales.py,target=${HOME}/scripts/compile_locales.py \
     make -f Makefile-docker compile_locales
 
 # More efficient caching by mounting the exact files we need
@@ -175,10 +175,10 @@ COPY --chown=olympia:olympia static/ ${HOME}/static/
 RUN \
     --mount=type=bind,src=src,target=${HOME}/src \
     --mount=type=bind,src=Makefile-docker,target=${HOME}/Makefile-docker \
+    --mount=type=bind,src=scripts/update_assets.py,target=${HOME}/scripts/update_assets.py \
     --mount=type=bind,src=manage.py,target=${HOME}/manage.py \
 <<EOF
-echo "from olympia.lib.settings_base import *" > settings_local.py
-DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets
+make -f Makefile-docker update_assets
 EOF
 
 FROM base AS production

Makefile-docker

@@ -72,14 +72,8 @@ data_load:
 	./manage.py data_load $(ARGS)
 
 .PHONY: update_assets
-update_assets:
-	# Copy files required in compress_assets to the static folder
-	# If changing this here, make sure to adapt tests in amo/test_commands.py
-	$(PYTHON_COMMAND) manage.py compress_assets
-	$(PYTHON_COMMAND) manage.py generate_jsi18n_files
-	# Collect static files: This MUST be run last or files will be missing
-	$(PYTHON_COMMAND) manage.py collectstatic --noinput
-
+update_assets: ## Update the static assets
+	$(HOME)/scripts/update_assets.py
 
 .PHONY: update_deps
 update_deps: ## Update the dependencies
@@ -218,7 +212,7 @@ extract_locales: ## extracts and merges translation strings
 .PHONE: compile_locales
 compile_locales: ## compiles translation strings
 	$(PIP_COMMAND) install --progress-bar=off --no-deps -r requirements/locale.txt
-	./locale/compile-mo.sh ./locale/
+	$(HOME)/scripts/compile_locales.py
 
 .PHONY: help_submake
 help_submake:

Makefile-os

@@ -161,17 +161,17 @@ docker_clean_build_cache: ## Remove buildx build cache
 .PHONY: clean_docker
 clean_docker: docker_compose_down docker_mysqld_volume_remove docker_clean_images docker_clean_volumes docker_clean_build_cache ## Remove all docker resources taking space on the host machine
 
-.PHONY: docker_update_deps
-docker_update_deps: docker_mysqld_volume_create ## Update the dependencies in the container based on the docker tag and target
+.PHONY: docker_sync_host
+docker_sync_host: docker_mysqld_volume_create ## Update the dependencies in the container based on the docker tag and target
 	docker compose run \
 		--rm \
 		--no-deps \
 		$(DOCKER_RUN_ARGS) \
 		web \
-		make update_deps
+		./scripts/sync_host_files.py
 
 .PHONY: up_pre
-up_pre: setup docker_pull_or_build docker_update_deps ## Pre-up the environment, setup files, volumes and host state
+up_pre: setup docker_pull_or_build docker_sync_host ## Pre-up the environment, setup files, volumes and host state
 
 .PHONY: up_start
 up_start: docker_mysqld_volume_create ## Start the docker containers

scripts/compile_locales.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess
+from concurrent.futures import ThreadPoolExecutor
+
+
+def process_po_file(pofile, attempt=0):
+    """Process a single .po file, creating corresponding .mo file."""
+    print('processing', pofile)
+    directory = os.path.dirname(pofile)
+    stem = os.path.splitext(os.path.basename(pofile))[0]
+    mo_path = os.path.join(directory, f'{stem}.mo')
+
+    # Touch the .mo file
+    open(mo_path, 'a').close()
+
+    try:
+        # Run dennis-cmd lint
+        subprocess.run(
+            ['dennis-cmd', 'lint', '--errorsonly', pofile],
+            capture_output=True,
+            check=False,
+        )
+        # If lint passes, run msgfmt
+        subprocess.run(['msgfmt', '-o', mo_path, pofile], check=True)
+        return
+    except subprocess.CalledProcessError as e:
+        if attempt < 3:
+            print(f'Failed attempt {attempt} for {pofile}, retrying...')
+            return process_po_file(pofile, attempt=attempt + 1)
+        raise e
+
+
+def main():
+    # Ensure 'dennis' is installed
+    try:
+        import dennis as _
+    except ImportError:
+        print(
+            'Error: dennis is not installed. Please install it with pip install dennis'
+        )
+        exit(1)
+
+    locale_dir = os.path.abspath(
+        os.path.join(
+            os.path.dirname(__file__),
+            '..',
+            'locale',
+        )
+    )
+
+    print(f'Compiling locales in {locale_dir}')
+
+    # Collect all files first
+    django_files = []
+    djangojs_files = []
+    for root, _, files in os.walk(locale_dir):
+        for file in files:
+            if file == 'django.po':
+                django_files.append(os.path.join(root, file))
+            elif file == 'djangojs.po':
+                djangojs_files.append(os.path.join(root, file))
+
+    # Process django.po files in parallel
+    with ThreadPoolExecutor() as executor:
+        executor.map(process_po_file, django_files + djangojs_files)
+
+
+if __name__ == '__main__':
+    main()

scripts/sync_host_files.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import shutil
+import subprocess
+
+
+def main():
+    BUILD_INFO = os.environ.get('BUILD_INFO')
+
+    subprocess.run(['make', 'update_deps'], check=True)
+
+    HOME = os.environ.get('HOME')
+
+    for dir in ['static-build', 'site-static']:
+        path = os.path.join(HOME, dir)
+        if os.path.exists(path):
+            shutil.rmtree(path)
+        os.makedirs(path, exist_ok=True)
+
+    with open(BUILD_INFO, 'r') as f:
+        build_info = json.load(f)
+
+    if build_info.get('target') == 'production':
+        subprocess.run(['make', 'compile_locales'], check=True)
+        subprocess.run(['make', 'update_assets'], check=True)
+
+
+if __name__ == '__main__':
+    main()

scripts/update_assets.py

@@ -0,0 +1,30 @@
+import os
+import subprocess
+
+
+def main():
+    script_prefix = ['python3', 'manage.py']
+
+    environment = os.environ.copy()
+    # Always run in production mode without any development settings
+    environment['DJANGO_SETTINGS_MODULE'] = 'olympia.lib.settings_base'
+
+    subprocess.run(
+        script_prefix + ['compress_assets'],
+        check=True,
+        env=environment,
+    )
+    subprocess.run(
+        script_prefix + ['generate_jsi18n_files'],
+        check=True,
+        env=environment,
+    )
+    subprocess.run(
+        script_prefix + ['collectstatic', '--noinput'],
+        check=True,
+        env=environment,
+    )
+
+
+if __name__ == '__main__':
+    main()