diff --git a/.env b/.env index e372c63a..03c20d76 100644 --- a/.env +++ b/.env @@ -45,6 +45,7 @@ VPN_CLIENT_NAME=default-management-vpn # Developer DEBUG_MODE=False DJANGO_LOG_LEVEL=INFO +# Celery workers USE_OPENWISP_CELERY_TASK_ROUTES_DEFAULTS=True OPENWISP_CELERY_COMMAND_FLAGS=--concurrency=1 USE_OPENWISP_CELERY_NETWORK=True @@ -54,3 +55,5 @@ OPENWISP_CELERY_MONITORING_COMMAND_FLAGS=--concurrency=1 OPENWISP_CELERY_MONITORING_CHECKS_COMMAND_FLAGS=--concurrency=1 USE_OPENWISP_CELERY_FIRMWARE=True OPENWISP_CELERY_FIRMWARE_COMMAND_FLAGS=--concurrency=1 +# Metric collection +METRIC_COLLECTION=True diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml index 3210b1b6..53024e15 100644 --- a/.github/workflows/branch.yml +++ b/.github/workflows/branch.yml @@ -25,6 +25,8 @@ jobs: - name: Setup run: | echo "127.0.0.1 dashboard.openwisp.org api.openwisp.org" | sudo tee -a /etc/hosts + # disable metric collection during builds + sed -i 's/METRIC_COLLECTION=True/METRIC_COLLECTION=False/' .env # the following action is equivalent to # echo "$DOCKER_HUB_SECRET" | docker login --username "$DOCKER_HUB_USERNAME" --password-stdin diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 298ded66..37642bdb 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -28,6 +28,8 @@ jobs: - name: Setup run: | echo "127.0.0.1 dashboard.openwisp.org api.openwisp.org" | sudo tee -a /etc/hosts + # disable metric collection during builds + sed -i 's/METRIC_COLLECTION=True/METRIC_COLLECTION=False/' .env - name: Build Images run: make compose-build nfs-build diff --git a/docs/ENV.md b/docs/ENV.md index 6e6b7f41..84b53223 100644 --- a/docs/ENV.md +++ b/docs/ENV.md @@ -165,6 +165,12 @@ Any OpenWISP Configuration of type `string`. `int`, `bool` or `json` is supporte - **Default:** None - **Example:** [{"name": "Name1","label": "Label1","boards": ["TestA", "TestB"]}, {"name": "Name2","label": "Label2","boards": ["TestC", "TestD"]}] +### `METRIC_COLLECTION` + +- **Explanation:** Whether the usage metric collection feature of openwisp-utils is enabled or not. +- **Valid Values:** True | False +- **Default:** True + ## Database ### `DB_NAME` diff --git a/images/common/openwisp/__init__.py b/images/common/openwisp/__init__.py index 76be18a5..aa039cd0 100644 --- a/images/common/openwisp/__init__.py +++ b/images/common/openwisp/__init__.py @@ -3,3 +3,5 @@ from .celery import app as celery_app __all__ = ['celery_app'] +__openwisp_version__ = '24.0.0a' +__openwisp_installation_method__ = 'docker-openwisp' diff --git a/images/common/openwisp/celery.py b/images/common/openwisp/celery.py index e207895c..31c35cf8 100644 --- a/images/common/openwisp/celery.py +++ b/images/common/openwisp/celery.py @@ -7,7 +7,12 @@ os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'openwisp.settings') -radius_schedule, topology_schedule, monitoring_schedule = {}, {}, {} +radius_schedule, topology_schedule, monitoring_schedule, metric_collection_schedule = ( + {}, + {}, + {}, + {}, +) task_routes = {} if env_bool(os.environ.get('USE_OPENWISP_CELERY_NETWORK')): @@ -59,6 +64,14 @@ }, } +if env_bool(os.environ.get('METRIC_COLLECTION', 'True')): + metric_collection_schedule = { + 'send_usage_metrics': { + 'task': 'openwisp_utils.metric_collection.tasks.send_usage_metrics', + 'schedule': timedelta(days=1), + }, + } + notification_schedule = { 'notification-delete-tasks': { 'task': 'openwisp_notifications.tasks.delete_old_notifications', @@ -79,6 +92,7 @@ **topology_schedule, **notification_schedule, **monitoring_schedule, + **metric_collection_schedule, }, ) app.config_from_object('django.conf:settings', namespace='CELERY') diff --git a/images/common/openwisp/settings.py b/images/common/openwisp/settings.py index 2f46d463..6c961b26 100644 --- a/images/common/openwisp/settings.py +++ b/images/common/openwisp/settings.py @@ -357,6 +357,8 @@ }, } +TEST_RUNNER = 'openwisp_utils.metric_collection.tests.runner.MockRequestPostRunner' + # Add Custom OpenWRT Images for openwisp firmware try: OPENWRT_IMAGES = json.loads(os.environ['OPENWISP_CUSTOM_OPENWRT_IMAGES']) @@ -408,6 +410,8 @@ INSTALLED_APPS.remove('openwisp_monitoring.device') if 'openwisp_monitoring.check' in INSTALLED_APPS: INSTALLED_APPS.remove('openwisp_monitoring.check') +if not env_bool(os.environ.get('METRIC_COLLECTION', 'True')): + INSTALLED_APPS.remove('openwisp_utils.metric_collection') if EMAIL_BACKEND == 'djcelery_email.backends.CeleryEmailBackend': INSTALLED_APPS.append('djcelery_email') diff --git a/images/openwisp_api/module_settings.py b/images/openwisp_api/module_settings.py index 88700af8..11958eea 100644 --- a/images/openwisp_api/module_settings.py +++ b/images/openwisp_api/module_settings.py @@ -8,11 +8,13 @@ 'django.contrib.humanize', # all-auth 'django.contrib.sites', + # overrides allauth templates + # must precede allauth + 'openwisp_users.accounts', 'allauth', 'allauth.account', 'allauth.socialaccount', 'django_extensions', - 'corsheaders', # openwisp modules 'openwisp_users', # openwisp-controller @@ -35,11 +37,12 @@ # openwisp-firmware-upgrader 'openwisp_firmware_upgrader', # openwisp radius + 'dj_rest_auth', + 'dj_rest_auth.registration', 'openwisp_radius', - # Admin theme required for the templates - # stored in openwisp_utils/admin_theme - 'openwisp_utils.admin_theme', # admin + 'openwisp_utils.admin_theme', + 'openwisp_utils.metric_collection', 'django.contrib.admin', 'django.forms', # other dependencies @@ -49,18 +52,20 @@ # rest framework 'rest_framework', 'rest_framework_gis', - 'django_filters', - # registration 'rest_framework.authtoken', - 'dj_rest_auth', - 'dj_rest_auth.registration', + 'django_filters', # social login 'allauth.socialaccount.providers.facebook', 'allauth.socialaccount.providers.google', - # other packages + # other dependencies + 'flat_json_widget', 'private_storage', 'drf_yasg', + 'import_export', + 'admin_auto_filters', 'channels', + 'pipeline', + 'corsheaders', ] EXTENDED_APPS = [ diff --git a/images/openwisp_base/Dockerfile b/images/openwisp_base/Dockerfile index 2bc4eaf1..cf06e680 100644 --- a/images/openwisp_base/Dockerfile +++ b/images/openwisp_base/Dockerfile @@ -65,7 +65,7 @@ RUN if [ "$OPENWISP_USERS_SOURCE" != "default" ] ; then \ pip install --no-cache-dir --user --upgrade --force-reinstall ${OPENWISP_USERS_SOURCE}; \ fi # TODO: Remove when next version of openwisp-utils is released -ARG OPENWISP_UTILS_SOURCE="openwisp-utils[celery,rest] @ https://github.com/openwisp/openwisp-utils/tarball/1d3b00cbd8b36686a39b3f24c42667e6482197b2" +ARG OPENWISP_UTILS_SOURCE="openwisp-utils[celery,rest] @ https://github.com/openwisp/openwisp-utils/tarball/0a7f9214388b634eec437af544fc31c1762a02ce" # hadolint ignore=DL3013 RUN if [ "$OPENWISP_UTILS_SOURCE" != "default" ] ; then \ pip install --no-cache-dir --user --upgrade --force-reinstall "${OPENWISP_UTILS_SOURCE}"; \ @@ -82,13 +82,19 @@ RUN pip install --no-cache-dir --user --upgrade ${DJANGO_SOURCE} COPY ./openwisp_base/requirements.txt /tmp/openwisp-deploy-requirements.txt RUN pip install --no-cache-dir --user --upgrade -r /tmp/openwisp-deploy-requirements.txt +RUN pip install --upgrade urllib3~=2.2.1 FROM system COPY --from=openwisp_python --chown=openwisp:root /home/openwisp/.local/ /usr/local COPY --chown=openwisp:root ./common/ /opt/openwisp/ -RUN mkdir /opt/openwisp/static && mkdir /opt/openwisp/media && mkdir /opt/openwisp/private && \ - mkdir /opt/openwisp/logs && chown -R openwisp:root /opt/openwisp +RUN mkdir /opt/openwisp/static && \ + mkdir /opt/openwisp/media && \ + mkdir /opt/openwisp/private && \ + mkdir /opt/openwisp/logs && \ + chown -R openwisp:root /opt/openwisp +# Maintain backward compatibility with code written for ansible-openwisp2 +RUN ln -s /opt/openwisp/openwisp /opt/openwisp/openwisp2 ENV DASHBOARD_APP_SERVICE=dashboard \ PYTHONUNBUFFERED=1 \ diff --git a/images/openwisp_dashboard/module_settings.py b/images/openwisp_dashboard/module_settings.py index c13be09a..68ea71f6 100644 --- a/images/openwisp_dashboard/module_settings.py +++ b/images/openwisp_dashboard/module_settings.py @@ -17,7 +17,6 @@ 'allauth', 'allauth.account', 'allauth.socialaccount', - 'corsheaders', 'django_extensions', # openwisp modules 'openwisp_users', @@ -41,9 +40,12 @@ # openwisp-firmware-upgrader 'openwisp_firmware_upgrader', # openwisp-radius + 'dj_rest_auth', + 'dj_rest_auth.registration', 'openwisp_radius', # admin 'openwisp_utils.admin_theme', + 'openwisp_utils.metric_collection', 'django.contrib.admin', 'django.forms', # other dependencies @@ -53,11 +55,8 @@ # rest framework 'rest_framework', 'rest_framework_gis', - 'django_filters', - # registration 'rest_framework.authtoken', - 'dj_rest_auth', - 'dj_rest_auth.registration', + 'django_filters', # social login 'allauth.socialaccount.providers.facebook', 'allauth.socialaccount.providers.google', @@ -69,6 +68,7 @@ 'admin_auto_filters', 'channels', 'pipeline', + 'corsheaders', ] EXTENDED_APPS = [