[feature] Enabled metric collection

.env

@@ -45,6 +45,7 @@ VPN_CLIENT_NAME=default-management-vpn
 # Developer
 DEBUG_MODE=False
 DJANGO_LOG_LEVEL=INFO
+# Celery workers
 USE_OPENWISP_CELERY_TASK_ROUTES_DEFAULTS=True
 OPENWISP_CELERY_COMMAND_FLAGS=--concurrency=1
 USE_OPENWISP_CELERY_NETWORK=True
@@ -54,3 +55,5 @@ OPENWISP_CELERY_MONITORING_COMMAND_FLAGS=--concurrency=1
 OPENWISP_CELERY_MONITORING_CHECKS_COMMAND_FLAGS=--concurrency=1
 USE_OPENWISP_CELERY_FIRMWARE=True
 OPENWISP_CELERY_FIRMWARE_COMMAND_FLAGS=--concurrency=1
+# Metric collection
+METRIC_COLLECTION=True

.github/workflows/branch.yml

@@ -25,6 +25,8 @@ jobs:
       - name: Setup
         run: |
           echo "127.0.0.1 dashboard.openwisp.org api.openwisp.org" | sudo tee -a /etc/hosts
+          # disable metric collection during builds
+          sed -i 's/METRIC_COLLECTION=True/METRIC_COLLECTION=False/' .env
 
       # the following action is equivalent to
       # echo "$DOCKER_HUB_SECRET" | docker login --username "$DOCKER_HUB_USERNAME" --password-stdin

.github/workflows/pull_request.yml

@@ -28,6 +28,8 @@ jobs:
       - name: Setup
         run: |
           echo "127.0.0.1 dashboard.openwisp.org api.openwisp.org" | sudo tee -a /etc/hosts
+          # disable metric collection during builds
+          sed -i 's/METRIC_COLLECTION=True/METRIC_COLLECTION=False/' .env
 
       - name: Build Images
         run: make compose-build nfs-build

docs/ENV.md

@@ -165,6 +165,12 @@ Any OpenWISP Configuration of type `string`. `int`, `bool` or `json` is supporte
 - **Default:** None
 - **Example:** [{"name": "Name1","label": "Label1","boards": ["TestA", "TestB"]}, {"name": "Name2","label": "Label2","boards": ["TestC", "TestD"]}]
 
+### `METRIC_COLLECTION`
+
+- **Explanation:** Whether the usage metric collection feature of openwisp-utils is enabled or not.
+- **Valid Values:** True | False
+- **Default:** True
+
 ## Database
 
 ### `DB_NAME`

images/common/openwisp/__init__.py

@@ -3,3 +3,5 @@
 from .celery import app as celery_app
 
 __all__ = ['celery_app']
+__openwisp_version__ = '24.0.0a'
+__openwisp_installation_method__ = 'docker-openwisp'

images/common/openwisp/celery.py

@@ -7,7 +7,12 @@
 
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'openwisp.settings')
 
-radius_schedule, topology_schedule, monitoring_schedule = {}, {}, {}
+radius_schedule, topology_schedule, monitoring_schedule, metric_collection_schedule = (
+    {},
+    {},
+    {},
+    {},
+)
 task_routes = {}
 
 if env_bool(os.environ.get('USE_OPENWISP_CELERY_NETWORK')):
@@ -59,6 +64,14 @@
         },
     }
 
+if env_bool(os.environ.get('METRIC_COLLECTION', 'True')):
+    metric_collection_schedule = {
+        'send_usage_metrics': {
+            'task': 'openwisp_utils.metric_collection.tasks.send_usage_metrics',
+            'schedule': timedelta(days=1),
+        },
+    }
+
 notification_schedule = {
     'notification-delete-tasks': {
         'task': 'openwisp_notifications.tasks.delete_old_notifications',
@@ -79,6 +92,7 @@
         **topology_schedule,
         **notification_schedule,
         **monitoring_schedule,
+        **metric_collection_schedule,
     },
 )
 app.config_from_object('django.conf:settings', namespace='CELERY')

images/common/openwisp/settings.py

@@ -357,6 +357,8 @@
     },
 }
 
+TEST_RUNNER = 'openwisp_utils.metric_collection.tests.runner.MockRequestPostRunner'
+
 # Add Custom OpenWRT Images for openwisp firmware
 try:
     OPENWRT_IMAGES = json.loads(os.environ['OPENWISP_CUSTOM_OPENWRT_IMAGES'])
@@ -408,6 +410,8 @@
         INSTALLED_APPS.remove('openwisp_monitoring.device')
     if 'openwisp_monitoring.check' in INSTALLED_APPS:
         INSTALLED_APPS.remove('openwisp_monitoring.check')
+if not env_bool(os.environ.get('METRIC_COLLECTION', 'True')):
+    INSTALLED_APPS.remove('openwisp_utils.metric_collection')
 if EMAIL_BACKEND == 'djcelery_email.backends.CeleryEmailBackend':
     INSTALLED_APPS.append('djcelery_email')
 

images/openwisp_api/module_settings.py

@@ -8,11 +8,13 @@
     'django.contrib.humanize',
     # all-auth
     'django.contrib.sites',
+    # overrides allauth templates
+    # must precede allauth
+    'openwisp_users.accounts',
     'allauth',
     'allauth.account',
     'allauth.socialaccount',
     'django_extensions',
-    'corsheaders',
     # openwisp modules
     'openwisp_users',
     # openwisp-controller
@@ -35,11 +37,12 @@
     # openwisp-firmware-upgrader
     'openwisp_firmware_upgrader',
     # openwisp radius
+    'dj_rest_auth',
+    'dj_rest_auth.registration',
     'openwisp_radius',
-    # Admin theme required for the templates
-    # stored in openwisp_utils/admin_theme
-    'openwisp_utils.admin_theme',
     # admin
+    'openwisp_utils.admin_theme',
+    'openwisp_utils.metric_collection',
     'django.contrib.admin',
     'django.forms',
     # other dependencies
@@ -49,18 +52,20 @@
     # rest framework
     'rest_framework',
     'rest_framework_gis',
-    'django_filters',
-    # registration
     'rest_framework.authtoken',
-    'dj_rest_auth',
-    'dj_rest_auth.registration',
+    'django_filters',
     # social login
     'allauth.socialaccount.providers.facebook',
     'allauth.socialaccount.providers.google',
-    # other packages
+    # other dependencies
+    'flat_json_widget',
     'private_storage',
     'drf_yasg',
+    'import_export',
+    'admin_auto_filters',
     'channels',
+    'pipeline',
+    'corsheaders',
 ]
 
 EXTENDED_APPS = [

images/openwisp_base/Dockerfile

@@ -65,7 +65,7 @@ RUN if [ "$OPENWISP_USERS_SOURCE" != "default" ] ; then \
         pip install --no-cache-dir --user --upgrade --force-reinstall ${OPENWISP_USERS_SOURCE}; \
     fi
 # TODO: Remove when next version of openwisp-utils is released
-ARG OPENWISP_UTILS_SOURCE="openwisp-utils[celery,rest] @ https://github.com/openwisp/openwisp-utils/tarball/1d3b00cbd8b36686a39b3f24c42667e6482197b2"
+ARG OPENWISP_UTILS_SOURCE="openwisp-utils[celery,rest] @ https://github.com/openwisp/openwisp-utils/tarball/0a7f9214388b634eec437af544fc31c1762a02ce"
 # hadolint ignore=DL3013
 RUN if [ "$OPENWISP_UTILS_SOURCE" != "default" ] ; then \
         pip install --no-cache-dir --user --upgrade --force-reinstall "${OPENWISP_UTILS_SOURCE}"; \
@@ -82,13 +82,19 @@ RUN pip install --no-cache-dir --user --upgrade ${DJANGO_SOURCE}
 
 COPY ./openwisp_base/requirements.txt /tmp/openwisp-deploy-requirements.txt
 RUN pip install --no-cache-dir --user --upgrade -r /tmp/openwisp-deploy-requirements.txt
+RUN pip install --upgrade urllib3~=2.2.1
 
 FROM system
 
 COPY --from=openwisp_python --chown=openwisp:root /home/openwisp/.local/ /usr/local
 COPY --chown=openwisp:root ./common/ /opt/openwisp/
-RUN mkdir /opt/openwisp/static && mkdir /opt/openwisp/media && mkdir /opt/openwisp/private && \
-    mkdir /opt/openwisp/logs && chown -R openwisp:root /opt/openwisp
+RUN mkdir /opt/openwisp/static && \
+    mkdir /opt/openwisp/media && \
+    mkdir /opt/openwisp/private && \
+    mkdir /opt/openwisp/logs && \
+    chown -R openwisp:root /opt/openwisp
+# Maintain backward compatibility with code written for ansible-openwisp2
+RUN ln -s /opt/openwisp/openwisp /opt/openwisp/openwisp2
 
 ENV DASHBOARD_APP_SERVICE=dashboard \
     PYTHONUNBUFFERED=1 \

images/openwisp_dashboard/module_settings.py

@@ -17,7 +17,6 @@
     'allauth',
     'allauth.account',
     'allauth.socialaccount',
-    'corsheaders',
     'django_extensions',
     # openwisp modules
     'openwisp_users',
@@ -41,9 +40,12 @@
     # openwisp-firmware-upgrader
     'openwisp_firmware_upgrader',
     # openwisp-radius
+    'dj_rest_auth',
+    'dj_rest_auth.registration',
     'openwisp_radius',
     # admin
     'openwisp_utils.admin_theme',
+    'openwisp_utils.metric_collection',
     'django.contrib.admin',
     'django.forms',
     # other dependencies
@@ -53,11 +55,8 @@
     # rest framework
     'rest_framework',
     'rest_framework_gis',
-    'django_filters',
-    # registration
     'rest_framework.authtoken',
-    'dj_rest_auth',
-    'dj_rest_auth.registration',
+    'django_filters',
     # social login
     'allauth.socialaccount.providers.facebook',
     'allauth.socialaccount.providers.google',
@@ -69,6 +68,7 @@
     'admin_auto_filters',
     'channels',
     'pipeline',
+    'corsheaders',
 ]
 
 EXTENDED_APPS = [