[feature] Added optional clean insights measurement collection #360

README.rst

@@ -54,6 +54,7 @@ Current features
 * `Admin Theme utilities <#admin-theme-utilities>`_
 * `REST API utilities <#rest-api-utilities>`_
 * `Test utilities <#test-utilities>`_
+* `Collection of Usage Metrics <#collection-of-usage-metrics>`_
 * `Quality assurance checks <#quality-assurance-checks>`_
 
 ------------
@@ -1347,6 +1348,60 @@ Usage:
 but not for complex background tasks which can take a long time to execute
 (eg: firmware upgrades, network operations with retry mechanisms).
 
+``openwisp_utils.tasks.retryable_requests``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A utility function for making HTTP requests with built-in retry logic.
+This function is useful for handling transient errors encountered during HTTP
+requests by automatically retrying failed requests with exponential backoff.
+It provides flexibility in configuring various retry parameters to suit
+different use cases.
+
+Usage:
+
+.. code-block:: python
+
+    from your_module import retryable_request
+
+    response = retryable_request(
+        method='GET',
+        url='https://openwisp.org',
+        timeout=(4, 8),
+        max_retries=3,
+        backoff_factor=1,
+        backoff_jitter=0.0,
+        status_forcelist=(429, 500, 502, 503, 504),
+        allowed_methods=('HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE', 'POST'),
+        retry_kwargs=None,
+        headers={'Authorization': 'Bearer token'}
+    )
+
+**Paramters:**
+
+- ``method`` (str): The HTTP method to be used for the request in lower
+  case (e.g., 'get', 'post', etc.).
+- ``timeout`` (tuple): A tuple containing two elements: connection timeout
+  and read timeout in seconds (default: (4, 8)).
+- ``max_retries`` (int): The maximum number of retry attempts in case of
+  request failure (default: 3).
+- ``backoff_factor`` (float): A factor by which the retry delay increases
+  after each retry (default: 1).
+- ``backoff_jitter`` (float): A jitter to apply to the backoff factor to prevent
+  retry storms (default: 0.0).
+- ``status_forcelist`` (tuple): A tuple of HTTP status codes for which retries
+  should be attempted (default: (429, 500, 502, 503, 504)).
+- ``allowed_methods`` (tuple): A tuple of HTTP methods that are allowed for
+  the request (default: ('HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE', 'POST')).
+- ``retry_kwargs`` (dict): Additional keyword arguments to be passed to the
+  retry mechanism (default: None).
+- ``**kwargs``: Additional keyword arguments to be passed to the underlying request
+  method (e.g. 'headers', etc.).
+
+Note: This method will raise a requests.exceptions.RetryError if the request
+remains unsuccessful even after all retry attempts have been exhausted.
+This exception indicates that the operation could not be completed successfully
+despite the retry mechanism.
+
 Storage utilities
 -----------------
 
@@ -1615,6 +1670,29 @@ This backend extends ``django.contrib.gis.db.backends.spatialite``
 database backend to implement a workaround for handling
 `issue with sqlite 3.36 and spatialite 5 <https://code.djangoproject.com/ticket/32935>`_.
 
+Collection of Usage Metrics
+---------------------------
+
+The openwisp-utils module includes an optional sub-app ``openwisp_utils.measurements``.
+This sub-app enables collection of following measurements:
+
+- Installed OpenWISP Version
+- Enabled OpenWISP modules: A list of the enabled OpenWISP modules
+  along with their respective versions
+- OS details: Information on the operating system, including its
+  version, kernel version, and platform
+- Whether the event is related to a new installation or an upgrade
+
+We collect data on OpenWISP usage to gauge user engagement, satisfaction,
+and upgrade patterns. This informs our development decisions, ensuring
+continuous improvement aligned with user needs.
+
+To enhance our understanding and management of this data, we have
+integrated `Clean Insights <https://cleaninsights.org/>`_, a privacy-preserving
+analytics tool. Clean Insights allows us to responsibly gather and analyze
+usage metrics without compromising user privacy. It provides us with the
+means to make data-driven decisions while respecting our users' rights and trust.
+
 Quality Assurance Checks
 ------------------------
 

openwisp_utils/measurements/apps.py

@@ -0,0 +1,42 @@
+from django.apps import AppConfig
+from django.conf import settings
+from django.db.models.signals import post_migrate
+
+
+class MeasurementsConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'openwisp_utils.measurements'
+    app_label = 'openwisp_measurements'
+
+    def ready(self):
+        super().ready()
+        self.connect_post_migrate_signal()
+
+    def connect_post_migrate_signal(self):
+        post_migrate.connect(self.post_migrate_receiver, sender=self)
+
+    @classmethod
+    def post_migrate_receiver(cls, **kwargs):
+        if getattr(settings, 'DEBUG', False):
+            # Do not send usage metrics in debug mode
+            # i.e. when running tests.
+            return
+
+        from .tasks import send_usage_metrics
+
+        is_new_install = False
+        if kwargs.get('plan'):
+            migration, migration_rolled_back = kwargs['plan'][0]
+            is_new_install = (
+                migration_rolled_back is False
+                and str(migration) == 'contenttypes.0001_initial'
+            )
+
+        # If the migration plan includes creating table
+        # for the ContentType model, then the installation is
+        # treated as a new installation.
+        if is_new_install:
+            # This is a new installation
+            send_usage_metrics.delay()
+        else:
+            send_usage_metrics.delay(upgrade_only=True)

openwisp_utils/measurements/migrations/0001_initial.py

@@ -0,0 +1,42 @@
+# Generated by Django 4.2.7 on 2023-12-06 15:30
+
+from django.db import migrations, models
+import django.utils.timezone
+import model_utils.fields
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name="OpenwispVersion",
+            fields=[
+                (
+                    "id",
+                    models.UUIDField(
+                        default=uuid.uuid4,
+                        editable=False,
+                        primary_key=True,
+                        serialize=False,
+                    ),
+                ),
+                (
+                    "created",
+                    model_utils.fields.AutoCreatedField(
+                        default=django.utils.timezone.now,
+                        editable=False,
+                        verbose_name="created",
+                    ),
+                ),
+                ("module_version", models.JSONField(blank=True, default=dict)),
+            ],
+            options={
+                "ordering": ("-created",),
+            },
+        ),
+    ]

openwisp_utils/measurements/models.py

@@ -0,0 +1,46 @@
+from django.db import models
+from openwisp_utils.base import TimeStampedEditableModel
+from packaging.version import parse as parse_version
+
+
+class OpenwispVersion(TimeStampedEditableModel):
+    modified = None
+    module_version = models.JSONField(default=dict, blank=True)
+
+    class Meta:
+        ordering = ('-created',)
+
+    @classmethod
+    def is_new_installation(cls):
+        return not cls.objects.exists()
+
+    @classmethod
+    def get_upgraded_modules(cls, current_versions):
+        """
+        Retrieves a dictionary of upgraded modules based on current versions.
+        Also updates the OpenwispVersion object with the new versions.
+
+        Args:
+            current_versions (dict): A dictionary containing the current versions of modules.
+
+        Returns:
+            dict: A dictionary containing the upgraded modules and their versions.
+        """
+        openwisp_version = cls.objects.first()
+        if not openwisp_version:
+            cls.objects.create(module_version=current_versions)
+            return {}
+        old_versions = openwisp_version.module_version
+        upgraded_modules = {}
+        for module, version in current_versions.items():
+            if module in old_versions and parse_version(
+                old_versions[module]
+            ) < parse_version(version):
+                upgraded_modules[module] = version
+            openwisp_version.module_version[module] = version
+        if upgraded_modules:
+            # Save the new versions in a new object
+            OpenwispVersion.objects.create(
+                module_version=openwisp_version.module_version
+            )
+        return upgraded_modules

openwisp_utils/measurements/tasks.py

@@ -0,0 +1,56 @@
+import logging
+
+from celery import shared_task
+from openwisp_utils.admin_theme.system_info import (
+    get_enabled_openwisp_modules,
+    get_openwisp_version,
+    get_os_details,
+)
+
+from ..tasks import OpenwispCeleryTask
+from ..utils import retryable_request
+from .models import OpenwispVersion
+from .utils import _get_events, get_openwisp_module_metrics, get_os_detail_metrics
+
+USER_METRIC_COLLECTION_URL = 'https://analytics.openwisp.io/cleaninsights.php'
+
+logger = logging.getLogger(__name__)
+
+
+def post_usage_metrics(events):
+    try:
+        response = retryable_request(
+            'post',
+            url=USER_METRIC_COLLECTION_URL,
+            json={
+                'idsite': 5,
+                'events': events,
+            },
+            max_retries=10,
+        )
+        assert response.status_code == 204
+    except Exception as error:
+        if isinstance(error, AssertionError):
+            message = f'HTTP {response.status_code} Response'
+        else:
+            message = str(error)
+        logger.error(
+            f'Collection of usage metrics failed, max retries exceeded. Error: {message}'
+        )
+
+
+@shared_task(base=OpenwispCeleryTask)
+def send_usage_metrics(upgrade_only=False):
+    current_versions = get_enabled_openwisp_modules()
+    current_versions.update({'OpenWISP Version': get_openwisp_version()})
+    metrics = []
+    metrics.extend(get_os_detail_metrics(get_os_details()))
+    if OpenwispVersion.is_new_installation():
+        metrics.extend(_get_events('Install', current_versions))
+        OpenwispVersion.objects.create(module_version=current_versions)
+    else:
+        upgraded_modules = OpenwispVersion.get_upgraded_modules(current_versions)
+        metrics.extend(_get_events('Upgrade', upgraded_modules))
+        if not upgrade_only:
+            metrics.extend(get_openwisp_module_metrics(current_versions))
+    post_usage_metrics(metrics)