Merge pull request #326 from Amsterdam/yrob/otel_instrumentation

Yrob/otel instrumentation

Dockerfile

@@ -1,16 +1,50 @@
-FROM amsterdam/python:3.9-buster
+FROM python:3.9-buster as builder
 MAINTAINER [email protected]
 
+ENV PYTHONUNBUFFERED 1
+
+RUN apt update && apt install --no-install-recommends -y \
+    curl \
+    gdal-bin \
+    libgeos-c1v5 \
+    libpq5 \
+    netcat-openbsd \
+    build-essential \
+    libgeos-dev \
+    libpq-dev \
+    python3-dev \
+    libffi-dev
+
+WORKDIR /app/
+
+COPY requirements.txt /app/
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Start runtime image
+FROM python:3.9-buster
+RUN apt update && apt install --no-install-recommends -y \
+    gdal-bin \
+    libgeos-c1v5 \
+    libpq5 \
+    netcat-openbsd 
+
+# Copy python build artifacts from builder image
+COPY --from=builder /usr/local/bin/ /usr/local/bin/
+COPY --from=builder /usr/local/lib/python3.9/site-packages/ /usr/local/lib/python3.9/site-packages/
+
+RUN adduser --system datapunt
+
+
 ENV PYTHONUNBUFFERED 1
 ARG BAG_OBJECTSTORE_PASSWORD
 ENV BAG_OBJECTSTORE_PASSWORD=$BAG_OBJECTSTORE_PASSWORD
 
-EXPOSE 8080
+
 WORKDIR /app/
-COPY requirements.txt /app/
 
-RUN pip install --upgrade pip
-RUN pip install -r requirements.txt
+RUN chown datapunt -R /app
+EXPOSE 8080
+
 
 RUN mkdir -p /static && chown datapunt /static
 
@@ -25,4 +59,3 @@ ENV BAG_SECRET_KEY=insecure
 RUN ./manage.py collectstatic
 
 CMD /app/docker-entrypoint.sh
-#

bag/bag/settings/docker.py

@@ -59,7 +59,7 @@
         # Log all unhandled exceptions
         'django.request': {
             'handlers': ['console'],
-            'level': 'ERROR',
+            'level': 'INFO',
             'propagate': False,
         },
     },

bag/bag/settings/settings.py

@@ -1,6 +1,9 @@
 import json
 import sys
 import sentry_sdk
+import logging 
+from azure.monitor.opentelemetry import configure_azure_monitor
+from opentelemetry.sdk.resources import SERVICE_NAME, Resource
 from sentry_sdk.integrations.django import DjangoIntegration
 
 from bag.settings.settings_common import *  # noqa F403
@@ -273,3 +276,39 @@
 else:
     APIKEY_LOCALKEYS = json.loads(apikey_localkeys_env)
 
+
+APPLICATIONINSIGHTS_CONNECTION_STRING = os.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING")
+from opentelemetry.instrumentation.django import DjangoInstrumentor
+if APPLICATIONINSIGHTS_CONNECTION_STRING is not None:
+    configure_azure_monitor(
+        logger_name="root",
+        instrumentation_options={
+            "azure_sdk": {"enabled": False},
+            "django": {"enabled": False},
+            "fastapi": {"enabled": False},
+            "flask": {"enabled": False}, 
+            "psycopg2": {"enabled": False},
+            "requests": {"enabled": False},
+            "urllib": {"enabled": False},
+            "urllib3": {"enabled": False},
+        },
+        resource=Resource.create({SERVICE_NAME: "bag_services"}),
+    )
+
+    # Enable the Python logger
+    logger = logging.getLogger("root")
+    logger.info("OpenTelemetry has been enabled")
+
+    def response_hook(span, request, response):
+        if span and span.is_recording():
+            if "Origin" in request.headers:
+                origin = request.headers['Origin']
+                span.set_attribute("Origin", origin)
+            if "Referer" in request.headers:
+                referer = request.headers['Referer']
+                span.set_attribute("Referer", referer)
+
+
+    # Instrument Django app
+    DjangoInstrumentor().instrument(response_hook=response_hook)
+    print("django instrumentor enabled")

bag/bag/settings/settings_common.py

@@ -273,7 +273,7 @@ def str2bool(v):
         # Log all unhandled exceptions
         'django.request': {
             'handlers': ['console'],
-            'level': 'ERROR',
+            'level': 'INFO',
             'propagate': False,
         },
 

requirements.txt

@@ -1,5 +1,5 @@
 appdirs==1.4.3
-asn1crypto==0.24.0
+asn1crypto==1.3.0
 astroid==2.0.1
 Babel==2.9.1
 cached-property==1.5.1
@@ -11,7 +11,7 @@ colorama==0.4.1
 coreapi==2.3.3
 coreschema==0.0.4
 coverage==4.5.2
-cryptography==41.0.6
+cryptography==42.0.4
 datadiensten-apikeyclient==0.3.5
 datapunt-authorization-django==1.3.2
 debtcollector==1.20.0
@@ -43,7 +43,7 @@ frosted==1.4.1
 graypy==0.3.1
 httplib2==0.19.0
 idna==2.8
-ipaddress==1.0.22
+ipaddress==1.0.23
 iso8601==0.1.12
 isort==4.3.4
 itypes==1.2.0
@@ -76,16 +76,16 @@ pyflakes==2.0.0
 PyJWT==2.8.0
 pyparsing==2.4.7
 pyslack-real==0.6.0
-python-dateutil==2.7.5
+python-dateutil==2.8.1
 python-keystoneclient==3.18.0
 python-swiftclient==3.6.0
-pytz==2018.9
+pytz==2019.3
 PyYAML==6.0.1
-requests==2.31.0
+requests==2.32.0
 rfc3986==1.2.0
 sentry-sdk==1.14.0
 simplejson==3.16.0
-six==1.12.0
+six==1.14.0
 sphinx-me==0.3
 sqlparse==0.4.4
 stevedore==1.30.0
@@ -94,6 +94,11 @@ texttable==1.5.0
 typing==3.6.6
 unicodecsv==0.14.1
 uritemplate==3.0.0
-urllib3==1.26.18
+uwsgi==2.0.22
+uwsgi-readiness-check==0.2.0
+uwsgitop==0.12
+urllib3==1.26.19
 websocket-client==0.54.0
 Werkzeug==3.0.1
+azure-identity==1.16.1
+azure-monitor-opentelemetry == 1.5.0