diff --git a/.ci/config.yml b/.ci/config.yml index f286d81c3..9f34989d1 100644 --- a/.ci/config.yml +++ b/.ci/config.yml @@ -95,3 +95,5 @@ SUPERSET_XAPI_DASHBOARD_SLUG: openedx-xapi ASPECTS_VERSION: 0.91.0 DOCKER_IMAGE_OPENEDX: edunext/openedx-aspects:{{ASPECTS_VERSION}} DOCKER_IMAGE_OPENEDX_DEV: edunext/openedx-aspects-dev:{{ASPECTS_VERSION}} +ASPECTS_ENABLE_EVENT_BUS_CONSUMER: true +ASPECTS_ENABLE_EVENT_BUS_PRODUCER: true diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 7aff5f698..b7c60dfec 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -20,9 +20,6 @@ jobs: runs-on: ubuntu-latest if: "!startsWith(github.head_ref, 'dependabot/')" steps: - - name: Checkout - run: - echo ${{github.head_ref}} - name: Log in to Docker Hub continue-on-error: true uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 @@ -31,10 +28,13 @@ jobs: password: ${{ secrets.EDUNEXT_DOCKER_PASSWORD }} - name: Checkout uses: actions/checkout@v4 + - name: Set outputs + id: vars + run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - name: Update ASPECTS_VERSION if: github.event.action == 'opened' || github.event.action == 'synchronize' run: | - ASPECTS_VERSION=pr-${{ github.event.pull_request.number }} + ASPECTS_VERSION=pr-${{ github.event.pull_request.number }}-${{ steps.vars.outputs.sha_short }} sed -i "s/ASPECTS_VERSION: .*/ASPECTS_VERSION: $ASPECTS_VERSION/" .ci/config.yml - name: setup python uses: actions/setup-python@v5 diff --git a/tutoraspects/patches/k8s-deployments b/tutoraspects/patches/k8s-deployments index d38c2979f..875422172 100644 --- a/tutoraspects/patches/k8s-deployments +++ b/tutoraspects/patches/k8s-deployments @@ -538,3 +538,62 @@ spec: hostPath: path: /var/log/ {% endif %} + +{% if ASPECTS_ENABLE_EVENT_BUS_CONSUMER %} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: aspects-consumer + labels: + app.kubernetes.io/name: aspects-consumer +spec: + selector: + matchLabels: + app.kubernetes.io/name: aspects-consumer + template: + metadata: + labels: + app.kubernetes.io/name: aspects-consumer + spec: + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + containers: + - name: aspects-consumer + image: {{ DOCKER_IMAGE_OPENEDX }} + env: + - name: SERVICE_VARIANT + value: lms + - name: DJANGO_SETTINGS_MODULE + value: lms.envs.tutor.production + volumeMounts: + - mountPath: /openedx/edx-platform/lms/envs/tutor/ + name: settings-lms + - mountPath: /openedx/edx-platform/cms/envs/tutor/ + name: settings-cms + - mountPath: /openedx/config + name: config + securityContext: + allowPrivilegeEscalation: false + command: + - "./manage.py" + - "lms" + - "consume_events" + - "-t" + - "analytics" + - "-g" + - "event_routing_backends" + - "--extra" + - '{"consumer_name": "aspects"}' + volumes: + - name: settings-lms + configMap: + name: openedx-settings-lms + - name: settings-cms + configMap: + name: openedx-settings-cms + - name: config + configMap: + name: openedx-config +{% endif %} diff --git a/tutoraspects/patches/local-docker-compose-dev-services b/tutoraspects/patches/local-docker-compose-dev-services index 7e1ab8e8d..89f83e685 100644 --- a/tutoraspects/patches/local-docker-compose-dev-services +++ b/tutoraspects/patches/local-docker-compose-dev-services @@ -45,3 +45,12 @@ clickhouse: - ../../env/plugins/aspects/apps/clickhouse/dev_config:/etc/clickhouse-server/config.d/ - ../../env/plugins/aspects/apps/clickhouse/users:/etc/clickhouse-server/users.d/ {% endif %} + +{% if ASPECTS_ENABLE_EVENT_BUS_CONSUMER %} +aspects-consumer: + <<: *openedx-service + environment: + DJANGO_SETTINGS_MODULE: lms.envs.tutor.development + command: | + ./manage.py lms consume_events -t analytics -g event_routing_backends --extra '{"consumer_name": "aspects"}' +{% endif %} diff --git a/tutoraspects/patches/local-docker-compose-services b/tutoraspects/patches/local-docker-compose-services index d66840b56..399560de9 100644 --- a/tutoraspects/patches/local-docker-compose-services +++ b/tutoraspects/patches/local-docker-compose-services @@ -95,3 +95,31 @@ vector: - VECTOR_LOG=warn restart: unless-stopped {% endif %} + +{% if ASPECTS_ENABLE_EVENT_BUS_CONSUMER %} +aspects-consumer: + image: {{ DOCKER_IMAGE_OPENEDX }} + stdin_open: true + tty: true + volumes: + # Settings & config + - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro + - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro + - ../apps/openedx/config:/openedx/config:ro + {%- for mount in iter_mounts(MOUNTS, "openedx", "lms") %} + - {{ mount }} + {%- endfor %} + environment: + SERVICE_VARIANT: lms + DJANGO_SETTINGS_MODULE: lms.envs.tutor.production + restart: unless-stopped + command: | + ./manage.py lms consume_events -t analytics -g event_routing_backends --extra '{"consumer_name": "aspects"}' + deploy: + mode: replicated + replicas: {{ ASPECTS_EVENT_BUS_CONSUMER_REPLICAS }} + depends_on: + {% if RUN_MYSQL %}- mysql{% endif %} + {% if RUN_MONGODB %}- mongodb{% endif %} + {% if RUN_REDIS %}- redis{% endif %} +{% endif %} diff --git a/tutoraspects/patches/openedx-common-settings b/tutoraspects/patches/openedx-common-settings index 3cbe0615e..4ce683d21 100644 --- a/tutoraspects/patches/openedx-common-settings +++ b/tutoraspects/patches/openedx-common-settings @@ -27,3 +27,25 @@ if not OPEN_EDX_FILTERS_CONFIG.get("org.openedx.learning.instructor.dashboard.re } OPEN_EDX_FILTERS_CONFIG["org.openedx.learning.instructor.dashboard.render.started.v1"]["pipeline"].append("platform_plugin_aspects.extensions.filters.AddSupersetTab") {% endif %} + +{% if ASPECTS_ENABLE_EVENT_BUS_PRODUCER %} +# Update the backends to use the event bus +EVENT_TRACKING_BACKENDS["event_transformer"]["ENGINE"] = "eventtracking.backends.event_bus.EventBusRoutingBackend" +# Update backend to send events in sync mode +EVENT_TRACKING_BACKENDS["event_transformer"]["OPTIONS"]["backends"]["xapi"]["ENGINE"] = "event_routing_backends.backends.sync_events_router.SyncEventsRouter" +EVENT_TRACKING_BACKENDS["event_transformer"]["OPTIONS"]["backends"]["caliper"]["ENGINE"] = "event_routing_backends.backends.sync_events_router.SyncEventsRouter" +# Enable the event bus producer +SEND_TRACKING_EVENT_EMITTED_SIGNAL = True +# Update the event bus producer config +EVENT_BUS_PRODUCER_CONFIG.update( + { + "org.openedx.analytics.tracking.event.emitted.v1": { + "analytics": { + "event_key_field": "tracking_log.name", "enabled": True + } + } + } +) + +INSTALLED_APPS.append("openedx_events") +{% endif %} diff --git a/tutoraspects/patches/openedx-dev-dockerfile-post-python-requirements b/tutoraspects/patches/openedx-dev-dockerfile-post-python-requirements new file mode 100644 index 000000000..f272b1dcb --- /dev/null +++ b/tutoraspects/patches/openedx-dev-dockerfile-post-python-requirements @@ -0,0 +1,4 @@ +RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ + pip install "platform-plugin-aspects==v0.5.0" +RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ + pip install "edx-event-routing-backends==v9.0.0" diff --git a/tutoraspects/patches/openedx-dockerfile-post-python-requirements b/tutoraspects/patches/openedx-dockerfile-post-python-requirements index 8decd527c..f272b1dcb 100644 --- a/tutoraspects/patches/openedx-dockerfile-post-python-requirements +++ b/tutoraspects/patches/openedx-dockerfile-post-python-requirements @@ -1,4 +1,4 @@ RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ pip install "platform-plugin-aspects==v0.5.0" RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ - pip install "edx-event-routing-backends==v8.3.1" + pip install "edx-event-routing-backends==v9.0.0" diff --git a/tutoraspects/plugin.py b/tutoraspects/plugin.py index c94e0c2f5..1f2717385 100644 --- a/tutoraspects/plugin.py +++ b/tutoraspects/plugin.py @@ -51,6 +51,9 @@ # in the ClickHouse database. Make sure that you understand the legal # consequences of data storage and privacy before turning this on! ("ASPECTS_ENABLE_PII", False), + ("ASPECTS_ENABLE_EVENT_BUS_CONSUMER", False), + ("ASPECTS_ENABLE_EVENT_BUS_PRODUCER", False), + ("ASPECTS_EVENT_BUS_CONSUMER_REPLICAS", 1), # User PII is cached in an in-memory dictionary for this many seconds. ("ASPECTS_PII_CACHE_LIFETIME", 900), # Markdown comprising the Help tab for the Operator and Instructor dashboards.