diff --git a/tutoraspects/patches/openedx-dev-dockerfile-post-python-requirements b/tutoraspects/patches/openedx-dev-dockerfile-post-python-requirements index e95051d42..61193d972 100644 --- a/tutoraspects/patches/openedx-dev-dockerfile-post-python-requirements +++ b/tutoraspects/patches/openedx-dev-dockerfile-post-python-requirements @@ -1,5 +1,5 @@ RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ - pip install "platform-plugin-aspects==v0.11.4" + pip install "platform-plugin-aspects==v0.11.2" RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ pip install "edx-event-routing-backends>=9.3.0,<9.4" diff --git a/tutoraspects/patches/openedx-dockerfile-post-python-requirements b/tutoraspects/patches/openedx-dockerfile-post-python-requirements index e95051d42..61193d972 100644 --- a/tutoraspects/patches/openedx-dockerfile-post-python-requirements +++ b/tutoraspects/patches/openedx-dockerfile-post-python-requirements @@ -1,5 +1,5 @@ RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ - pip install "platform-plugin-aspects==v0.11.4" + pip install "platform-plugin-aspects==v0.11.2" RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ pip install "edx-event-routing-backends>=9.3.0,<9.4" diff --git a/tutoraspects/plugin.py b/tutoraspects/plugin.py index ee80a862c..9432f8fb1 100644 --- a/tutoraspects/plugin.py +++ b/tutoraspects/plugin.py @@ -385,8 +385,8 @@ # DBT_PROFILE_* settings get passed into the dbt_profile.yml file. # For now we are pulling this from github, which should allow maximum # flexibility for forking, running branches, specific versions, etc. - ("DBT_REPOSITORY", "https://github.com/openedx/aspects-dbt"), - ("DBT_BRANCH", "v3.33.7"), + ("DBT_REPOSITORY", "https://github.com/saraburns1/aspects-dbt"), + ("DBT_BRANCH", "fixrewatched_again"), ("DBT_SSH_KEY", ""), ("DBT_STATE_DIR", "/app/aspects-dbt/state"), ("DBT_PROFILES_DIR", "/app/aspects/dbt/"), diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Info_b2fff8.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Info_b2fff8.yaml index d0cfff243..af3541b6c 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Info_b2fff8.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Info_b2fff8.yaml @@ -50,7 +50,7 @@ params: viz_type: table query_context: datasource: - id: 651 + id: 261 type: table force: false form_data: @@ -61,6 +61,7 @@ query_context: operator: TEMPORAL_RANGE subject: emission_time all_columns: [] + annotation_layers: [] color_pn: false column_config: active_count: @@ -74,9 +75,7 @@ query_context: tag_list: truncateLongCells: true conditional_formatting: [] - dashboards: - - 355 - datasource: 651__table + datasource: 261__table extra_form_data: {} force: false groupby: @@ -98,7 +97,7 @@ query_context: result_type: full row_limit: 50000 show_cell_bars: false - slice_id: 1298 + slice_id: 561 table_timestamp_format: smart_date temporal_columns_lookup: {} time_grain_sqla: P1M diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_5b00f7.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_5b00f7.yaml index 22364c178..74a8128dc 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_5b00f7.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_5b00f7.yaml @@ -20,9 +20,10 @@ params: - org metrics: - num_videos + - avg_video_length + - num_videos_watched - watched_percent - rewatched_percent - - avg_video_length order_by_cols: [] order_desc: true percent_metrics: [] @@ -34,7 +35,7 @@ params: viz_type: table query_context: datasource: - id: 65 + id: 161 type: table force: false form_data: @@ -48,8 +49,8 @@ query_context: d3NumberFormat: ',.1%' conditional_formatting: [] dashboards: - - 22 - datasource: 65__table + - 241 + datasource: 161__table extra_form_data: {} force: false groupby: @@ -57,9 +58,10 @@ query_context: - org metrics: - num_videos + - avg_video_length + - num_videos_watched - watched_percent - rewatched_percent - - avg_video_length order_by_cols: [] order_desc: true percent_metrics: [] @@ -68,7 +70,7 @@ query_context: result_type: full row_limit: 50000 show_cell_bars: false - slice_id: 82 + slice_id: 1121 table_timestamp_format: smart_date temporal_columns_lookup: {} viz_type: table @@ -86,9 +88,10 @@ query_context: filters: [] metrics: - num_videos + - avg_video_length + - num_videos_watched - watched_percent - rewatched_percent - - avg_video_length order_desc: true orderby: - - num_videos diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_6b7610.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_6b7610.yaml index eb8c74164..d3d3142d8 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_6b7610.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_6b7610.yaml @@ -20,21 +20,23 @@ params: - course_name metrics: - num_videos + - avg_video_length + - num_videos_watched - watched_percent - rewatched_percent - - avg_video_length order_by_cols: [] order_desc: true percent_metrics: [] query_mode: aggregate row_limit: 50000 + server_page_length: 10 show_cell_bars: false table_timestamp_format: smart_date temporal_columns_lookup: {} viz_type: table query_context: datasource: - id: 65 + id: 161 type: table force: false form_data: @@ -48,8 +50,8 @@ query_context: d3NumberFormat: ',.1%' conditional_formatting: [] dashboards: - - 22 - datasource: 65__table + - 241 + datasource: 161__table extra_form_data: {} force: false groupby: @@ -57,9 +59,10 @@ query_context: - course_name metrics: - num_videos + - avg_video_length + - num_videos_watched - watched_percent - rewatched_percent - - avg_video_length order_by_cols: [] order_desc: true percent_metrics: [] @@ -67,8 +70,9 @@ query_context: result_format: json result_type: full row_limit: 50000 + server_page_length: 10 show_cell_bars: false - slice_id: 145 + slice_id: 641 table_timestamp_format: smart_date temporal_columns_lookup: {} viz_type: table @@ -86,9 +90,10 @@ query_context: filters: [] metrics: - num_videos + - avg_video_length + - num_videos_watched - watched_percent - rewatched_percent - - avg_video_length order_desc: true orderby: - - num_videos diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Course_Comparison_Dashboard.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Course_Comparison_Dashboard.yaml index be7c7ec51..3556f7f68 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Course_Comparison_Dashboard.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Course_Comparison_Dashboard.yaml @@ -171,12 +171,11 @@ metadata: - '#9EE5E5' - '#D1C6BC' cross_filters_enabled: true - default_filters: '{}' expanded_slices: - '1252': true - '399': true - '422': true - '776': true + '101': true + '1461': true + '461': true + '61': true global_chart_configuration: chartsInScope: - 314 @@ -350,7 +349,7 @@ position: children: [] id: CHART-explore-1467-1 meta: - chartId: 963 + chartId: 1121 height: 50 sliceName: Video Engagement uuid: 5b00f792-233c-4ee6-9411-7df49f4b5061 @@ -365,7 +364,7 @@ position: children: [] id: CHART-explore-1500-1 meta: - chartId: 314 + chartId: 541 height: 50 sliceName: Course Info uuid: b2fff8a7-3ace-4075-9a3f-5fa5b99f81bb @@ -380,7 +379,7 @@ position: children: [] id: CHART-explore-1501-1 meta: - chartId: 757 + chartId: 201 height: 50 sliceName: Course Info uuid: ccdd7d98-4722-490b-a0b2-077380a7c7eb @@ -395,7 +394,7 @@ position: children: [] id: CHART-explore-1502-1 meta: - chartId: 1154 + chartId: 161 height: 50 sliceName: Enrollment Counts uuid: 7eaf5bc2-0067-424b-a50b-607cf54c086a @@ -410,7 +409,7 @@ position: children: [] id: CHART-explore-1503-1 meta: - chartId: 407 + chartId: 1141 height: 50 sliceName: Enrollment Counts uuid: 8230a366-957d-4667-a2ff-3cbcb2f25f60 @@ -425,7 +424,7 @@ position: children: [] id: CHART-explore-1504-1 meta: - chartId: 399 + chartId: 461 height: 50 sliceName: Learner Performance Breakdown uuid: b73b0a5c-4861-402e-a39c-d73333a4d911 @@ -440,7 +439,7 @@ position: children: [] id: CHART-explore-1505-1 meta: - chartId: 776 + chartId: 101 height: 50 sliceName: Learner Performance Breakdown uuid: d414b144-467b-456b-b899-9cfb3579faba @@ -455,7 +454,7 @@ position: children: [] id: CHART-explore-1506-1 meta: - chartId: 1252 + chartId: 61 height: 50 sliceName: Learner Performance uuid: b518f341-9ed4-4c23-bb88-6c1d497aa260 @@ -470,7 +469,7 @@ position: children: [] id: CHART-explore-1507-1 meta: - chartId: 422 + chartId: 1461 height: 50 sliceName: Learner Performance uuid: b0e170ce-26f1-4452-a3c0-49b41fa6e529 @@ -485,7 +484,7 @@ position: children: [] id: CHART-explore-1508-1 meta: - chartId: 1407 + chartId: 641 height: 50 sliceName: Video Engagement uuid: 6b7610a7-41d0-4d14-be81-62a5a8f22d62 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/tag_filter.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/tag_filter.yaml index 99c8f06f1..b7824dbd2 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/tag_filter.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/tag_filter.yaml @@ -2,6 +2,18 @@ _file_name: tag_filter.yaml always_filter_main_dttm: false cache_timeout: null columns: +- advanced_data_type: null + column_name: course_key + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: String + verbose_name: Course Key - advanced_data_type: null column_name: id description: null @@ -38,18 +50,6 @@ columns: python_date_format: null type: String verbose_name: Tag -- advanced_data_type: null - column_name: course_key - description: null - expression: null - extra: {} - filterable: true - groupby: true - is_active: true - is_dttm: false - python_date_format: null - type: String - verbose_name: Course Key database_uuid: 21174b6c-4d40-4958-8161-d6c3cf5e77b6 default_endpoint: null description: null diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/watched_video_duration.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/watched_video_duration.yaml index d744a10bf..fcc0e4aae 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/watched_video_duration.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/watched_video_duration.yaml @@ -6,8 +6,7 @@ columns: column_name: rewatched_time description: null expression: null - extra: - warning_markdown: null + extra: {} filterable: true groupby: true is_active: true @@ -19,8 +18,7 @@ columns: column_name: watched_time description: null expression: null - extra: - warning_markdown: null + extra: {} filterable: true groupby: true is_active: true @@ -32,8 +30,7 @@ columns: column_name: course_name description: null expression: '' - extra: - warning_markdown: null + extra: {} filterable: true groupby: true is_active: true @@ -45,8 +42,7 @@ columns: column_name: video_count description: null expression: null - extra: - warning_markdown: null + extra: {} filterable: true groupby: true is_active: true @@ -58,8 +54,7 @@ columns: column_name: video_duration description: null expression: null - extra: - warning_markdown: null + extra: {} filterable: true groupby: true is_active: true @@ -71,8 +66,7 @@ columns: column_name: actor_id description: null expression: null - extra: - warning_markdown: null + extra: {} filterable: true groupby: true is_active: true @@ -84,8 +78,7 @@ columns: column_name: course_key description: null expression: null - extra: - warning_markdown: null + extra: {} filterable: true groupby: true is_active: true @@ -97,8 +90,7 @@ columns: column_name: course_run description: null expression: null - extra: - warning_markdown: null + extra: {} filterable: true groupby: true is_active: true @@ -110,8 +102,7 @@ columns: column_name: org description: null expression: null - extra: - warning_markdown: null + extra: {} filterable: true groupby: true is_active: true @@ -119,6 +110,18 @@ columns: python_date_format: null type: String verbose_name: Org +- advanced_data_type: null + column_name: video_count_watched + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: UInt64 + verbose_name: Video Count Watched database_uuid: 21174b6c-4d40-4958-8161-d6c3cf5e77b6 default_endpoint: null description: null @@ -130,7 +133,7 @@ metrics: - currency: null d3format: null description: null - expression: if(sum(video_duration) > 0,(sum(rewatched_time)/count(actor_id))/sum(video_duration),0) + expression: if(sum(video_duration) > 0,sum(rewatched_time)/sum(video_duration),0) extra: warning_markdown: '' metric_name: rewatched_percent @@ -140,7 +143,7 @@ metrics: - currency: null d3format: null description: null - expression: if(sum(watched_time)>0,(sum(watched_time)/count(actor_id))/sum(video_duration),0) + expression: if(sum(watched_time)>0,sum(watched_time)/sum(video_duration),0) extra: warning_markdown: '' metric_name: watched_percent @@ -157,6 +160,15 @@ metrics: metric_type: null verbose_name: Avg Video Length (seconds) warning_text: null +- currency: null + d3format: null + description: null + expression: sum(video_count_watched) + extra: {} + metric_name: num_videos_watched + metric_type: null + verbose_name: Number of Videos Watched + warning_text: null - currency: null d3format: null description: null diff --git a/tutoraspects/templates/aspects/jobs/init/clickhouse/init-clickhouse.sh b/tutoraspects/templates/aspects/jobs/init/clickhouse/init-clickhouse.sh index 92d1721ff..715a5a26e 100644 --- a/tutoraspects/templates/aspects/jobs/init/clickhouse/init-clickhouse.sh +++ b/tutoraspects/templates/aspects/jobs/init/clickhouse/init-clickhouse.sh @@ -48,7 +48,7 @@ GRANT {{ ON_CLUSTER }} SELECT ON {{ ASPECTS_XAPI_DATABASE }}.* TO '{{ ASPECTS_CL GRANT {{ ON_CLUSTER }} INSERT, SELECT ON {{ ASPECTS_EVENT_SINK_DATABASE }}.* TO '{{ ASPECTS_CLICKHOUSE_CMS_USER }}'; GRANT {{ ON_CLUSTER }} SELECT ON {{ ASPECTS_EVENT_SINK_DATABASE }}.* TO '{{ ASPECTS_CLICKHOUSE_REPORT_USER }}'; -GRANT {{ ON_CLUSTER }} CREATE TABLE, DROP TABLE, CREATE VIEW, DROP VIEW, SELECT, INSERT, UPDATE, DELETE ON {{ DBT_PROFILE_TARGET_DATABASE }}.* TO '{{ ASPECTS_CLICKHOUSE_REPORT_USER }}'; +GRANT {{ ON_CLUSTER }} CREATE TABLE, DROP TABLE, CREATE VIEW, DROP VIEW, SELECT, INSERT, UPDATE, DELETE, dictGet ON {{ DBT_PROFILE_TARGET_DATABASE }}.* TO '{{ ASPECTS_CLICKHOUSE_REPORT_USER }}'; GRANT {{ ON_CLUSTER }} SELECT, DROP TABLE, DROP VIEW ON {{ ASPECTS_XAPI_DATABASE }}.* TO '{{ ASPECTS_CLICKHOUSE_REPORT_USER }}'; GRANT {{ ON_CLUSTER }} INSERT, SELECT, DELETE ON {{ ASPECTS_EVENT_SINK_DATABASE }}.* TO '{{ ASPECTS_CLICKHOUSE_CMS_USER }}'; diff --git a/tutoraspects/templates/openedx-assets/queries/watched_video_duration.sql b/tutoraspects/templates/openedx-assets/queries/watched_video_duration.sql index 263bf950d..ca9284c7e 100644 --- a/tutoraspects/templates/openedx-assets/queries/watched_video_duration.sql +++ b/tutoraspects/templates/openedx-assets/queries/watched_video_duration.sql @@ -1,3 +1,24 @@ +{% raw -%} +with + course_keys as ( + select [] as course_key + {% if filter_values("course_name") != [] %} + union all + select array(course_key) as course_key + from {% endraw -%} {{ ASPECTS_EVENT_SINK_DATABASE }}.course_names {% raw -%} + where course_name in {{ filter_values("course_name") | where_in }} + {% endif %} + {% if filter_values("tag") != [] %} + union distinct + select array(course_key) as course_key + from + {% endraw -%} {{ DBT_PROFILE_TARGET_DATABASE }}.most_recent_course_tags {% raw -%} + where + tag + in (select replaceAll(arrayJoin({{ filter_values("tag") }}), '- ', '')) + {% endif %} + ) + {%- endraw %} select names.org as org, names.course_key as course_key, @@ -11,8 +32,9 @@ select from {{ DBT_PROFILE_TARGET_DATABASE }}.watched_video_duration( {% raw -%} - org_filter ={{ filter_values("org") }}, - course_name_filter ={{ filter_values("course_name") }} + org_filter = coalesce({{ filter_values("org") }}, []), + course_key_filter + = coalesce((select array_concat_agg(course_key) from course_keys), []) {%- endraw %} ) as a left join diff --git a/tutoraspects/translations/translate_utils.py b/tutoraspects/translations/translate_utils.py index fe1bd6719..ad3db81e7 100644 --- a/tutoraspects/translations/translate_utils.py +++ b/tutoraspects/translations/translate_utils.py @@ -82,7 +82,7 @@ class DashboardAsset(TranslatableAsset): "metadata.native_filter_configuration.name", "metadata.native_filter_configuration.description", "position.*.meta.text", - "position.*.meta.code", + # "position.*.meta.code", "position.*.meta.sliceNameOverride", ]