From 85e3f208483a1b64a6d28a9ddf747b5cb1e164be Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Fri, 15 Mar 2024 08:57:17 -0500 Subject: [PATCH 01/11] fix: only filter datasets on dashboards --- .../templates/openedx-assets/queries/fact_course_grades.sql | 4 ++-- .../queries/fact_learner_problem_course_summary.sql | 2 +- .../openedx-assets/queries/fact_learner_problem_summary.sql | 2 +- .../templates/openedx-assets/queries/fact_problem_grades.sql | 2 +- .../openedx-assets/queries/fact_problem_responses.sql | 2 +- .../openedx-assets/queries/fact_transcript_usage.sql | 4 ++-- .../templates/openedx-assets/queries/fact_video_plays.sql | 2 +- .../openedx-assets/queries/fact_watched_video_segments.sql | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tutoraspects/templates/openedx-assets/queries/fact_course_grades.sql b/tutoraspects/templates/openedx-assets/queries/fact_course_grades.sql index 92ae63466..6ce647ca2 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_course_grades.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_course_grades.sql @@ -3,10 +3,10 @@ with grades as ( from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_grades where grade_type = 'course' {% raw %} - {% if filter_values('course_name') != [] %} + {% if get_filters('course_name', remove_filter=True) != [] && filter_values('course_name') != [] %} and entity_name in {{ filter_values('course_name', remove_filter=True) | where_in }} {% else %} - and 0=1 + and 1=0 {% endif %} {% endraw %} {% include 'openedx-assets/queries/common_filters.sql' %} diff --git a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql index bf4468a0c..62d4cc807 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql @@ -134,7 +134,7 @@ SELECT FROM summary WHERE {% raw %} - {% if filter_values('course_name') != [] %} + {% if get_filters('course_name', remove_filter=True) != [] && filter_values('course_name') != [] %} course_name in {{ filter_values('course_name') | where_in }} {% else %} 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql index e55e91418..c37eac7e0 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql @@ -118,7 +118,7 @@ SELECT FROM summary where {% raw %} - {% if filter_values('problem_name_with_location') != [] %} + {% if get_filters('problem_name_with_location', remove_filter=True) != [] && filter_values('problem_name_with_location') != [] %} problem_name_with_location in {{ filter_values('problem_name_with_location') | where_in }} {% else %} 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql b/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql index e401148dd..27a6ceff9 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql @@ -5,7 +5,7 @@ with grades as ( grade_type = 'problem' {% raw %} - {% if filter_values('problem_name_with_location') != [] %} + {% if get_filters('problem_name_with_location', remove_filter=True) != [] && filter_values('problem_name_with_location') != [] %} and entity_name_with_location in {{ filter_values('problem_name_with_location', remove_filter=True) | where_in }} {% else %} and 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql b/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql index 0962f17a9..61ab2c72a 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql @@ -25,7 +25,7 @@ from problem_responses where {% raw %} - {% if filter_values('problem_name_with_location') != [] %} + {% if get_filters('problem_name_with_location', remove_filter=True) != [] && filter_values('problem_name_with_location') != [] %} problem_name_with_location in {{ filter_values('problem_name_with_location') | where_in }} {% else %} 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql b/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql index bba2968c7..6a0466f7f 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql @@ -3,10 +3,10 @@ select * from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_transcript_usage where {% raw %} - {% if filter_values('course_name') != [] %} + {% if get_filters('course_name', remove_filter=True) != [] && filter_values('course_name') != [] %} course_name in {{ filter_values('course_name') | where_in }} {% else %} - 0=1 + 1=0 {% endif %} {% endraw %} {% include 'openedx-assets/queries/common_filters.sql' %} diff --git a/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql b/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql index 4bdc18c9b..d9cf1acd9 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql @@ -3,7 +3,7 @@ select * from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_video_plays where {% raw %} - {% if filter_values('course_name') != [] %} + {% if get_filters('course_name', remove_filter=True) != [] && filter_values('course_name') != [] %} course_name in {{ filter_values('course_name') | where_in }} {% else %} 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql b/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql index bb64b1301..66b30562d 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql @@ -75,7 +75,7 @@ select from enriched_segments where {% raw %} - {% if filter_values('video_name_with_location') != [] %} + {% if get_filters('video_name_with_location', remove_filter=True) != [] && filter_values('video_name_with_location') != [] %} video_name_with_location in {{ filter_values('video_name_with_location') | where_in }} {% else %} 1=0 From 2f62e5d8157160504dd9f91c5cb3b18b8fe182ba Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Fri, 15 Mar 2024 08:58:58 -0500 Subject: [PATCH 02/11] fix: restore query context for charts --- tutoraspects/asset_command_helpers.py | 3 +- .../charts/Course_Grade_Distribution.yaml | 12 +++++-- .../Currently_Enrolled_Learners_Per_Day.yaml | 11 +++++-- .../assets/charts/Distinct_forum_users.yaml | 12 +++++-- .../charts/Distribution_Of_Attempts.yaml | 2 ++ ...tribution_Of_Hints_Per_Correct_Answer.yaml | 2 ++ .../Distribution_Of_Problem_Grades.yaml | 12 +++++-- .../charts/Distribution_Of_Responses.yaml | 12 +++++-- .../charts/Enrollment_Events_Per_Day.yaml | 18 +++++++--- .../Enrollments_By_Enrollment_Mode.yaml | 7 ++++ .../assets/charts/Posts_per_user.yaml | 4 +-- .../assets/charts/Responses_Per_Problem.yaml | 3 ++ .../Transcripts_Captions_Per_Video.yaml | 12 +++++-- .../assets/charts/Watched_Video_Segments.yaml | 12 +++++-- .../assets/charts/Watches_Per_Video.yaml | 12 +++++-- .../dashboards/Instructor_Dashboard.yaml | 33 +++++++++---------- 16 files changed, 124 insertions(+), 43 deletions(-) diff --git a/tutoraspects/asset_command_helpers.py b/tutoraspects/asset_command_helpers.py index af26fce26..24ef35519 100644 --- a/tutoraspects/asset_command_helpers.py +++ b/tutoraspects/asset_command_helpers.py @@ -157,12 +157,11 @@ class ChartAsset(Asset): path = "charts" omitted_vars = [ - "query_context", "params.dashboards", "params.datasource", "params.slice_id", ] - raw_vars = ["sqlExpression"] + raw_vars = ["sqlExpression", "query_context"] class DashboardAsset(Asset): diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Grade_Distribution.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Grade_Distribution.yaml index 74e2acff5..679a0c2a6 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Grade_Distribution.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Grade_Distribution.yaml @@ -6,12 +6,16 @@ dataset_uuid: d777bd95-2110-46db-a1c4-8358be81a85a description: The distribution of grades for a course, out of 100%. Grades are grouped in ranges of 10%. params: - adhoc_filters: [] + adhoc_filters: + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: emission_time bottom_margin: auto color_scheme: supersetColors columns: [] extra_form_data: {} - granularity_sqla: emission_time groupby: - grade_bucket metrics: @@ -29,6 +33,10 @@ params: - null y_axis_format: SMART_NUMBER y_axis_label: Number Of Students +query_context: '{"datasource":{"id":236,"type":"table"},"force":false,"queries":[{"filters":[{"col":"emission_time","op":"TEMPORAL_RANGE","val":"No + filter"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":["grade_bucket"],"metrics":["students"],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"236__table","viz_type":"dist_bar","slice_id":298,"metrics":["students"],"adhoc_filters":[{"clause":"WHERE","subject":"emission_time","operator":"TEMPORAL_RANGE","comparator":"No + filter","expressionType":"SIMPLE"}],"groupby":["grade_bucket"],"columns":[],"row_limit":10000,"order_desc":true,"color_scheme":"supersetColors","show_legend":false,"rich_tooltip":true,"order_bars":true,"y_axis_format":"SMART_NUMBER","y_axis_label":"Number + Of Students","y_axis_bounds":[null,null],"x_axis_label":"Course Grade (out of 100%)","bottom_margin":"auto","x_ticks_layout":"auto","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Course Grade Distribution uuid: f9adbc85-1f50-4c04-ace3-31ba7390de5e version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Currently_Enrolled_Learners_Per_Day.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Currently_Enrolled_Learners_Per_Day.yaml index 9e302b841..0695c2526 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Currently_Enrolled_Learners_Per_Day.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Currently_Enrolled_Learners_Per_Day.yaml @@ -19,13 +19,17 @@ params: operatorId: EQUALS sqlExpression: null subject: enrollment_status + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: enrollment_status_date color_picker: a: 1 b: 135 g: 122 r: 0 extra_form_data: {} - granularity_sqla: enrollment_status_date header_font_size: 0.4 metric: count rolling_type: None @@ -34,9 +38,12 @@ params: subheader_font_size: 0.15 time_format: smart_date time_grain_sqla: P1D - time_range: No filter viz_type: big_number + x_axis: enrollment_status_date y_axis_format: SMART_NUMBER +query_context: '{"datasource":{"id":188,"type":"table"},"force":false,"queries":[{"filters":[{"col":"enrollment_status","op":"==","val":"registered"},{"col":"enrollment_status_date","op":"TEMPORAL_RANGE","val":"No + filter"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":[{"timeGrain":"P1D","columnType":"BASE_AXIS","sqlExpression":"enrollment_status_date","label":"enrollment_status_date","expressionType":"SQL"}],"metrics":["count"],"annotation_layers":[],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"post_processing":[{"operation":"pivot","options":{"index":["enrollment_status_date"],"columns":[],"aggregates":{"count":{"operator":"mean"}},"drop_missing_columns":true}},{"operation":"flatten"}]}],"form_data":{"datasource":"188__table","viz_type":"big_number","slice_id":344,"x_axis":"enrollment_status_date","time_grain_sqla":"P1D","metric":"count","adhoc_filters":[{"clause":"WHERE","comparator":"registered","expressionType":"SIMPLE","filterOptionName":"filter_hcnm4t7piq6_hfbtt65nqqs","isExtra":false,"isNew":false,"operator":"==","operatorId":"EQUALS","sqlExpression":null,"subject":"enrollment_status"},{"clause":"WHERE","subject":"enrollment_status_date","operator":"TEMPORAL_RANGE","comparator":"No + filter","expressionType":"SIMPLE"}],"show_trend_line":true,"start_y_axis_at_zero":true,"color_picker":{"a":1,"b":135,"g":122,"r":0},"header_font_size":0.4,"subheader_font_size":0.15,"y_axis_format":"SMART_NUMBER","time_format":"smart_date","rolling_type":"None","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Currently Enrolled Learners Per Day uuid: ed2fe731-6544-422f-bc55-42f399f48b2c version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distinct_forum_users.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distinct_forum_users.yaml index c488832d5..da88796b5 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distinct_forum_users.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distinct_forum_users.yaml @@ -5,9 +5,13 @@ certified_by: null dataset_uuid: 39d1e786-c0c8-4c56-81c8-56fb0df88001 description: null params: - adhoc_filters: [] + adhoc_filters: + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: emission_time extra_form_data: {} - granularity_sqla: emission_time header_font_size: 0.4 metric: aggregate: COUNT_DISTINCT @@ -36,9 +40,11 @@ params: sqlExpression: null subheader_font_size: 0.15 time_format: smart_date - time_range: No filter viz_type: big_number_total y_axis_format: SMART_NUMBER +query_context: '{"datasource":{"id":242,"type":"table"},"force":false,"queries":[{"filters":[{"col":"emission_time","op":"TEMPORAL_RANGE","val":"No + filter"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":[],"metrics":[{"aggregate":"COUNT_DISTINCT","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"actor_id","description":null,"expression":null,"filterable":true,"groupby":true,"id":445,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"String","type_generic":1,"verbose_name":null,"warning_markdown":null},"datasourceWarning":false,"expressionType":"SIMPLE","hasCustomLabel":false,"label":"COUNT_DISTINCT(actor_id)","optionName":"metric_5y4uvwa13v4_f12i3twecs6","sqlExpression":null}],"annotation_layers":[],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"242__table","viz_type":"big_number_total","slice_id":395,"metric":{"aggregate":"COUNT_DISTINCT","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"actor_id","description":null,"expression":null,"filterable":true,"groupby":true,"id":445,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"String","type_generic":1,"verbose_name":null,"warning_markdown":null},"datasourceWarning":false,"expressionType":"SIMPLE","hasCustomLabel":false,"label":"COUNT_DISTINCT(actor_id)","optionName":"metric_5y4uvwa13v4_f12i3twecs6","sqlExpression":null},"adhoc_filters":[{"clause":"WHERE","subject":"emission_time","operator":"TEMPORAL_RANGE","comparator":"No + filter","expressionType":"SIMPLE"}],"header_font_size":0.4,"subheader_font_size":0.15,"y_axis_format":"SMART_NUMBER","time_format":"smart_date","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Distinct forum users uuid: feb323ad-c819-49ca-a336-584bd9ff1a2e version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Attempts.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Attempts.yaml index 35b0cad7c..51841b393 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Attempts.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Attempts.yaml @@ -38,6 +38,8 @@ params: - null y_axis_format: SMART_NUMBER y_axis_label: Students +query_context: '{"datasource":{"id":209,"type":"table"},"force":false,"queries":[{"filters":[{"col":"success","op":"==","val":"true"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":["attempts"],"metrics":["students"],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"209__table","viz_type":"dist_bar","slice_id":331,"metrics":["students"],"adhoc_filters":[{"clause":"WHERE","comparator":"true","expressionType":"SIMPLE","filterOptionName":"filter_0fpmws3t1h6a_md2ud9xse7m","isExtra":false,"isNew":false,"operator":"==","operatorId":"EQUALS","sqlExpression":null,"subject":"success"}],"groupby":["attempts"],"columns":[],"row_limit":10000,"order_desc":true,"color_scheme":"supersetColors","show_legend":false,"rich_tooltip":true,"order_bars":true,"y_axis_format":"SMART_NUMBER","y_axis_label":"Students","y_axis_bounds":[null,null],"x_axis_label":"Number + Of Attempts To Find Correct Answer","bottom_margin":"auto","x_ticks_layout":"auto","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Distribution Of Attempts uuid: db90930f-f16e-4c32-8050-0e4abae28f4c version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Hints_Per_Correct_Answer.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Hints_Per_Correct_Answer.yaml index 7de1ed6ef..ba11490e6 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Hints_Per_Correct_Answer.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Hints_Per_Correct_Answer.yaml @@ -29,6 +29,8 @@ params: - null y_axis_format: SMART_NUMBER y_axis_label: Students +query_context: '{"datasource":{"id":191,"type":"table"},"force":false,"queries":[{"filters":[],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":["total_hints"],"metrics":["students"],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"191__table","viz_type":"dist_bar","slice_id":307,"metrics":["students"],"adhoc_filters":[],"groupby":["total_hints"],"columns":[],"row_limit":10000,"order_desc":true,"color_scheme":"supersetColors","show_legend":false,"rich_tooltip":true,"y_axis_format":"SMART_NUMBER","y_axis_label":"Students","y_axis_bounds":[null,null],"x_axis_label":"Hints + / Answer Displayed Before Correct Answer Chosen","bottom_margin":"auto","x_ticks_layout":"auto","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Distribution Of Hints Per Correct Answer uuid: ee94be4c-6fdd-4295-b43c-40890d6c549d version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Problem_Grades.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Problem_Grades.yaml index aa0aac2ea..7e32dd219 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Problem_Grades.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Problem_Grades.yaml @@ -7,12 +7,16 @@ description: 'This chart shows the number of students who scored within a certai percentage of points for this problem. For problems that are pass/fail it will only show the lowest and highest percentage ranges. ' params: - adhoc_filters: [] + adhoc_filters: + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: emission_time bottom_margin: auto color_scheme: supersetColors columns: [] extra_form_data: {} - granularity_sqla: emission_time groupby: - grade_bucket metrics: @@ -30,6 +34,10 @@ params: - null y_axis_format: SMART_NUMBER y_axis_label: Students +query_context: '{"datasource":{"id":215,"type":"table"},"force":false,"queries":[{"filters":[{"col":"emission_time","op":"TEMPORAL_RANGE","val":"No + filter"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":["grade_bucket"],"metrics":["students"],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"215__table","viz_type":"dist_bar","slice_id":382,"metrics":["students"],"adhoc_filters":[{"clause":"WHERE","subject":"emission_time","operator":"TEMPORAL_RANGE","comparator":"No + filter","expressionType":"SIMPLE"}],"groupby":["grade_bucket"],"columns":[],"row_limit":10000,"order_desc":true,"color_scheme":"supersetColors","show_legend":false,"rich_tooltip":true,"order_bars":true,"y_axis_format":"SMART_NUMBER","y_axis_label":"Students","y_axis_bounds":[null,null],"x_axis_label":"Percentage + Grade","bottom_margin":"auto","x_ticks_layout":"auto","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Distribution Of Problem Grades uuid: 4f7e3606-f5de-4643-97c0-bbb6340a3df2 version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Responses.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Responses.yaml index 4c8430535..1a3702359 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Responses.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Responses.yaml @@ -7,12 +7,16 @@ description: This chart shows how often an answer or combination of answers (for is selected by learners. Some problems allow learners to submit a response more than once, this chart will include all of the responses in that case. params: - adhoc_filters: [] + adhoc_filters: + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: emission_time bottom_margin: auto color_scheme: supersetColors columns: [] extra_form_data: {} - granularity_sqla: emission_time groupby: - responses metrics: @@ -29,6 +33,10 @@ params: - null y_axis_format: SMART_NUMBER y_axis_label: Responses +query_context: '{"datasource":{"id":225,"type":"table"},"force":false,"queries":[{"filters":[{"col":"emission_time","op":"TEMPORAL_RANGE","val":"No + filter"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":["responses"],"metrics":["total_responses"],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"225__table","viz_type":"dist_bar","slice_id":352,"metrics":["total_responses"],"adhoc_filters":[{"clause":"WHERE","subject":"emission_time","operator":"TEMPORAL_RANGE","comparator":"No + filter","expressionType":"SIMPLE"}],"groupby":["responses"],"columns":[],"row_limit":10000,"order_desc":true,"color_scheme":"supersetColors","show_legend":false,"rich_tooltip":true,"y_axis_format":"SMART_NUMBER","y_axis_label":"Responses","y_axis_bounds":[null,null],"x_axis_label":"Answers + Chosen","bottom_margin":"auto","x_ticks_layout":"auto","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Distribution Of Responses uuid: f1651c44-a8f4-4b44-ad49-962823009319 version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollment_Events_Per_Day.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollment_Events_Per_Day.yaml index a8063f0b9..36c794a15 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollment_Events_Per_Day.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollment_Events_Per_Day.yaml @@ -7,14 +7,18 @@ description: A count of the number of enrollments and un-enrollments per day. Le can enroll and unenroll multiple times, in this chart each individual enrollment and unenrollment will be counted. params: - adhoc_filters: [] + adhoc_filters: + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: emission_time annotation_layers: [] color_scheme: supersetColors comparison_type: values extra_form_data: {} forecastInterval: 0.8 forecastPeriods: 10 - granularity_sqla: emission_time groupby: - expressionType: SQL label: Enrollment Status @@ -32,13 +36,17 @@ params: row_limit: 10000 show_empty_columns: true show_legend: false - stack: true + sort_series_type: sum + stack: null time_grain_sqla: P1D - time_range: No filter tooltipTimeFormat: smart_date truncate_metric: true viz_type: echarts_timeseries_bar xAxisLabelRotation: 45 + x_axis: emission_time + x_axis_sort_asc: true + x_axis_sort_series: name + x_axis_sort_series_ascending: true x_axis_time_format: '%Y-%m-%d' x_axis_title_margin: 15 y_axis_bounds: @@ -48,6 +56,8 @@ params: y_axis_title: Number Of Events y_axis_title_margin: 15 y_axis_title_position: Left +query_context: |- + {% raw %}{"datasource":{"id":224,"type":"table"},"force":false,"queries":[{"filters":[{"col":"emission_time","op":"TEMPORAL_RANGE","val":"No filter"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":[{"timeGrain":"P1D","columnType":"BASE_AXIS","sqlExpression":"emission_time","label":"emission_time","expressionType":"SQL"},{"expressionType":"SQL","label":"Enrollment Status","sqlExpression":"{{translate_column('enrollment_status')}}"}],"metrics":["count"],"orderby":[["count",false]],"annotation_layers":[],"row_limit":10000,"series_columns":[{"expressionType":"SQL","label":"Enrollment Status","sqlExpression":"{{translate_column('enrollment_status')}}"}],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"time_offsets":[],"post_processing":[{"operation":"pivot","options":{"index":["emission_time"],"columns":["Enrollment Status"],"aggregates":{"count":{"operator":"mean"}},"drop_missing_columns":false}},{"operation":"rename","options":{"columns":{"count":null},"level":0,"inplace":true}},{"operation":"flatten"}]}],"form_data":{"datasource":"224__table","viz_type":"echarts_timeseries_bar","slice_id":332,"x_axis":"emission_time","time_grain_sqla":"P1D","x_axis_sort_asc":true,"x_axis_sort_series":"name","x_axis_sort_series_ascending":true,"metrics":["count"],"groupby":[{"expressionType":"SQL","label":"Enrollment Status","sqlExpression":"{{translate_column('enrollment_status')}}"}],"adhoc_filters":[{"clause":"WHERE","subject":"emission_time","operator":"TEMPORAL_RANGE","comparator":"No filter","expressionType":"SIMPLE"}],"order_desc":true,"row_limit":10000,"truncate_metric":true,"show_empty_columns":true,"comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"orientation":"vertical","x_axis_title_margin":15,"y_axis_title":"Number Of Events","y_axis_title_margin":15,"y_axis_title_position":"Left","sort_series_type":"sum","color_scheme":"supersetColors","stack":null,"only_total":true,"show_legend":false,"legendType":"scroll","legendOrientation":"top","legendMargin":null,"x_axis_time_format":"%Y-%m-%d","xAxisLabelRotation":45,"y_axis_format":"SMART_NUMBER","y_axis_bounds":[null,null],"rich_tooltip":true,"tooltipTimeFormat":"smart_date","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}{% endraw %} slice_name: Enrollment Events Per Day uuid: bb1147cc-b7bc-44b7-b06a-79b0db6626aa version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollments_By_Enrollment_Mode.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollments_By_Enrollment_Mode.yaml index 058e24c5e..cd34a4bb7 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollments_By_Enrollment_Mode.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollments_By_Enrollment_Mode.yaml @@ -18,6 +18,11 @@ params: operatorId: EQUALS sqlExpression: null subject: enrollment_status + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: emission_time color_scheme: supersetColors date_format: smart_date extra_form_data: {} @@ -40,6 +45,8 @@ params: show_legend: true sort_by_metric: true viz_type: pie +query_context: |- + {% raw %}{"datasource":{"id":224,"type":"table"},"force":false,"queries":[{"filters":[{"col":"enrollment_status","op":"==","val":"registered"},{"col":"emission_time","op":"TEMPORAL_RANGE","val":"No filter"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":[{"expressionType":"SQL","label":"Enrollment Mode","sqlExpression":"{{translate_column('enrollment_mode')}}"}],"metrics":["count"],"orderby":[["count",false]],"annotation_layers":[],"row_limit":100,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"224__table","viz_type":"pie","slice_id":411,"groupby":[{"expressionType":"SQL","label":"Enrollment Mode","sqlExpression":"{{translate_column('enrollment_mode')}}"}],"metric":"count","adhoc_filters":[{"clause":"WHERE","comparator":"registered","expressionType":"SIMPLE","filterOptionName":"filter_itrvc9c0jxb_g4qd4b18na4","isExtra":false,"isNew":false,"operator":"==","operatorId":"EQUALS","sqlExpression":null,"subject":"enrollment_status"},{"clause":"WHERE","subject":"emission_time","operator":"TEMPORAL_RANGE","comparator":"No filter","expressionType":"SIMPLE"}],"row_limit":100,"sort_by_metric":true,"color_scheme":"supersetColors","show_labels_threshold":5,"show_legend":true,"legendType":"scroll","legendOrientation":"top","label_type":"key","number_format":"SMART_NUMBER","date_format":"smart_date","show_labels":true,"labels_outside":true,"outerRadius":70,"innerRadius":30,"extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}{% endraw %} slice_name: Enrollments By Enrollment Mode uuid: 05ed7102-5464-4e2f-86ae-31700b787cc3 version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Posts_per_user.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Posts_per_user.yaml index 76d1bdd0a..3382ff2cf 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Posts_per_user.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Posts_per_user.yaml @@ -10,7 +10,6 @@ params: color_scheme: supersetColors columns: [] extra_form_data: {} - granularity_sqla: null groupby: - num_posts metrics: @@ -19,7 +18,6 @@ params: rich_tooltip: true row_limit: 10000 show_legend: true - time_range: No filter timeseries_limit_metric: aggregate: null column: null @@ -37,6 +35,8 @@ params: - null y_axis_format: SMART_NUMBER y_axis_label: Number of users +query_context: '{"datasource":{"id":235,"type":"table"},"force":false,"queries":[{"filters":[],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":["num_posts"],"metrics":["users"],"annotation_layers":[],"row_limit":10000,"series_limit":0,"series_limit_metric":{"aggregate":null,"column":null,"datasourceWarning":false,"expressionType":"SQL","hasCustomLabel":false,"label":"num_posts","optionName":"metric_tdazpzouidq_76qbq0kf86v","sqlExpression":"num_posts"},"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"235__table","viz_type":"dist_bar","slice_id":310,"metrics":["users"],"adhoc_filters":[],"groupby":["num_posts"],"columns":[],"row_limit":10000,"timeseries_limit_metric":{"aggregate":null,"column":null,"datasourceWarning":false,"expressionType":"SQL","hasCustomLabel":false,"label":"num_posts","optionName":"metric_tdazpzouidq_76qbq0kf86v","sqlExpression":"num_posts"},"order_desc":true,"color_scheme":"supersetColors","show_legend":true,"rich_tooltip":true,"y_axis_format":"SMART_NUMBER","y_axis_label":"Number + of users","y_axis_bounds":[null,null],"x_axis_label":"Posts per user","bottom_margin":"auto","x_ticks_layout":"auto","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Posts per user uuid: bc191ce7-f39d-48db-86a9-d19949f4211d version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Responses_Per_Problem.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Responses_Per_Problem.yaml index e56e2dd64..d16ddf9ea 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Responses_Per_Problem.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Responses_Per_Problem.yaml @@ -55,6 +55,9 @@ params: - null y_axis_format: SMART_NUMBER y_axis_label: Number Of Students +query_context: '{"datasource":{"id":222,"type":"table"},"force":false,"queries":[{"filters":[],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":["problem_name_with_location","success"],"metrics":[{"aggregate":"COUNT_DISTINCT","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"actor_id","description":null,"expression":null,"filterable":true,"groupby":true,"id":301,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"String","type_generic":1,"verbose_name":null,"warning_markdown":null},"expressionType":"SIMPLE","hasCustomLabel":true,"isNew":false,"label":"Learners","optionName":"metric_g889221mh0r_lf203igqxgg","sqlExpression":null}],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"222__table","viz_type":"dist_bar","slice_id":339,"metrics":[{"aggregate":"COUNT_DISTINCT","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"actor_id","description":null,"expression":null,"filterable":true,"groupby":true,"id":301,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"String","type_generic":1,"verbose_name":null,"warning_markdown":null},"expressionType":"SIMPLE","hasCustomLabel":true,"isNew":false,"label":"Learners","optionName":"metric_g889221mh0r_lf203igqxgg","sqlExpression":null}],"adhoc_filters":[],"groupby":["problem_name_with_location"],"columns":["success"],"row_limit":10000,"order_desc":true,"color_scheme":"supersetColors","show_legend":false,"rich_tooltip":true,"bar_stacked":true,"order_bars":false,"y_axis_format":"SMART_NUMBER","y_axis_label":"Number + Of Students","show_controls":false,"y_axis_bounds":[null,null],"x_axis_label":"Question + Title","bottom_margin":"auto","x_ticks_layout":"auto","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Responses Per Problem uuid: a3e79162-4ace-4349-ab34-89aa60ae75ed version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Transcripts_Captions_Per_Video.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Transcripts_Captions_Per_Video.yaml index 3a2fadb8b..0c26e6069 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Transcripts_Captions_Per_Video.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Transcripts_Captions_Per_Video.yaml @@ -6,12 +6,16 @@ dataset_uuid: a96a4b13-a429-442d-83ca-5d6f94010909 description: |- This chart helps understand how many learners are using the video's transcripts or closed captions. If the video has not transcripts or captions the chart will be empty. params: - adhoc_filters: [] + adhoc_filters: + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: emission_time bottom_margin: auto color_scheme: supersetColors columns: [] extra_form_data: {} - granularity_sqla: emission_time groupby: - video_name_with_location metrics: @@ -29,6 +33,10 @@ params: - null y_axis_format: SMART_NUMBER y_axis_label: Number of Users / Uses +query_context: '{"datasource":{"id":220,"type":"table"},"force":false,"queries":[{"filters":[{"col":"emission_time","op":"TEMPORAL_RANGE","val":"No + filter"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":["video_name_with_location"],"metrics":["total_transcript_usage","unique_transcript_users"],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"220__table","viz_type":"dist_bar","slice_id":410,"metrics":["total_transcript_usage","unique_transcript_users"],"adhoc_filters":[{"clause":"WHERE","subject":"emission_time","operator":"TEMPORAL_RANGE","comparator":"No + filter","expressionType":"SIMPLE"}],"groupby":["video_name_with_location"],"columns":[],"row_limit":10000,"order_desc":true,"color_scheme":"supersetColors","show_legend":true,"rich_tooltip":true,"y_axis_format":"SMART_NUMBER","y_axis_label":"Number + of Users / Uses","y_axis_bounds":[null,null],"x_axis_label":"Video Title","bottom_margin":"auto","x_ticks_layout":"auto","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Transcripts / Captions Per Video uuid: 6b830def-f3ca-4b4c-9455-7a7b7354bce8 version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watched_Video_Segments.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watched_Video_Segments.yaml index 727bb7760..87d2df9d2 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watched_Video_Segments.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watched_Video_Segments.yaml @@ -6,13 +6,17 @@ dataset_uuid: c2c391b3-3403-4f05-bc0b-3de53bd366ec description: This chart shows which parts of a video are most watched, and which are most re-watched. Each segment represents 5 seconds of the video. params: - adhoc_filters: [] + adhoc_filters: + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: started_at bar_stacked: true bottom_margin: auto color_scheme: supersetColors columns: [] extra_form_data: {} - granularity_sqla: started_at groupby: - segment_start metrics: @@ -31,6 +35,10 @@ params: - null y_axis_format: SMART_NUMBER y_axis_label: Number Of Viewers / Views +query_context: '{"datasource":{"id":183,"type":"table"},"force":false,"queries":[{"filters":[{"col":"started_at","op":"TEMPORAL_RANGE","val":"No + filter"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":["segment_start"],"metrics":["unique_viewers","repeat_views"],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"183__table","viz_type":"dist_bar","slice_id":371,"metrics":["unique_viewers","repeat_views"],"adhoc_filters":[{"clause":"WHERE","subject":"started_at","operator":"TEMPORAL_RANGE","comparator":"No + filter","expressionType":"SIMPLE"}],"groupby":["segment_start"],"columns":[],"row_limit":10000,"order_desc":true,"color_scheme":"supersetColors","show_legend":true,"rich_tooltip":true,"bar_stacked":true,"order_bars":true,"y_axis_format":"SMART_NUMBER","y_axis_label":"Number + Of Viewers / Views","y_axis_bounds":[null,null],"x_axis_label":"Seconds Of Video","bottom_margin":"auto","x_ticks_layout":"auto","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Watched Video Segments uuid: 2985a9db-c338-4008-af52-2930b81ee2e5 version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watches_Per_Video.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watches_Per_Video.yaml index 82da00741..c847530c1 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watches_Per_Video.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watches_Per_Video.yaml @@ -7,12 +7,16 @@ description: This chart shows how many unique learners have watched each video, how many repeat views each has gotten. If a video has never been played it will not appear in this chart. params: - adhoc_filters: [] + adhoc_filters: + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: emission_time bottom_margin: auto color_scheme: supersetColors columns: [] extra_form_data: {} - granularity_sqla: emission_time groupby: - video_name_with_location metrics: @@ -30,6 +34,10 @@ params: - null y_axis_format: SMART_NUMBER y_axis_label: Number of Views / Viewers +query_context: '{"datasource":{"id":214,"type":"table"},"force":false,"queries":[{"filters":[{"col":"emission_time","op":"TEMPORAL_RANGE","val":"No + filter"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":["video_name_with_location"],"metrics":["total_plays","unique_watchers"],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"214__table","viz_type":"dist_bar","slice_id":359,"metrics":["total_plays","unique_watchers"],"adhoc_filters":[{"clause":"WHERE","subject":"emission_time","operator":"TEMPORAL_RANGE","comparator":"No + filter","expressionType":"SIMPLE"}],"groupby":["video_name_with_location"],"columns":[],"row_limit":10000,"order_desc":true,"color_scheme":"supersetColors","show_legend":true,"rich_tooltip":true,"y_axis_format":"SMART_NUMBER","y_axis_label":"Number + of Views / Viewers","y_axis_bounds":[null,null],"x_axis_label":"Video Title","bottom_margin":"auto","x_ticks_layout":"auto","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Watches Per Video uuid: 829c1d5b-2844-4115-876a-34ad3b3cad64 version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Instructor_Dashboard.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Instructor_Dashboard.yaml index b74a7d7f2..60254b554 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Instructor_Dashboard.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Instructor_Dashboard.yaml @@ -307,15 +307,14 @@ metadata: datasetUuid: 417b2035-8fa1-4c60-a405-4b1947c3c966 type: NATIVE_FILTER refresh_frequency: 0 - shared_label_colors: - count: '#454E7C' + shared_label_colors: {} timed_refresh_immune_slices: [] position: CHART-AZZnl_lpMv: children: [] id: CHART-AZZnl_lpMv meta: - chartId: 13 + chartId: 359 height: 50 sliceName: Watches Per Video uuid: 829c1d5b-2844-4115-876a-34ad3b3cad64 @@ -330,7 +329,7 @@ position: children: [] id: CHART-GFCO8s2cxv meta: - chartId: 25 + chartId: 352 height: 50 sliceName: Distribution Of Responses uuid: f1651c44-a8f4-4b44-ad49-962823009319 @@ -345,7 +344,7 @@ position: children: [] id: CHART-GJJ8VYQ03v meta: - chartId: 55 + chartId: 310 height: 50 sliceName: Posts per user uuid: bc191ce7-f39d-48db-86a9-d19949f4211d @@ -360,7 +359,7 @@ position: children: [] id: CHART-Jr-gNVms2Q meta: - chartId: 34 + chartId: 332 height: 50 sliceName: Enrollment Events Per Day uuid: bb1147cc-b7bc-44b7-b06a-79b0db6626aa @@ -375,7 +374,7 @@ position: children: [] id: CHART-OMy4wjRBWt meta: - chartId: 45 + chartId: 371 height: 50 sliceName: Watched Video Segments uuid: 2985a9db-c338-4008-af52-2930b81ee2e5 @@ -390,7 +389,7 @@ position: children: [] id: CHART-RTO33WE9FH meta: - chartId: 59 + chartId: 339 height: 50 sliceName: Responses Per Problem uuid: a3e79162-4ace-4349-ab34-89aa60ae75ed @@ -405,7 +404,7 @@ position: children: [] id: CHART-Tej2oLPBAl meta: - chartId: 58 + chartId: 410 height: 50 sliceName: Transcripts / Captions Per Video uuid: 6b830def-f3ca-4b4c-9455-7a7b7354bce8 @@ -420,7 +419,7 @@ position: children: [] id: CHART-evjVO-ZSSd meta: - chartId: 27 + chartId: 411 height: 50 sliceName: Enrollments By Enrollment Mode uuid: 05ed7102-5464-4e2f-86ae-31700b787cc3 @@ -435,7 +434,7 @@ position: children: [] id: CHART-lTr8DL3XuI meta: - chartId: 57 + chartId: 307 height: 50 sliceName: Distribution Of Hints Per Correct Answer uuid: ee94be4c-6fdd-4295-b43c-40890d6c549d @@ -450,7 +449,7 @@ position: children: [] id: CHART-o56v9yEe2I meta: - chartId: 11 + chartId: 382 height: 50 sliceName: Distribution Of Problem Grades uuid: 4f7e3606-f5de-4643-97c0-bbb6340a3df2 @@ -465,7 +464,7 @@ position: children: [] id: CHART-qG1WaGKl_b meta: - chartId: 26 + chartId: 395 height: 50 sliceName: Distinct forum users uuid: feb323ad-c819-49ca-a336-584bd9ff1a2e @@ -480,7 +479,7 @@ position: children: [] id: CHART-rnb6PSwCOS meta: - chartId: 65 + chartId: 344 height: 50 sliceName: Currently Enrolled Learners Per Day uuid: ed2fe731-6544-422f-bc55-42f399f48b2c @@ -495,7 +494,7 @@ position: children: [] id: CHART-tWnaoVNNTH meta: - chartId: 70 + chartId: 331 height: 50 sliceName: Distribution Of Attempts uuid: db90930f-f16e-4c32-8050-0e4abae28f4c @@ -510,7 +509,7 @@ position: children: [] id: CHART-w-k4N2T_L8 meta: - chartId: 16 + chartId: 298 height: 50 sliceName: Course Grade Distribution uuid: f9adbc85-1f50-4c04-ace3-31ba7390de5e @@ -899,9 +898,7 @@ position: - TAB-7PGDduCA7 - TAB-CLiLC4zxo - TAB-eE0OQxuju - # {% if ASPECTS_INSTRUCTOR_HELP_MARKDOWN %} - TAB-nXpinnLEX - # {% endif %} id: TABS-SNeKAJcjhd meta: {} parents: From cbe1918b44fd6355e8e83b3e9f878e04a3136a43 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Fri, 15 Mar 2024 08:59:53 -0500 Subject: [PATCH 03/11] fix: use default locale on failed oauth remote connection --- .../pythonpath/openedx_sso_security_manager.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/openedx_sso_security_manager.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/openedx_sso_security_manager.py index 218c5f3c6..5f0698517 100644 --- a/tutoraspects/templates/aspects/apps/superset/pythonpath/openedx_sso_security_manager.py +++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/openedx_sso_security_manager.py @@ -61,14 +61,18 @@ def get_preferences(self, username): openedx_apis = current_app.config["OPENEDX_API_URLS"] url = openedx_apis["get_preference"].format(username=username) oauth_remote = self.oauth_remotes.get("openedxsso") - response = oauth_remote.get(url, token=self.get_oauth_token()).json() - locale_preference = response.get("pref-lang", "en").replace("-", "_") + locale_preference = "en" + try: + response = oauth_remote.get(url, token=self.get_oauth_token()).json() + locale_preference = response.get("pref-lang", "en").replace("-", "_") + except Exception as e: + return locale_preference if locale_preference not in current_app.config["DASHBOARD_LOCALES"]: - log.warning( - f"Language {locale_preference} is not supported by Superset" - ) - locale_preference = "en" + log.warning( + f"Language {locale_preference} is not supported by Superset" + ) + return locale_preference return locale_preference From 5a40b6cf02424edb099d437c7159ac453de4a3d9 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Fri, 15 Mar 2024 13:26:58 -0500 Subject: [PATCH 04/11] feat: add performance metrics commands --- tutoraspects/commands_v0.py | 15 ++ tutoraspects/commands_v1.py | 16 ++ .../pythonpath/performance_metrics.py | 139 ++++++++++++++++++ .../assets/charts/Query_performance.yaml | 43 ++++++ .../assets/datasets/query_log.yaml | 125 ++++++++++++++++ .../queries/slowest_clickhouse_queries.sql | 2 +- 6 files changed, 339 insertions(+), 1 deletion(-) create mode 100644 tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py create mode 100644 tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Query_performance.yaml create mode 100644 tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/query_log.yaml diff --git a/tutoraspects/commands_v0.py b/tutoraspects/commands_v0.py index cbbcf2cde..7b3dec040 100644 --- a/tutoraspects/commands_v0.py +++ b/tutoraspects/commands_v0.py @@ -106,6 +106,21 @@ def alembic(context, command) -> None: """ runner.run_job("aspects", command) +# Ex: "tutor local do performance-metrics " +@click.command(context_settings={"ignore_unknown_options": True}) +@click.pass_obj +def performance_metrics(context) -> None: + """ + Job to import Superset assets. + """ + config = tutor_config.load(context.root) + runner = context.job_runner(config) + + command = """echo 'Performance...' && + python /app/pythonpath/performance_metrics.py && + echo 'Done!'; + """ + runner.run_job("superset", command) # Ex: "tutor local do import_assets " @click.command(context_settings={"ignore_unknown_options": True}) diff --git a/tutoraspects/commands_v1.py b/tutoraspects/commands_v1.py index c9d799da5..8540b44a5 100644 --- a/tutoraspects/commands_v1.py +++ b/tutoraspects/commands_v1.py @@ -129,6 +129,21 @@ def import_assets() -> list[tuple[str, str]]: ), ] +# Ex: "tutor local do import_assets " +@click.command(context_settings={"ignore_unknown_options": True}) +def performance_metrics() -> list[tuple[str, str]]: + """ + Job to import Superset assets. + """ + return [ + ( + "superset", + "echo 'Performance...' && " + "python /app/pythonpath/performance_metrics.py &&" + "echo 'Done!';", + ), + ] + # Ex: "tutor local do dump_data_to_clickhouse " @click.command(context_settings={"ignore_unknown_options": True}) @@ -311,6 +326,7 @@ def check_superset_assets(): dump_data_to_clickhouse, transform_tracking_logs, import_assets, + performance_metrics, ) COMMANDS = (aspects,) diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py new file mode 100644 index 000000000..2b04522c3 --- /dev/null +++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py @@ -0,0 +1,139 @@ +from superset.app import create_app +app = create_app() +app.app_context().push() + +""" +The plan + +1. Load the instructor dashboard +2. Load the instructor dashboard datasets +3. Perform the following operations for every dataset: + a. Load the dataset + b. Perform queries on the dataset with/without filters + c. Perform aggregations on the dataset + d. Measure the time taken for each operation + e. Repeat 5 times and take metrics such as average, median, max, min, etc. +""" +from superset.extensions import db +from flask import g +from superset.models.dashboard import Dashboard +from superset.models.slice import Slice +from superset.charts.schemas import ChartDataQueryContextSchema +from superset.charts.data.commands.get_data_command import ChartDataCommand +from superset import security_manager +import json +from datetime import datetime +import sqlparse +from unittest.mock import patch +import uuid +import time + +ASPECTS_VERSION = "{{ASPECTS_VERSION}}" +UUID = str(uuid.uuid4())[0:6] +RUN_ID = f'aspects-{ASPECTS_VERSION}-{UUID}' + +report_format = """ + {i}. {slice} + Superset time: {superset_time}""" + +query_format = """ Query duration: {query_duration_ms} s. + Result rows: {result_rows} + Memory Usage (MB): {memory_usage_mb} + Row count Clickhouse {rowcount:} + Filters: {filters}""" + + +def performance_metrics(): + """Measure the performance of the dashboard.""" + with patch('clickhouse_connect.common.build_client_name') as mock_build_client_name: + mock_build_client_name.return_value = RUN_ID + embedable_dashboards = {{ SUPERSET_EMBEDDABLE_DASHBOARDS }} + dashboards = db.session.query(Dashboard).filter(Dashboard.slug.in_(embedable_dashboards)).all() + report = [] + for dashboard in dashboards: + print("Dashboard:", dashboard.slug) + for slice in dashboard.slices: + report.append(meassure_chart(slice)) + return report + + +def meassure_chart(slice, extra_filters=[]): + """ + Meassure the performance of a chart and return the results. + """ + print("Fetching slice data:", slice) + query_context = json.loads(slice.query_context) + query_context.update({ + "result_format": "json", + "result_type": "full", + "force": True, + }) + + if extra_filters: + query_context["filters"].extend(extra_filters) + + g.user = security_manager.find_user(username="{{SUPERSET_ADMIN_USERNAME}}") + query_context = ChartDataQueryContextSchema().load(query_context) + command = ChartDataCommand(query_context) + + start_time = datetime.now() + result = command.run() + end_time = datetime.now() + + result["time_elapsed"] = '{} s.'.format((end_time-start_time).total_seconds()) + result["slice"] = slice + return result + +def get_query_log_from_clickhouse(report): + """ + Get the query log from clickhouse and print the results. + """ + chart_uuid = "bb13bb31-c797-4ed3-a7f9-7825cc6dc482" + + slice = db.session.query(Slice).filter(Slice.uuid == chart_uuid).one() + + query_context = json.loads(slice.query_context) + query_context["queries"][0]["filters"].append( + { + 'col': 'http_user_agent', 'op': '==', 'val': RUN_ID + } + ) + slice.query_context = json.dumps(query_context) + + result = meassure_chart(slice) + + clickhouse_queries = {} + for query in result["queries"]: + for row in query["data"]: + parsed_sql = str(sqlparse.parse(row.pop("query"))[0]) + clickhouse_queries[parsed_sql] = row + + + print(f"\nSuperset Reports: {RUN_ID}") + for i, result in enumerate(report): + print(report_format.format( + i=(i+1), + slice=result["slice"], + superset_time=result["time_elapsed"] + )) + for i, query in enumerate(result["queries"]): + parsed_sql = str(sqlparse.parse(query["query"])[0]).replace(";", "") + "\n FORMAT Native" + clickhouse_report = clickhouse_queries.get(parsed_sql, {}) + print(query_format.format( + query_duration_ms=clickhouse_report.get("query_duration_ms") / 1000, + memory_usage_mb =clickhouse_report.get("memory_usage_mb"), + result_rows=clickhouse_report.get("result_rows"), + rowcount=query["rowcount"], + filters=query["applied_filters"], + )) + + + + +if __name__ == "__main__": + print("Running performance metrics. RUN ID:", RUN_ID) + report = performance_metrics() + # Clickhouse query log takes some seconds to log queries. + print("Waiting for clickhouse log...") + time.sleep(10) + get_query_log_from_clickhouse(report) diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Query_performance.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Query_performance.yaml new file mode 100644 index 000000000..32af356d8 --- /dev/null +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Query_performance.yaml @@ -0,0 +1,43 @@ +_file_name: Query_performance.yaml +cache_timeout: null +certification_details: null +certified_by: null +dataset_uuid: 53099fd2-6739-45e5-9a93-fcb9fa70119f +description: null +params: + adhoc_filters: + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: event_time + all_columns: [] + color_pn: true + extra_form_data: {} + groupby: + - http_user_agent + - query_duration_ms + - result_rows + - event_time + - memory_usage_mb + - tables + - query + order_by_cols: [] + order_desc: true + percent_metrics: [] + query_mode: aggregate + row_limit: 1000 + server_page_length: 10 + show_cell_bars: true + table_timestamp_format: smart_date + temporal_columns_lookup: + event_time: true + time_grain_sqla: P1D + viz_type: table +query_context: '{"datasource":{"id":266,"type":"table"},"force":false,"queries":[{"filters":[{"col":"event_time","op":"TEMPORAL_RANGE","val":"No + filter"}],"extras":{"time_grain_sqla":"P1D","having":"","where":""},"applied_time_extras":{},"columns":["http_user_agent","query_duration_ms","result_rows",{"timeGrain":"P1D","columnType":"BASE_AXIS","sqlExpression":"event_time","label":"event_time","expressionType":"SQL"},"memory_usage_mb","tables","query"],"metrics":[],"orderby":[],"annotation_layers":[],"row_limit":1000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"post_processing":[]}],"form_data":{"datasource":"266__table","viz_type":"table","slice_id":422,"query_mode":"aggregate","groupby":["http_user_agent","query_duration_ms","result_rows","event_time","memory_usage_mb","tables","query"],"time_grain_sqla":"P1D","temporal_columns_lookup":{"event_time":true},"all_columns":[],"percent_metrics":[],"adhoc_filters":[{"clause":"WHERE","comparator":"No + filter","expressionType":"SIMPLE","operator":"TEMPORAL_RANGE","subject":"event_time"}],"order_by_cols":[],"row_limit":1000,"server_page_length":10,"order_desc":true,"table_timestamp_format":"smart_date","show_cell_bars":true,"color_pn":true,"extra_form_data":{},"dashboards":[],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' +slice_name: Query performance +uuid: bb13bb31-c797-4ed3-a7f9-7825cc6dc482 +version: 1.0.0 +viz_type: table diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/query_log.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/query_log.yaml new file mode 100644 index 000000000..6ed95e148 --- /dev/null +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/query_log.yaml @@ -0,0 +1,125 @@ +_file_name: query_log.yaml +cache_timeout: null +columns: +- advanced_data_type: null + column_name: event_time + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: true + python_date_format: null + type: DateTime + verbose_name: Event Time +- advanced_data_type: null + column_name: query + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: String + verbose_name: Query +- advanced_data_type: null + column_name: tables + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: Array(LowCardinality(String)) + verbose_name: null +- advanced_data_type: null + column_name: http_user_agent + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: LowCardinality(String) + verbose_name: null +- advanced_data_type: null + column_name: memory_usage_mb + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: Float64 + verbose_name: null +- advanced_data_type: null + column_name: query_duration_ms + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: UInt64 + verbose_name: null +- advanced_data_type: null + column_name: result_rows + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: UInt64 + verbose_name: null +database_uuid: 21174b6c-4d40-4958-8161-d6c3cf5e77b6 +default_endpoint: null +description: null +extra: null +fetch_values_predicate: null +filter_select_enabled: true +main_dttm_col: null +metrics: +- currency: null + d3format: null + description: null + expression: COUNT(*) + extra: + warning_markdown: '' + metric_name: count + metric_type: count + verbose_name: COUNT(*) + warning_text: null +normalize_columns: false +offset: 0 +params: null +schema: main +sql: | + SELECT + query_duration_ms, + result_rows, + memory_usage / (1024 * 1024 ) AS memory_usage_mb, + event_time, + tables, + query, + http_user_agent + FROM system.query_log + WHERE (has(databases, '{{ASPECTS_XAPI_DATABASE}}') OR has(databases, '{{ASPECTS_EVENT_SINK_DATABASE}}') OR has(databases, '{{DBT_PROFILE_TARGET_DATABASE}}')) AND (http_user_agent LIKE 'aspects-%') AND (type = 'QueryFinish') + ORDER BY event_time DESC +table_name: query_log +template_params: null +uuid: 53099fd2-6739-45e5-9a93-fcb9fa70119f +version: 1.0.0 diff --git a/tutoraspects/templates/openedx-assets/queries/slowest_clickhouse_queries.sql b/tutoraspects/templates/openedx-assets/queries/slowest_clickhouse_queries.sql index 530da31f6..4a450f40d 100644 --- a/tutoraspects/templates/openedx-assets/queries/slowest_clickhouse_queries.sql +++ b/tutoraspects/templates/openedx-assets/queries/slowest_clickhouse_queries.sql @@ -7,6 +7,6 @@ SELECT FROM system.query_log WHERE - TYPE = 'QueryFinish' + type = 'QueryFinish' ORDER BY query_duration_ms DESC; From 5a2dcba2cbe811fd432b03af60e6a8146b359a8a Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Fri, 15 Mar 2024 14:25:05 -0500 Subject: [PATCH 05/11] build: add performance metrics step in CI --- .github/workflows/integration-test.yml | 6 ++++++ tutoraspects/commands_v1.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index a1aeed495..a6a226401 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -74,6 +74,8 @@ jobs: tutor local do dump-data-to-clickhouse --options "--object course_overviews" make extract_translations tutor local do import-assets + - name: Performance metrics + run: tutor local do performance-metrics - name: Tutor stop run: tutor local stop @@ -136,6 +138,8 @@ jobs: tutor dev do dump-data-to-clickhouse --options "--object course_overviews" make extract_translations tutor dev do import-assets + - name: Performance metrics + run: tutor dev do performance-metrics - name: Tutor stop run: tutor dev stop @@ -218,6 +222,8 @@ jobs: tutor k8s do dump-data-to-clickhouse --options "--object course_overviews" make extract_translations tutor k8s do import-assets + - name: Performance metrics + run: tutor k8s do performance-metrics - name: Check failure logs if: failure() run: | diff --git a/tutoraspects/commands_v1.py b/tutoraspects/commands_v1.py index 8540b44a5..281b53b59 100644 --- a/tutoraspects/commands_v1.py +++ b/tutoraspects/commands_v1.py @@ -129,7 +129,7 @@ def import_assets() -> list[tuple[str, str]]: ), ] -# Ex: "tutor local do import_assets " +# Ex: "tutor local do performance-metrics " @click.command(context_settings={"ignore_unknown_options": True}) def performance_metrics() -> list[tuple[str, str]]: """ From 6a097aa363933d443cd1b94ccf5f79fc3dadee32 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Fri, 15 Mar 2024 14:35:47 -0500 Subject: [PATCH 06/11] chore: quality fixes --- tutoraspects/commands_v0.py | 2 + tutoraspects/commands_v1.py | 1 + .../pythonpath/performance_metrics.py | 100 +++++++++--------- 3 files changed, 52 insertions(+), 51 deletions(-) diff --git a/tutoraspects/commands_v0.py b/tutoraspects/commands_v0.py index 7b3dec040..1826af9f7 100644 --- a/tutoraspects/commands_v0.py +++ b/tutoraspects/commands_v0.py @@ -106,6 +106,7 @@ def alembic(context, command) -> None: """ runner.run_job("aspects", command) + # Ex: "tutor local do performance-metrics " @click.command(context_settings={"ignore_unknown_options": True}) @click.pass_obj @@ -122,6 +123,7 @@ def performance_metrics(context) -> None: """ runner.run_job("superset", command) + # Ex: "tutor local do import_assets " @click.command(context_settings={"ignore_unknown_options": True}) @click.pass_obj diff --git a/tutoraspects/commands_v1.py b/tutoraspects/commands_v1.py index 281b53b59..76c16f628 100644 --- a/tutoraspects/commands_v1.py +++ b/tutoraspects/commands_v1.py @@ -129,6 +129,7 @@ def import_assets() -> list[tuple[str, str]]: ), ] + # Ex: "tutor local do performance-metrics " @click.command(context_settings={"ignore_unknown_options": True}) def performance_metrics() -> list[tuple[str, str]]: diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py index 2b04522c3..c1356bfbc 100644 --- a/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py +++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py @@ -1,36 +1,27 @@ from superset.app import create_app + app = create_app() app.app_context().push() -""" -The plan - -1. Load the instructor dashboard -2. Load the instructor dashboard datasets -3. Perform the following operations for every dataset: - a. Load the dataset - b. Perform queries on the dataset with/without filters - c. Perform aggregations on the dataset - d. Measure the time taken for each operation - e. Repeat 5 times and take metrics such as average, median, max, min, etc. -""" -from superset.extensions import db -from flask import g -from superset.models.dashboard import Dashboard -from superset.models.slice import Slice -from superset.charts.schemas import ChartDataQueryContextSchema -from superset.charts.data.commands.get_data_command import ChartDataCommand -from superset import security_manager + import json +import time +import uuid from datetime import datetime -import sqlparse from unittest.mock import patch -import uuid -import time + +import sqlparse +from flask import g +from superset import security_manager +from superset.charts.data.commands.get_data_command import ChartDataCommand +from superset.charts.schemas import ChartDataQueryContextSchema +from superset.extensions import db +from superset.models.dashboard import Dashboard +from superset.models.slice import Slice ASPECTS_VERSION = "{{ASPECTS_VERSION}}" UUID = str(uuid.uuid4())[0:6] -RUN_ID = f'aspects-{ASPECTS_VERSION}-{UUID}' +RUN_ID = f"aspects-{ASPECTS_VERSION}-{UUID}" report_format = """ {i}. {slice} @@ -45,10 +36,14 @@ def performance_metrics(): """Measure the performance of the dashboard.""" - with patch('clickhouse_connect.common.build_client_name') as mock_build_client_name: + with patch("clickhouse_connect.common.build_client_name") as mock_build_client_name: mock_build_client_name.return_value = RUN_ID - embedable_dashboards = {{ SUPERSET_EMBEDDABLE_DASHBOARDS }} - dashboards = db.session.query(Dashboard).filter(Dashboard.slug.in_(embedable_dashboards)).all() + embedable_dashboards = {{SUPERSET_EMBEDDABLE_DASHBOARDS}} + dashboards = ( + db.session.query(Dashboard) + .filter(Dashboard.slug.in_(embedable_dashboards)) + .all() + ) report = [] for dashboard in dashboards: print("Dashboard:", dashboard.slug) @@ -63,11 +58,13 @@ def meassure_chart(slice, extra_filters=[]): """ print("Fetching slice data:", slice) query_context = json.loads(slice.query_context) - query_context.update({ - "result_format": "json", - "result_type": "full", - "force": True, - }) + query_context.update( + { + "result_format": "json", + "result_type": "full", + "force": True, + } + ) if extra_filters: query_context["filters"].extend(extra_filters) @@ -80,10 +77,11 @@ def meassure_chart(slice, extra_filters=[]): result = command.run() end_time = datetime.now() - result["time_elapsed"] = '{} s.'.format((end_time-start_time).total_seconds()) + result["time_elapsed"] = "{} s.".format((end_time - start_time).total_seconds()) result["slice"] = slice return result + def get_query_log_from_clickhouse(report): """ Get the query log from clickhouse and print the results. @@ -94,9 +92,7 @@ def get_query_log_from_clickhouse(report): query_context = json.loads(slice.query_context) query_context["queries"][0]["filters"].append( - { - 'col': 'http_user_agent', 'op': '==', 'val': RUN_ID - } + {"col": "http_user_agent", "op": "==", "val": RUN_ID} ) slice.query_context = json.dumps(query_context) @@ -108,26 +104,28 @@ def get_query_log_from_clickhouse(report): parsed_sql = str(sqlparse.parse(row.pop("query"))[0]) clickhouse_queries[parsed_sql] = row - print(f"\nSuperset Reports: {RUN_ID}") for i, result in enumerate(report): - print(report_format.format( - i=(i+1), - slice=result["slice"], - superset_time=result["time_elapsed"] - )) + print( + report_format.format( + i=(i + 1), slice=result["slice"], superset_time=result["time_elapsed"] + ) + ) for i, query in enumerate(result["queries"]): - parsed_sql = str(sqlparse.parse(query["query"])[0]).replace(";", "") + "\n FORMAT Native" + parsed_sql = ( + str(sqlparse.parse(query["query"])[0]).replace(";", "") + + "\n FORMAT Native" + ) clickhouse_report = clickhouse_queries.get(parsed_sql, {}) - print(query_format.format( - query_duration_ms=clickhouse_report.get("query_duration_ms") / 1000, - memory_usage_mb =clickhouse_report.get("memory_usage_mb"), - result_rows=clickhouse_report.get("result_rows"), - rowcount=query["rowcount"], - filters=query["applied_filters"], - )) - - + print( + query_format.format( + query_duration_ms=clickhouse_report.get("query_duration_ms") / 1000, + memory_usage_mb=clickhouse_report.get("memory_usage_mb"), + result_rows=clickhouse_report.get("result_rows"), + rowcount=query["rowcount"], + filters=query["applied_filters"], + ) + ) if __name__ == "__main__": From bd0e34365dd03c1ac87bc8e7281f14aa9e74f828 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Fri, 15 Mar 2024 15:04:39 -0500 Subject: [PATCH 07/11] chore: use logger instead of print --- .../pythonpath/performance_metrics.py | 41 +++++++++++-------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py index c1356bfbc..e2e064456 100644 --- a/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py +++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py @@ -5,6 +5,7 @@ import json +import logging import time import uuid from datetime import datetime @@ -19,19 +20,22 @@ from superset.models.dashboard import Dashboard from superset.models.slice import Slice +logger = logging.getLogger("performance_metrics") + ASPECTS_VERSION = "{{ASPECTS_VERSION}}" UUID = str(uuid.uuid4())[0:6] RUN_ID = f"aspects-{ASPECTS_VERSION}-{UUID}" -report_format = """ - {i}. {slice} - Superset time: {superset_time}""" +report_format = "{i}. {slice}\n" "Superset time: {superset_time}\n" -query_format = """ Query duration: {query_duration_ms} s. - Result rows: {result_rows} - Memory Usage (MB): {memory_usage_mb} - Row count Clickhouse {rowcount:} - Filters: {filters}""" +query_format = ( + "-----------------------------------\n" + "Query duration: {query_duration_ms} s.\n" + "Result rows: {result_rows}\n" + "Memory Usage (MB): {memory_usage_mb}\n" + "Row count (superset) {rowcount:}\n" + "Filters: {filters}\n\n" +) def performance_metrics(): @@ -46,9 +50,13 @@ def performance_metrics(): ) report = [] for dashboard in dashboards: - print("Dashboard:", dashboard.slug) + logger.info(f"Dashboard: {dashboard.slug}") for slice in dashboard.slices: - report.append(meassure_chart(slice)) + result = meassure_chart(slice) + for query in result["queries"]: + # Remove the data from the query to avoid memory issues on large datasets. + query.pop("data") + report.append(result) return report @@ -56,7 +64,7 @@ def meassure_chart(slice, extra_filters=[]): """ Meassure the performance of a chart and return the results. """ - print("Fetching slice data:", slice) + logger.info(f"Fetching slice data: {slice}") query_context = json.loads(slice.query_context) query_context.update( { @@ -104,9 +112,9 @@ def get_query_log_from_clickhouse(report): parsed_sql = str(sqlparse.parse(row.pop("query"))[0]) clickhouse_queries[parsed_sql] = row - print(f"\nSuperset Reports: {RUN_ID}") + report_str = f"\nSuperset Reports: {RUN_ID}\n\n" for i, result in enumerate(report): - print( + report_str+=( report_format.format( i=(i + 1), slice=result["slice"], superset_time=result["time_elapsed"] ) @@ -117,7 +125,7 @@ def get_query_log_from_clickhouse(report): + "\n FORMAT Native" ) clickhouse_report = clickhouse_queries.get(parsed_sql, {}) - print( + report_str+=( query_format.format( query_duration_ms=clickhouse_report.get("query_duration_ms") / 1000, memory_usage_mb=clickhouse_report.get("memory_usage_mb"), @@ -126,12 +134,13 @@ def get_query_log_from_clickhouse(report): filters=query["applied_filters"], ) ) + logger.info(report_str) if __name__ == "__main__": - print("Running performance metrics. RUN ID:", RUN_ID) + logger.info(f"Running performance metrics. RUN ID: {RUN_ID}") report = performance_metrics() # Clickhouse query log takes some seconds to log queries. - print("Waiting for clickhouse log...") + logger.info("Waiting for clickhouse log...") time.sleep(10) get_query_log_from_clickhouse(report) From 6e15856db56dd859e03aa204c59267c38b0fa314 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Fri, 15 Mar 2024 15:24:39 -0500 Subject: [PATCH 08/11] chore: restore instructor dashboard chart ids --- .../pythonpath/performance_metrics.py | 10 ++++--- .../dashboards/Instructor_Dashboard.yaml | 30 ++++++++++--------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py index e2e064456..45da19870 100644 --- a/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py +++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py @@ -26,11 +26,10 @@ UUID = str(uuid.uuid4())[0:6] RUN_ID = f"aspects-{ASPECTS_VERSION}-{UUID}" -report_format = "{i}. {slice}\n" "Superset time: {superset_time}\n" +report_format = "{i}. {slice}\n" "Superset time: {superset_time} (s).\n" query_format = ( - "-----------------------------------\n" - "Query duration: {query_duration_ms} s.\n" + "Query duration: {query_duration_ms} (s).\n" "Result rows: {result_rows}\n" "Memory Usage (MB): {memory_usage_mb}\n" "Row count (superset) {rowcount:}\n" @@ -85,7 +84,7 @@ def meassure_chart(slice, extra_filters=[]): result = command.run() end_time = datetime.now() - result["time_elapsed"] = "{} s.".format((end_time - start_time).total_seconds()) + result["time_elapsed"] = (end_time - start_time).total_seconds() result["slice"] = slice return result @@ -112,6 +111,9 @@ def get_query_log_from_clickhouse(report): parsed_sql = str(sqlparse.parse(row.pop("query"))[0]) clickhouse_queries[parsed_sql] = row + # Sort report by slowest queries + report = sorted(report, key=lambda x: x["time_elapsed"], reverse=True) + report_str = f"\nSuperset Reports: {RUN_ID}\n\n" for i, result in enumerate(report): report_str+=( diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Instructor_Dashboard.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Instructor_Dashboard.yaml index 60254b554..06885d666 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Instructor_Dashboard.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Instructor_Dashboard.yaml @@ -314,7 +314,7 @@ position: children: [] id: CHART-AZZnl_lpMv meta: - chartId: 359 + chartId: 13 height: 50 sliceName: Watches Per Video uuid: 829c1d5b-2844-4115-876a-34ad3b3cad64 @@ -329,7 +329,7 @@ position: children: [] id: CHART-GFCO8s2cxv meta: - chartId: 352 + chartId: 25 height: 50 sliceName: Distribution Of Responses uuid: f1651c44-a8f4-4b44-ad49-962823009319 @@ -344,7 +344,7 @@ position: children: [] id: CHART-GJJ8VYQ03v meta: - chartId: 310 + chartId: 55 height: 50 sliceName: Posts per user uuid: bc191ce7-f39d-48db-86a9-d19949f4211d @@ -359,7 +359,7 @@ position: children: [] id: CHART-Jr-gNVms2Q meta: - chartId: 332 + chartId: 34 height: 50 sliceName: Enrollment Events Per Day uuid: bb1147cc-b7bc-44b7-b06a-79b0db6626aa @@ -374,7 +374,7 @@ position: children: [] id: CHART-OMy4wjRBWt meta: - chartId: 371 + chartId: 45 height: 50 sliceName: Watched Video Segments uuid: 2985a9db-c338-4008-af52-2930b81ee2e5 @@ -389,7 +389,7 @@ position: children: [] id: CHART-RTO33WE9FH meta: - chartId: 339 + chartId: 59 height: 50 sliceName: Responses Per Problem uuid: a3e79162-4ace-4349-ab34-89aa60ae75ed @@ -404,7 +404,7 @@ position: children: [] id: CHART-Tej2oLPBAl meta: - chartId: 410 + chartId: 58 height: 50 sliceName: Transcripts / Captions Per Video uuid: 6b830def-f3ca-4b4c-9455-7a7b7354bce8 @@ -419,7 +419,7 @@ position: children: [] id: CHART-evjVO-ZSSd meta: - chartId: 411 + chartId: 27 height: 50 sliceName: Enrollments By Enrollment Mode uuid: 05ed7102-5464-4e2f-86ae-31700b787cc3 @@ -434,7 +434,7 @@ position: children: [] id: CHART-lTr8DL3XuI meta: - chartId: 307 + chartId: 57 height: 50 sliceName: Distribution Of Hints Per Correct Answer uuid: ee94be4c-6fdd-4295-b43c-40890d6c549d @@ -449,7 +449,7 @@ position: children: [] id: CHART-o56v9yEe2I meta: - chartId: 382 + chartId: 11 height: 50 sliceName: Distribution Of Problem Grades uuid: 4f7e3606-f5de-4643-97c0-bbb6340a3df2 @@ -464,7 +464,7 @@ position: children: [] id: CHART-qG1WaGKl_b meta: - chartId: 395 + chartId: 26 height: 50 sliceName: Distinct forum users uuid: feb323ad-c819-49ca-a336-584bd9ff1a2e @@ -479,7 +479,7 @@ position: children: [] id: CHART-rnb6PSwCOS meta: - chartId: 344 + chartId: 65 height: 50 sliceName: Currently Enrolled Learners Per Day uuid: ed2fe731-6544-422f-bc55-42f399f48b2c @@ -494,7 +494,7 @@ position: children: [] id: CHART-tWnaoVNNTH meta: - chartId: 331 + chartId: 70 height: 50 sliceName: Distribution Of Attempts uuid: db90930f-f16e-4c32-8050-0e4abae28f4c @@ -509,7 +509,7 @@ position: children: [] id: CHART-w-k4N2T_L8 meta: - chartId: 298 + chartId: 16 height: 50 sliceName: Course Grade Distribution uuid: f9adbc85-1f50-4c04-ace3-31ba7390de5e @@ -898,7 +898,9 @@ position: - TAB-7PGDduCA7 - TAB-CLiLC4zxo - TAB-eE0OQxuju + # {% if ASPECTS_INSTRUCTOR_HELP_MARKDOWN %} - TAB-nXpinnLEX + # {% endif %} id: TABS-SNeKAJcjhd meta: {} parents: From 05ef907ac9a3bfc26c87cb66fe957f09d8cacaec Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Fri, 15 Mar 2024 15:57:19 -0500 Subject: [PATCH 09/11] fix: inject datasource_id --- .../aspects/apps/superset/pythonpath/performance_metrics.py | 4 ++++ .../templates/openedx-assets/queries/fact_course_grades.sql | 2 +- .../queries/fact_learner_problem_course_summary.sql | 2 +- .../openedx-assets/queries/fact_learner_problem_summary.sql | 2 +- .../templates/openedx-assets/queries/fact_problem_grades.sql | 2 +- .../openedx-assets/queries/fact_problem_responses.sql | 2 +- .../openedx-assets/queries/fact_transcript_usage.sql | 2 +- .../templates/openedx-assets/queries/fact_video_plays.sql | 2 +- .../openedx-assets/queries/fact_watched_video_segments.sql | 2 +- 9 files changed, 12 insertions(+), 8 deletions(-) diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py index 45da19870..6e5187a3e 100644 --- a/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py +++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py @@ -70,6 +70,10 @@ def meassure_chart(slice, extra_filters=[]): "result_format": "json", "result_type": "full", "force": True, + "datasource": { + "type": "table", + "id": slice.datasource_id, + }, } ) diff --git a/tutoraspects/templates/openedx-assets/queries/fact_course_grades.sql b/tutoraspects/templates/openedx-assets/queries/fact_course_grades.sql index 6ce647ca2..be0ca87c9 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_course_grades.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_course_grades.sql @@ -3,7 +3,7 @@ with grades as ( from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_grades where grade_type = 'course' {% raw %} - {% if get_filters('course_name', remove_filter=True) != [] && filter_values('course_name') != [] %} + {% if get_filters('course_name', remove_filter=True) != [] and filter_values('course_name') != [] %} and entity_name in {{ filter_values('course_name', remove_filter=True) | where_in }} {% else %} and 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql index 62d4cc807..77eb094a7 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql @@ -134,7 +134,7 @@ SELECT FROM summary WHERE {% raw %} - {% if get_filters('course_name', remove_filter=True) != [] && filter_values('course_name') != [] %} + {% if get_filters('course_name', remove_filter=True) != [] and filter_values('course_name') != [] %} course_name in {{ filter_values('course_name') | where_in }} {% else %} 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql index c37eac7e0..4362a6bcc 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql @@ -118,7 +118,7 @@ SELECT FROM summary where {% raw %} - {% if get_filters('problem_name_with_location', remove_filter=True) != [] && filter_values('problem_name_with_location') != [] %} + {% if get_filters('problem_name_with_location', remove_filter=True) != [] and filter_values('problem_name_with_location') != [] %} problem_name_with_location in {{ filter_values('problem_name_with_location') | where_in }} {% else %} 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql b/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql index 27a6ceff9..f55ab7f15 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql @@ -5,7 +5,7 @@ with grades as ( grade_type = 'problem' {% raw %} - {% if get_filters('problem_name_with_location', remove_filter=True) != [] && filter_values('problem_name_with_location') != [] %} + {% if get_filters('problem_name_with_location', remove_filter=True) != [] and filter_values('problem_name_with_location') != [] %} and entity_name_with_location in {{ filter_values('problem_name_with_location', remove_filter=True) | where_in }} {% else %} and 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql b/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql index 61ab2c72a..57e7700c7 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql @@ -25,7 +25,7 @@ from problem_responses where {% raw %} - {% if get_filters('problem_name_with_location', remove_filter=True) != [] && filter_values('problem_name_with_location') != [] %} + {% if get_filters('problem_name_with_location', remove_filter=True) != [] and filter_values('problem_name_with_location') != [] %} problem_name_with_location in {{ filter_values('problem_name_with_location') | where_in }} {% else %} 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql b/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql index 6a0466f7f..5134fe985 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql @@ -3,7 +3,7 @@ select * from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_transcript_usage where {% raw %} - {% if get_filters('course_name', remove_filter=True) != [] && filter_values('course_name') != [] %} + {% if get_filters('course_name', remove_filter=True) != [] and filter_values('course_name') != [] %} course_name in {{ filter_values('course_name') | where_in }} {% else %} 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql b/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql index d9cf1acd9..bb491e30b 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql @@ -3,7 +3,7 @@ select * from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_video_plays where {% raw %} - {% if get_filters('course_name', remove_filter=True) != [] && filter_values('course_name') != [] %} + {% if get_filters('course_name', remove_filter=True) != [] and filter_values('course_name') != [] %} course_name in {{ filter_values('course_name') | where_in }} {% else %} 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql b/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql index 66b30562d..298fefd20 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql @@ -75,7 +75,7 @@ select from enriched_segments where {% raw %} - {% if get_filters('video_name_with_location', remove_filter=True) != [] && filter_values('video_name_with_location') != [] %} + {% if get_filters('video_name_with_location', remove_filter=True) != [] and filter_values('video_name_with_location') != [] %} video_name_with_location in {{ filter_values('video_name_with_location') | where_in }} {% else %} 1=0 From cbf3bf9ac95fb62cb762fef5252267bda9712024 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Mon, 18 Mar 2024 09:57:45 -0500 Subject: [PATCH 10/11] chore: only filter datasets on dashboard --- .../templates/openedx-assets/queries/fact_course_grades.sql | 3 ++- .../queries/fact_learner_problem_course_summary.sql | 3 ++- .../openedx-assets/queries/fact_learner_problem_summary.sql | 3 ++- .../templates/openedx-assets/queries/fact_problem_grades.sql | 3 ++- .../openedx-assets/queries/fact_problem_responses.sql | 3 ++- .../templates/openedx-assets/queries/fact_transcript_usage.sql | 3 ++- .../templates/openedx-assets/queries/fact_video_plays.sql | 3 ++- .../openedx-assets/queries/fact_watched_video_segments.sql | 3 ++- 8 files changed, 16 insertions(+), 8 deletions(-) diff --git a/tutoraspects/templates/openedx-assets/queries/fact_course_grades.sql b/tutoraspects/templates/openedx-assets/queries/fact_course_grades.sql index be0ca87c9..6341e83c1 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_course_grades.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_course_grades.sql @@ -3,7 +3,8 @@ with grades as ( from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_grades where grade_type = 'course' {% raw %} - {% if get_filters('course_name', remove_filter=True) != [] and filter_values('course_name') != [] %} + {% if get_filters('course_name', remove_filter=True) == [] %} + {% elif filter_values('course_name') != [] %} and entity_name in {{ filter_values('course_name', remove_filter=True) | where_in }} {% else %} and 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql index 77eb094a7..b3c3a1d35 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql @@ -134,7 +134,8 @@ SELECT FROM summary WHERE {% raw %} - {% if get_filters('course_name', remove_filter=True) != [] and filter_values('course_name') != [] %} + {% if get_filters('course_name', remove_filter=True) == [] %} + {% elif filter_values('course_name') != [] %} course_name in {{ filter_values('course_name') | where_in }} {% else %} 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql index 4362a6bcc..fa6193dd8 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql @@ -118,7 +118,8 @@ SELECT FROM summary where {% raw %} - {% if get_filters('problem_name_with_location', remove_filter=True) != [] and filter_values('problem_name_with_location') != [] %} + {% if get_filters('problem_name_with_location', remove_filter=True) == [] %} + {% elif filter_values('problem_name_with_location') != [] %} problem_name_with_location in {{ filter_values('problem_name_with_location') | where_in }} {% else %} 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql b/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql index f55ab7f15..6d6d9e15d 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql @@ -5,7 +5,8 @@ with grades as ( grade_type = 'problem' {% raw %} - {% if get_filters('problem_name_with_location', remove_filter=True) != [] and filter_values('problem_name_with_location') != [] %} + {% if get_filters('problem_name_with_location', remove_filter=True) != [] %} + {% elif filter_values('problem_name_with_location') != [] %} and entity_name_with_location in {{ filter_values('problem_name_with_location', remove_filter=True) | where_in }} {% else %} and 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql b/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql index 57e7700c7..772e010f7 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql @@ -25,7 +25,8 @@ from problem_responses where {% raw %} - {% if get_filters('problem_name_with_location', remove_filter=True) != [] and filter_values('problem_name_with_location') != [] %} + {% if get_filters('problem_name_with_location', remove_filter=True) == [] %} + {% elif filter_values('problem_name_with_location') != [] %} problem_name_with_location in {{ filter_values('problem_name_with_location') | where_in }} {% else %} 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql b/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql index 5134fe985..e1def078f 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql @@ -3,7 +3,8 @@ select * from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_transcript_usage where {% raw %} - {% if get_filters('course_name', remove_filter=True) != [] and filter_values('course_name') != [] %} + {% if get_filters('course_name', remove_filter=True) == [] %} + {% elif filter_values('course_name') != [] %} course_name in {{ filter_values('course_name') | where_in }} {% else %} 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql b/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql index bb491e30b..1db47bd4e 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql @@ -3,7 +3,8 @@ select * from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_video_plays where {% raw %} - {% if get_filters('course_name', remove_filter=True) != [] and filter_values('course_name') != [] %} + {% if get_filters('course_name', remove_filter=True) == [] %} + {% elif filter_values('course_name') != [] %} course_name in {{ filter_values('course_name') | where_in }} {% else %} 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql b/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql index 298fefd20..97f713272 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql @@ -75,7 +75,8 @@ select from enriched_segments where {% raw %} - {% if get_filters('video_name_with_location', remove_filter=True) != [] and filter_values('video_name_with_location') != [] %} + {% if get_filters('video_name_with_location', remove_filter=True) == [] %} + {% elif filter_values('video_name_with_location') != [] %} video_name_with_location in {{ filter_values('video_name_with_location') | where_in }} {% else %} 1=0 From a8ec751fd6e36fbcf184104704aee19db9b0e741 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Mon, 18 Mar 2024 10:16:45 -0500 Subject: [PATCH 11/11] chore: address PR suggestions --- tutoraspects/commands_v0.py | 2 +- tutoraspects/commands_v1.py | 2 +- .../apps/superset/pythonpath/performance_metrics.py | 10 ++++++---- .../queries/fact_learner_problem_course_summary.sql | 1 + .../queries/fact_learner_problem_summary.sql | 1 + .../openedx-assets/queries/fact_problem_grades.sql | 2 +- .../openedx-assets/queries/fact_problem_responses.sql | 1 + .../openedx-assets/queries/fact_transcript_usage.sql | 1 + .../openedx-assets/queries/fact_video_plays.sql | 1 + .../queries/fact_watched_video_segments.sql | 1 + 10 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tutoraspects/commands_v0.py b/tutoraspects/commands_v0.py index 1826af9f7..bb58e9d87 100644 --- a/tutoraspects/commands_v0.py +++ b/tutoraspects/commands_v0.py @@ -112,7 +112,7 @@ def alembic(context, command) -> None: @click.pass_obj def performance_metrics(context) -> None: """ - Job to import Superset assets. + Job to measure performance metrics of charts and its queries in Superset and ClickHouse. """ config = tutor_config.load(context.root) runner = context.job_runner(config) diff --git a/tutoraspects/commands_v1.py b/tutoraspects/commands_v1.py index 76c16f628..0bc6ab6d5 100644 --- a/tutoraspects/commands_v1.py +++ b/tutoraspects/commands_v1.py @@ -134,7 +134,7 @@ def import_assets() -> list[tuple[str, str]]: @click.command(context_settings={"ignore_unknown_options": True}) def performance_metrics() -> list[tuple[str, str]]: """ - Job to import Superset assets. + Job to measure performance metrics of charts and its queries in Superset and ClickHouse. """ return [ ( diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py index 6e5187a3e..7d3746b22 100644 --- a/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py +++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py @@ -39,6 +39,8 @@ def performance_metrics(): """Measure the performance of the dashboard.""" + # Mock the client name to identify the queries in the clickhouse system.query_log table by + # by the http_user_agent field. with patch("clickhouse_connect.common.build_client_name") as mock_build_client_name: mock_build_client_name.return_value = RUN_ID embedable_dashboards = {{SUPERSET_EMBEDDABLE_DASHBOARDS}} @@ -51,7 +53,7 @@ def performance_metrics(): for dashboard in dashboards: logger.info(f"Dashboard: {dashboard.slug}") for slice in dashboard.slices: - result = meassure_chart(slice) + result = measure_chart(slice) for query in result["queries"]: # Remove the data from the query to avoid memory issues on large datasets. query.pop("data") @@ -59,9 +61,9 @@ def performance_metrics(): return report -def meassure_chart(slice, extra_filters=[]): +def measure_chart(slice, extra_filters=[]): """ - Meassure the performance of a chart and return the results. + Measure the performance of a chart and return the results. """ logger.info(f"Fetching slice data: {slice}") query_context = json.loads(slice.query_context) @@ -107,7 +109,7 @@ def get_query_log_from_clickhouse(report): ) slice.query_context = json.dumps(query_context) - result = meassure_chart(slice) + result = measure_chart(slice) clickhouse_queries = {} for query in result["queries"]: diff --git a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql index b3c3a1d35..19d27b027 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql @@ -135,6 +135,7 @@ FROM summary WHERE {% raw %} {% if get_filters('course_name', remove_filter=True) == [] %} + 1=1 {% elif filter_values('course_name') != [] %} course_name in {{ filter_values('course_name') | where_in }} {% else %} diff --git a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql index fa6193dd8..692f06beb 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql @@ -119,6 +119,7 @@ FROM summary where {% raw %} {% if get_filters('problem_name_with_location', remove_filter=True) == [] %} + 1=1 {% elif filter_values('problem_name_with_location') != [] %} problem_name_with_location in {{ filter_values('problem_name_with_location') | where_in }} {% else %} diff --git a/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql b/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql index 6d6d9e15d..d9be4306d 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql @@ -5,7 +5,7 @@ with grades as ( grade_type = 'problem' {% raw %} - {% if get_filters('problem_name_with_location', remove_filter=True) != [] %} + {% if get_filters('problem_name_with_location', remove_filter=True) == [] %} {% elif filter_values('problem_name_with_location') != [] %} and entity_name_with_location in {{ filter_values('problem_name_with_location', remove_filter=True) | where_in }} {% else %} diff --git a/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql b/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql index 772e010f7..e320b21db 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql @@ -26,6 +26,7 @@ from where {% raw %} {% if get_filters('problem_name_with_location', remove_filter=True) == [] %} + 1=1 {% elif filter_values('problem_name_with_location') != [] %} problem_name_with_location in {{ filter_values('problem_name_with_location') | where_in }} {% else %} diff --git a/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql b/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql index e1def078f..3059cc779 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql @@ -4,6 +4,7 @@ from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_transcript_usage where {% raw %} {% if get_filters('course_name', remove_filter=True) == [] %} + 1=1 {% elif filter_values('course_name') != [] %} course_name in {{ filter_values('course_name') | where_in }} {% else %} diff --git a/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql b/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql index 1db47bd4e..9609b93f5 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql @@ -4,6 +4,7 @@ from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_video_plays where {% raw %} {% if get_filters('course_name', remove_filter=True) == [] %} + 1=1 {% elif filter_values('course_name') != [] %} course_name in {{ filter_values('course_name') | where_in }} {% else %} diff --git a/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql b/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql index 97f713272..6b9dcfb6a 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql @@ -76,6 +76,7 @@ from enriched_segments where {% raw %} {% if get_filters('video_name_with_location', remove_filter=True) == [] %} + 1=1 {% elif filter_values('video_name_with_location') != [] %} video_name_with_location in {{ filter_values('video_name_with_location') | where_in }} {% else %}