feat: allow to translate dataset text on chart

.github/workflows/integration-test.yml

@@ -2,7 +2,6 @@ name: Tutor Integration Test
 
 on:
   pull_request:
-    branches: [ main ]
     paths-ignore:
       - .github/**
       - .ci/**

scripts/translate_utils.py

@@ -86,12 +86,13 @@ class DatasetAsset(TranslatableAsset):
     "table_name": ("datasets", DatasetAsset),
 }
 
+BASE_PATH = "tutoraspects/templates/aspects/build/aspects-superset/"
 
 def get_text_for_translations(root_path):
     assets_path = (
         os.path.join(
             root_path,
-            "tutoraspects/templates/aspects/build/aspects-superset/"
+            BASE_PATH,
             "openedx-assets/assets/"
         )
     )
@@ -112,6 +113,11 @@ def get_text_for_translations(root_path):
             asset = yaml.safe_load(asset_str)
             strings.extend(mark_text_for_translation(asset))
 
+    with open(BASE_PATH + "localization/datasets_strings.yaml", 'r') as file:
+        dataset_strings = yaml.safe_load(file.read())
+        for key in dataset_strings:
+            strings.extend(dataset_strings[key])
+            print(f"Extracted {len(dataset_strings[key])} strings for dataset {key}")
     return strings
 
 

tutoraspects/asset_command_helpers.py

@@ -53,6 +53,7 @@ class Asset:
     templated_vars = None
     required_vars = None
     omitted_vars = None
+    raw_vars = None
 
     def __init__(self):
         if not self.path:
@@ -92,6 +93,12 @@ def get_omitted_vars(self):
         """
         return self.omitted_vars or []
 
+    def get_raw_vars(self):
+        """
+        Returns a list of variables which should be omitted from the content.
+        """
+        return self.raw_vars or []
+
     def remove_content(self, content: dict):
         """
         Remove any variables from the content which should be omitted.
@@ -122,18 +129,32 @@ def omit_templated_vars(self, content: dict, existing: dict):
                     continue
                 if isinstance(existing[key], str):
                     if "{{" in existing.get(key, "") or "{%" in existing.get(key, ""):
-                        content[key] = existing[key]
+                        if key in self.get_raw_vars():
+                            raw_expression = "{% raw %}" + content[key] + "{% endraw %}"
+                            content[key] = raw_expression
+                        else:
+                            content[key] = existing[key]
+
                 if isinstance(existing[key], dict):
                     self.omit_templated_vars(content[key], existing[key])
 
+                if isinstance(existing[key], list):
+                    for i, item in enumerate(content[key]):
+                        if isinstance(item, dict):
+                            try:
+                                existing[key][i]
+                                self.omit_templated_vars(item, existing[key][i] or None)
+                            except IndexError:
+                                pass
 
 class ChartAsset(Asset):
     """
     Chart assets.
     """
 
     path = "charts"
-    omitted_vars = ["query_context"]
+    omitted_vars = ["query_context", "params.dashboards", "params.datasource", "params.slice_id"]
+    raw_vars = ["sqlExpression"]
 
 
 class DashboardAsset(Asset):

tutoraspects/patches/local-docker-compose-dev-services

@@ -3,7 +3,7 @@ superset:
   {% include 'base-docker-compose-services' %}
     OPENEDX_LMS_ROOT_URL: "http://{{ LMS_HOST }}:8000"
     OAUTH2_CLIENT_ID: {{ SUPERSET_OAUTH2_CLIENT_ID_DEV }}
-  command: ["bash", "/app/docker/docker-bootstrap.sh", "app-gunicorn"]
+  command: ["bash", "/app/docker/docker-bootstrap.sh", "app"]
   ports:
     - 8088:{{ SUPERSET_PORT }}
   depends_on:

tutoraspects/templates/aspects/apps/superset/docker/docker-bootstrap.sh

@@ -47,7 +47,7 @@ elif [[ "${1}" == "beat" ]]; then
   celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid -l INFO -s "${SUPERSET_HOME}"/celerybeat-schedule
 elif [[ "${1}" == "app" ]]; then
   echo "Starting web app..."
-  flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0
+  flask run -p 8088 --with-threads --reload --debugger --debug --host=0.0.0.0
 elif [[ "${1}" == "app-gunicorn" ]]; then
   echo "Starting web app..."
   /usr/bin/run-server.sh

tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py

@@ -17,6 +17,7 @@
 from superset.models.dashboard import Dashboard
 from superset.utils.database import get_or_create_db
 from superset.models.embedded_dashboard import EmbeddedDashboard
+from pythonpath.localization import get_translation
 
 BASE_DIR = "/app/assets/superset"
 
@@ -34,26 +35,9 @@
 
 FILE_NAME_ATTRIBUTE = "_file_name"
 
-TRANSLATIONS_FILE_PATH = "/app/localization/locale.yaml"
 ASSETS_FILE_PATH = "/app/pythonpath/assets.yaml"
 ASSETS_PATH = "/app/openedx-assets"
 
-merged_data = {}
-with open(TRANSLATIONS_FILE_PATH, "r") as file:
-    yaml_content = file.read()
-    yaml_documents = yaml_content.split("\n---\n")
-
-    for doc in yaml_documents:
-        data = yaml.safe_load(doc)
-        if data is not None:
-            for lang, translations in data.items():
-                if lang not in merged_data:
-                    merged_data[lang] = {}
-                merged_data[lang].update(translations)
-
-
-ASSETS_TRANSLATIONS = merged_data
-
 
 def main():
     create_assets()
@@ -297,14 +281,6 @@ def update_embeddable_uuids():
         db.session.add(embedded_dashboard)
         db.session.commit()
 
-def get_translation(text, language):
-    """Get a translation for a text in a language"""
-    default_text = f"{text}"
-    LANGUAGE = ASSETS_TRANSLATIONS.get(language, {})
-    if not LANGUAGE:
-        return default_text
-    return ASSETS_TRANSLATIONS.get(language, {}).get(text) or default_text
-
 
 if __name__ == "__main__":
     main()

tutoraspects/templates/aspects/apps/superset/pythonpath/localization.py

@@ -0,0 +1,34 @@
+"""
+This file is used to load translations from the locale.yaml file and provide a function to get translations
+"""
+import yaml
+
+TRANSLATIONS_FILE_PATH = "/app/localization/locale.yaml"
+DATASET_STRINGS_FILE_PATH = "/app/localization/datasets_strings.yaml"
+
+merged_data = {}
+with open(TRANSLATIONS_FILE_PATH, "r") as file:
+    yaml_content = file.read()
+    yaml_documents = yaml_content.split("\n---\n")
+
+    for doc in yaml_documents:
+        data = yaml.safe_load(doc)
+        if data is not None:
+            for lang, translations in data.items():
+                if lang not in merged_data:
+                    merged_data[lang] = {}
+                merged_data[lang].update(translations)
+
+
+ASSETS_TRANSLATIONS = merged_data
+
+DATASET_STRINGS = yaml.safe_load(open(DATASET_STRINGS_FILE_PATH, "r"))
+
+
+def get_translation(text, language):
+    """Get a translation for a text in a language"""
+    default_text = f"{text}"
+    LANGUAGE = ASSETS_TRANSLATIONS.get(language, {})
+    if not LANGUAGE:
+        return default_text
+    return ASSETS_TRANSLATIONS.get(language, {}).get(text) or default_text

tutoraspects/templates/aspects/apps/superset/pythonpath/openedx_jinja_filters.py

@@ -4,7 +4,11 @@
 cf https://superset.apache.org/docs/installation/sql-templating/
 """
 from superset.extensions import security_manager
+from pythonpath.localization import get_translation, DATASET_STRINGS
+from superset import security_manager
+import logging
 
+log = logging.getLogger(__name__)
 ALL_COURSES = "1 = 1"
 NO_COURSES = "1 = 0"
 
@@ -44,4 +48,31 @@ def can_view_courses(username, field_name="course_id", **kwargs):
         return NO_COURSES
 
 
+def translate_column(column_name):
+    """
+    Translate a string to the given language.
+    """
+    roles = security_manager.get_user_roles()
+    log.info(f"Roles: {roles}")
+    lang = "en"
+    if roles:
+        for role in roles:
+            role_str = role.name.split(" - ")
+            if len(role_str) > 1:
+                lang = role_str[1]
+                break
+
+    strings = DATASET_STRINGS.get(column_name, [])
+    case_format = """CASE \n {cases} \n ELSE {column_name} \n END"""
+    single_case_format = "WHEN {column_name} = '{string}' THEN '{translation}'"
+    cases = "\n".join(
+        single_case_format.format(column_name=column_name, string=string, translation=get_translation(string, lang))
+        for string in strings
+    )
+
+    log.info(f"Translating {column_name} to {lang}")
+    log.info(case_format.format(column_name=column_name, cases=cases))
+
+    return case_format.format(column_name=column_name, cases=cases)
+
 {{patch("superset-jinja-filters")}}

tutoraspects/templates/aspects/apps/superset/pythonpath/superset_config_docker.py

@@ -108,6 +108,7 @@ def can_view_courses_wrapper(*args, **kwargs):
 
 JINJA_CONTEXT_ADDONS = {
     'can_view_courses': can_view_courses_wrapper,
+    'translate_column': translate_column,
     {% for filter in SUPERSET_EXTRA_JINJA_FILTERS %}'{{ filter }}': {{filter}},{% endfor %}
 }
 

tutoraspects/templates/aspects/build/aspects-superset/localization/datasets_strings.yaml

@@ -0,0 +1,4 @@
+enrollment_mode:
+- audit
+- verified
+- honor

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Grade_Distribution.yaml

@@ -25,7 +25,6 @@ params:
   row_limit: 10000
   show_legend: false
   slice_id: 352
-  time_range: 'DATEADD(DATETIME("now"), -1, month) : DATEADD(DATETIME("now"), 1, day)'
   viz_type: dist_bar
   x_axis_label: Course Grade (out of 100%)
   x_ticks_layout: auto

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Attempts.yaml

@@ -34,7 +34,6 @@ params:
   row_limit: 10000
   show_legend: false
   slice_id: 30
-  time_range: 'DATEADD(DATETIME("now"), -1, month) : DATEADD(DATETIME("now"), 1, day)'
   viz_type: dist_bar
   x_axis_label: Number Of Attempts To Find Correct Answer
   x_ticks_layout: auto

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Hints_Per_Correct_Answer.yaml

@@ -25,7 +25,6 @@ params:
   row_limit: 10000
   show_legend: false
   slice_id: 166
-  time_range: 'DATEADD(DATETIME("now"), -1, month) : DATEADD(DATETIME("now"), 1, day)'
   viz_type: dist_bar
   x_axis_label: Hints / Answer Displayed Before Correct Answer Chosen
   x_ticks_layout: auto

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Problem_Grades.yaml

@@ -26,7 +26,6 @@ params:
   row_limit: 10000
   show_legend: false
   slice_id: 345
-  time_range: 'DATEADD(DATETIME("now"), -1, month) : DATEADD(DATETIME("now"), 1, day)'
   viz_type: dist_bar
   x_axis_label: Percentage Grade
   x_ticks_layout: auto

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Responses.yaml

@@ -25,7 +25,6 @@ params:
   row_limit: 10000
   show_legend: false
   slice_id: 314
-  time_range: 'DATEADD(DATETIME("now"), -1, month) : DATEADD(DATETIME("now"), 1, day)'
   viz_type: dist_bar
   x_axis_label: Answers Chosen
   x_ticks_layout: auto

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollments_By_Enrollment_Mode.yaml

@@ -19,14 +19,13 @@ params:
     sqlExpression: null
     subject: enrollment_status
   color_scheme: supersetColors
-  dashboards:
-  - 23
-  datasource: 11__table
   date_format: smart_date
   extra_form_data: {}
-  granularity_sqla: emission_time
   groupby:
-  - enrollment_mode
+  - expressionType: SQL
+    label: Enrollment Mode
+    sqlExpression: |-
+      {% raw %}{{translate_column('enrollment_mode')}}{% endraw %}
   innerRadius: 30
   label_type: key
   labels_outside: true
@@ -39,11 +38,8 @@ params:
   show_labels: true
   show_labels_threshold: 5
   show_legend: true
-  slice_id: 40
   sort_by_metric: true
-  time_range: 'DATEADD(DATETIME("now"), -1, month) : DATEADD(DATETIME("now"), 1, day)'
   viz_type: pie
-query_context: null
 slice_name: Enrollments By Enrollment Mode
 uuid: 05ed7102-5464-4e2f-86ae-31700b787cc3
 version: 1.0.0

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Responses_Per_Problem.yaml

@@ -51,7 +51,6 @@ params:
   show_controls: false
   show_legend: false
   slice_id: 262
-  time_range: 'DATEADD(DATETIME("now"), -1, month) : DATEADD(DATETIME("now"), 1, day)'
   viz_type: dist_bar
   x_axis_label: Question Title
   x_ticks_layout: auto

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Transcripts_Captions_Per_Video.yaml

@@ -48,7 +48,6 @@ params:
   row_limit: 10000
   show_legend: true
   slice_id: 47
-  time_range: 'DATEADD(DATETIME("now"), -1, month) : DATEADD(DATETIME("now"), 1, day)'
   viz_type: dist_bar
   x_axis_label: Video Title
   x_ticks_layout: auto

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watched_Video_Segments.yaml

@@ -27,7 +27,6 @@ params:
   row_limit: 10000
   show_legend: true
   slice_id: 204
-  time_range: 'DATEADD(DATETIME("now"), -1, month) : DATEADD(DATETIME("now"), 1, day)'
   viz_type: dist_bar
   x_axis_label: Seconds Of Video
   x_ticks_layout: auto

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watches_Per_Video.yaml

@@ -49,7 +49,6 @@ params:
   row_limit: 10000
   show_legend: true
   slice_id: 25
-  time_range: 'DATEADD(DATETIME("now"), -1, month) : DATEADD(DATETIME("now"), 1, day)'
   viz_type: dist_bar
   x_axis_label: Video Title
   x_ticks_layout: auto