feat: allow to translate dataset text on chart

.github/workflows/integration-test.yml

@@ -2,7 +2,6 @@ name: Tutor Integration Test
 
 on:
   pull_request:
-    branches: [ main ]
     paths-ignore:
       - .github/**
       - .ci/**

scripts/translate_utils.py

@@ -86,12 +86,13 @@ class DatasetAsset(TranslatableAsset):
     "table_name": ("datasets", DatasetAsset),
 }
 
+BASE_PATH = "tutoraspects/templates/aspects/build/aspects-superset/"
 
 def get_text_for_translations(root_path):
     assets_path = (
         os.path.join(
             root_path,
-            "tutoraspects/templates/aspects/build/aspects-superset/"
+            BASE_PATH,
             "openedx-assets/assets/"
         )
     )
@@ -112,6 +113,11 @@ def get_text_for_translations(root_path):
             asset = yaml.safe_load(asset_str)
             strings.extend(mark_text_for_translation(asset))
 
+    with open(BASE_PATH + "localization/datasets_strings.yaml", 'r') as file:
+        dataset_strings = yaml.safe_load(file.read())
+        for key in dataset_strings:
+            strings.extend(dataset_strings[key])
+            print(f"Extracted {len(dataset_strings[key])} strings for dataset {key}")
     return strings
 
 

tutoraspects/asset_command_helpers.py

@@ -53,6 +53,7 @@ class Asset:
     templated_vars = None
     required_vars = None
     omitted_vars = None
+    raw_vars = None
 
     def __init__(self):
         if not self.path:
@@ -92,6 +93,12 @@ def get_omitted_vars(self):
         """
         return self.omitted_vars or []
 
+    def get_raw_vars(self):
+        """
+        Returns a list of variables which should be omitted from the content.
+        """
+        return self.raw_vars or []
+
     def remove_content(self, content: dict):
         """
         Remove any variables from the content which should be omitted.
@@ -116,15 +123,31 @@ def omit_templated_vars(self, content: dict, existing: dict):
         Omit templated variables from the content if they are not present in
         the existing file content.
         """
-        if existing:
-            for key in content.keys():
-                if key not in existing.keys():
-                    continue
-                if isinstance(existing[key], str):
-                    if "{{" in existing.get(key, "") or "{%" in existing.get(key, ""):
+        if not existing:
+            return
+
+        for key in content.keys():
+            if key not in existing.keys():
+                continue
+            if isinstance(existing[key], str):
+                if "{{" in existing.get(key, "") or "{%" in existing.get(key, ""):
+                    if key in self.get_raw_vars():
+                        raw_expression = "{% raw %}" + content[key] + "{% endraw %}"
+                        content[key] = raw_expression
+                    else:
                         content[key] = existing[key]
-                if isinstance(existing[key], dict):
-                    self.omit_templated_vars(content[key], existing[key])
+
+            if isinstance(existing[key], dict):
+                self.omit_templated_vars(content[key], existing[key])
+
+            if isinstance(existing[key], list):
+                for i, item in enumerate(content[key]):
+                    if isinstance(item, dict):
+                        try:
+                            tmp = existing[key][i]
+                            self.omit_templated_vars(item, tmp or None)
+                        except IndexError:
+                            pass
 
 
 class ChartAsset(Asset):
@@ -133,7 +156,13 @@ class ChartAsset(Asset):
     """
 
     path = "charts"
-    omitted_vars = ["query_context"]
+    omitted_vars = [
+        "query_context",
+        "params.dashboards",
+        "params.datasource",
+        "params.slice_id",
+    ]
+    raw_vars = ["sqlExpression"]
 
 
 class DashboardAsset(Asset):

tutoraspects/patches/local-docker-compose-dev-services

@@ -3,7 +3,8 @@ superset:
   {% include 'base-docker-compose-services' %}
     OPENEDX_LMS_ROOT_URL: "http://{{ LMS_HOST }}:8000"
     OAUTH2_CLIENT_ID: {{ SUPERSET_OAUTH2_CLIENT_ID_DEV }}
-  command: ["bash", "/app/docker/docker-bootstrap.sh", "app-gunicorn"]
+    SUPERSET_ENV: development
+  command: ["bash", "/app/docker/docker-bootstrap.sh", "app"]
   ports:
     - 8088:{{ SUPERSET_PORT }}
   depends_on:
@@ -16,6 +17,7 @@ superset-worker:
   {% include 'base-docker-compose-services' %}
     OPENEDX_LMS_ROOT_URL: "http://{{ LMS_HOST }}:8000"
     OAUTH2_CLIENT_ID: {{ SUPERSET_OAUTH2_CLIENT_ID_DEV }}
+    SUPERSET_ENV: development
   command: ["bash", "/app/docker/docker-bootstrap.sh", "worker"]
   healthcheck:
     test: ["CMD-SHELL", "celery inspect ping -A superset.tasks.celery_app:app -d celery@$$HOSTNAME"]
@@ -27,6 +29,7 @@ superset-worker-beat:
   {% include 'base-docker-compose-services' %}
     OPENEDX_LMS_ROOT_URL: "http://{{ LMS_HOST }}:8000"
     OAUTH2_CLIENT_ID: {{ SUPERSET_OAUTH2_CLIENT_ID_DEV }}
+    SUPERSET_ENV: development
   command: ["bash", "/app/docker/docker-bootstrap.sh", "beat"]
   healthcheck:
     disable: true

tutoraspects/patches/local-docker-compose-services

@@ -48,6 +48,7 @@ superset:
   {% include 'base-docker-compose-services' %}
     OPENEDX_LMS_ROOT_URL: "{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://{{ LMS_HOST }}"
     OAUTH2_CLIENT_ID: {{ SUPERSET_OAUTH2_CLIENT_ID }}
+    SUPERSET_ENV: production
   command: ["bash", "/app/docker/docker-bootstrap.sh", "app-gunicorn"]
   ports:
     - 8088:{{ SUPERSET_PORT }}
@@ -61,6 +62,7 @@ superset-worker:
   {% include 'base-docker-compose-services' %}
     OPENEDX_LMS_ROOT_URL: "{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://{{ LMS_HOST }}"
     OAUTH2_CLIENT_ID: {{ SUPERSET_OAUTH2_CLIENT_ID }}
+    SUPERSET_ENV: production
   command: ["bash", "/app/docker/docker-bootstrap.sh", "worker"]
   healthcheck:
     test: ["CMD-SHELL", "celery inspect ping -A superset.tasks.celery_app:app -d celery@$$HOSTNAME"]
@@ -72,6 +74,7 @@ superset-worker-beat:
   {% include 'base-docker-compose-services' %}
     OPENEDX_LMS_ROOT_URL: "{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://{{ LMS_HOST }}"
     OAUTH2_CLIENT_ID: {{ SUPERSET_OAUTH2_CLIENT_ID }}
+    SUPERSET_ENV: production
   command: ["bash", "/app/docker/docker-bootstrap.sh", "beat"]
   healthcheck:
     disable: true

tutoraspects/templates/aspects/apps/superset/docker/docker-bootstrap.sh

@@ -47,7 +47,7 @@ elif [[ "${1}" == "beat" ]]; then
   celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid -l INFO -s "${SUPERSET_HOME}"/celerybeat-schedule
 elif [[ "${1}" == "app" ]]; then
   echo "Starting web app..."
-  flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0
+  flask run -p 8088 --with-threads --reload --debugger --debug --host=0.0.0.0
 elif [[ "${1}" == "app-gunicorn" ]]; then
   echo "Starting web app..."
   /usr/bin/run-server.sh

tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py

@@ -17,6 +17,7 @@
 from superset.models.dashboard import Dashboard
 from superset.utils.database import get_or_create_db
 from superset.models.embedded_dashboard import EmbeddedDashboard
+from pythonpath.localization import get_translation
 
 BASE_DIR = "/app/assets/superset"
 
@@ -34,25 +35,8 @@
 
 FILE_NAME_ATTRIBUTE = "_file_name"
 
-TRANSLATIONS_FILE_PATH = "/app/localization/locale.yaml"
 ASSETS_FILE_PATH = "/app/pythonpath/assets.yaml"
-ASSETS_PATH = "/app/openedx-assets"
-
-merged_data = {}
-with open(TRANSLATIONS_FILE_PATH, "r") as file:
-    yaml_content = file.read()
-    yaml_documents = yaml_content.split("\n---\n")
-
-    for doc in yaml_documents:
-        data = yaml.safe_load(doc)
-        if data is not None:
-            for lang, translations in data.items():
-                if lang not in merged_data:
-                    merged_data[lang] = {}
-                merged_data[lang].update(translations)
-
-
-ASSETS_TRANSLATIONS = merged_data
+ASSETS_PATH = "/app/openedx-assets/assets"
 
 
 def main():
@@ -297,14 +281,6 @@ def update_embeddable_uuids():
         db.session.add(embedded_dashboard)
         db.session.commit()
 
-def get_translation(text, language):
-    """Get a translation for a text in a language"""
-    default_text = f"{text}"
-    LANGUAGE = ASSETS_TRANSLATIONS.get(language, {})
-    if not LANGUAGE:
-        return default_text
-    return ASSETS_TRANSLATIONS.get(language, {}).get(text) or default_text
-
 
 if __name__ == "__main__":
     main()

tutoraspects/templates/aspects/apps/superset/pythonpath/localization.py

@@ -0,0 +1,33 @@
+"""
+This file is used to load translations from the locale.yaml file and provide a function to get translations
+"""
+
+import yaml
+
+TRANSLATIONS_FILE_PATH = "/app/localization/locale.yaml"
+DATASET_STRINGS_FILE_PATH = "/app/localization/datasets_strings.yaml"
+
+ASSETS_TRANSLATIONS = {}
+with open(TRANSLATIONS_FILE_PATH, "r") as file:
+    yaml_content = file.read()
+    yaml_documents = yaml_content.split("\n---\n")
+
+    for doc in yaml_documents:
+        data = yaml.safe_load(doc)
+        if data is not None:
+            for lang, translations in data.items():
+                if lang not in ASSETS_TRANSLATIONS:
+                    ASSETS_TRANSLATIONS[lang] = {}
+                ASSETS_TRANSLATIONS[lang].update(translations)
+
+
+DATASET_STRINGS = yaml.safe_load(open(DATASET_STRINGS_FILE_PATH, "r"))
+
+
+def get_translation(text, language):
+    """Get a translation for a text in a language"""
+    default_text = f"{text}"
+    LANGUAGE = ASSETS_TRANSLATIONS.get(language, {})
+    if not LANGUAGE:
+        return default_text
+    return ASSETS_TRANSLATIONS.get(language, {}).get(text) or default_text

tutoraspects/templates/aspects/apps/superset/pythonpath/openedx_jinja_filters.py

@@ -3,8 +3,14 @@
 
 cf https://superset.apache.org/docs/installation/sql-templating/
 """
+
 from superset.extensions import security_manager
+from pythonpath.localization import get_translation, DATASET_STRINGS
+from superset import security_manager
+import logging
+from flask import g
 
+log = logging.getLogger(__name__)
 ALL_COURSES = "1 = 1"
 NO_COURSES = "1 = 0"
 
@@ -44,4 +50,27 @@ def can_view_courses(username, field_name="course_id", **kwargs):
         return NO_COURSES
 
 
+def translate_column(column_name):
+    """
+    Translate a string to the given language.
+    """
+    lang = security_manager.get_preferences(g.user.username)
+
+    strings = DATASET_STRINGS.get(column_name, [])
+    if not strings:
+        return column_name
+    case_format = """CASE \n {cases} \n ELSE {column_name} \n END"""
+    single_case_format = "WHEN {column_name} = '{string}' THEN '{translation}'"
+    cases = "\n".join(
+        single_case_format.format(
+            column_name=column_name,
+            string=string,
+            translation=get_translation(string, lang),
+        )
+        for string in strings
+    )
+
+    return case_format.format(column_name=column_name, cases=cases)
+
+
 {{patch("superset-jinja-filters")}}

tutoraspects/templates/aspects/apps/superset/pythonpath/openedx_sso_security_manager.py

@@ -57,25 +57,27 @@ def decoded_user_info(self):
             self.access_token, algorithms=["HS256"], options={"verify_signature": False}
         )
 
-    def oauth_user_info(self, provider, response=None):
-        if provider == "openedxsso":
-            user_profile = self.decoded_user_info()
-
-            openedx_apis = current_app.config["OPENEDX_API_URLS"]
-            url = openedx_apis["get_preference"].format(
-                username=user_profile["preferred_username"]
-            )
-            oauth_remote = self.oauth_remotes.get(provider)
-
-            response = oauth_remote.get(url).json()
-            locale_preference = response.get("pref-lang", "en").replace("-", "_")
+    def get_preferences(self, username):
+        openedx_apis = current_app.config["OPENEDX_API_URLS"]
+        url = openedx_apis["get_preference"].format(username=username)
+        oauth_remote = self.oauth_remotes.get("openedxsso")
+        response = oauth_remote.get(url, token=self.get_oauth_token()).json()
+        locale_preference = response.get("pref-lang", "en").replace("-", "_")
 
-            if locale_preference not in current_app.config["DASHBOARD_LOCALES"]:
+        if locale_preference not in current_app.config["DASHBOARD_LOCALES"]:
                 log.warning(
                     f"Language {locale_preference} is not supported by Superset"
                 )
                 locale_preference = "en"
 
+        return locale_preference
+
+    def oauth_user_info(self, provider, response=None):
+        if provider == "openedxsso":
+            user_profile = self.decoded_user_info()
+
+            locale_preference = self.get_preferences(user_profile["preferred_username"])
+
             user_roles = self._get_user_roles(
                 user_profile.get("preferred_username"), locale_preference
             )

tutoraspects/templates/aspects/apps/superset/pythonpath/superset_config_docker.py

@@ -108,6 +108,7 @@ def can_view_courses_wrapper(*args, **kwargs):
 
 JINJA_CONTEXT_ADDONS = {
     'can_view_courses': can_view_courses_wrapper,
+    'translate_column': translate_column,
     {% for filter in SUPERSET_EXTRA_JINJA_FILTERS %}'{{ filter }}': {{filter}},{% endfor %}
 }
 

tutoraspects/templates/aspects/apps/superset/scripts/import-assets.sh

@@ -1,4 +1,4 @@
-
+mkdir -p /app/assets/
 cd /app/assets/
 rm -rf superset
 

tutoraspects/templates/aspects/build/aspects-superset/localization/datasets_strings.yaml

@@ -0,0 +1,7 @@
+enrollment_mode:
+- audit
+- verified
+- honor
+enrollment_status:
+- registered
+- unregistered

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Grade_Distribution.yaml

@@ -18,14 +18,13 @@ params:
   groupby:
   - grade_bucket
   metrics:
-  - count
+  - students
   order_bars: true
   order_desc: true
   rich_tooltip: true
   row_limit: 10000
   show_legend: false
   slice_id: 352
-  time_range: 'DATEADD(DATETIME("now"), -1, month) : DATEADD(DATETIME("now"), 1, day)'
   viz_type: dist_bar
   x_axis_label: Course Grade (out of 100%)
   x_ticks_layout: auto

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Attempts.yaml

@@ -27,14 +27,13 @@ params:
   groupby:
   - attempts
   metrics:
-  - count
+  - students
   order_bars: true
   order_desc: true
   rich_tooltip: true
   row_limit: 10000
   show_legend: false
   slice_id: 30
-  time_range: 'DATEADD(DATETIME("now"), -1, month) : DATEADD(DATETIME("now"), 1, day)'
   viz_type: dist_bar
   x_axis_label: Number Of Attempts To Find Correct Answer
   x_ticks_layout: auto