More schema 219 -> 300 plugin refactoring - structure control forms a…

…nd logic.

threedi_schematisation_editor/__init__.py

@@ -6,6 +6,9 @@
 from qgis.PyQt.QtGui import QCursor, QIcon
 from qgis.PyQt.QtWidgets import QAction, QComboBox, QDialog, QMenu
 
+from threedi_schematisation_editor.deps.custom_imports import patch_wheel_imports
+
+patch_wheel_imports()
 import threedi_schematisation_editor.data_models as dm
 from threedi_schematisation_editor.communication import UICommunication
 from threedi_schematisation_editor.custom_widgets import ImportStructuresDialog, LoadSchematisationDialog
@@ -17,10 +20,10 @@
     add_settings_entry,
     can_write_in_dir,
     check_enable_macros_option,
-    ensure_valid_schema,
     get_filepath,
     get_icon_path,
     is_gpkg_connection_exists,
+    migrate_schematisation_schema,
 )
 from threedi_schematisation_editor.workspace import WorkspaceContextManager
 
@@ -197,11 +200,6 @@ def check_macros_status(self):
             )
             self.uc.bar_warn(msg, dur=10)
 
-    def select_user_layers_geopackage(self):
-        name_filter = "3Di User Layers (*.gpkg *.GPKG)"
-        filename = get_filepath(self.iface.mainWindow(), extension_filter=name_filter, save=False)
-        return filename
-
     def on_3di_project_read(self):
         custom_vars = self.project.customVariables()
         try:
@@ -230,11 +228,19 @@ def load_schematisation(self, model_gpkg=None):
             result = schematisation_loader.exec_()
             if result != QDialog.Accepted:
                 return
-            model_gpkg = schematisation_loader.selected_schematisation_gpkg
-        if not can_write_in_dir(os.path.dirname(model_gpkg)):
-            warn_msg = "You don't have required write permissions to load data from the selected location."
-            self.uc.show_warn(warn_msg)
-            return
+            schematisation_filepath = schematisation_loader.selected_schematisation_filepath
+            if not can_write_in_dir(os.path.dirname(schematisation_filepath)):
+                warn_msg = "You don't have required write permissions to load data from the selected location."
+                self.uc.show_warn(warn_msg)
+                return
+            if schematisation_filepath.endswith(".sqlite"):
+                migration_succeed, migration_feedback_msg = migrate_schematisation_schema(schematisation_filepath)
+                if not migration_succeed:
+                    self.uc.show_warn(migration_feedback_msg)
+                    return
+                model_gpkg = schematisation_filepath.rsplit(".", 1)[0] + ".gpkg"
+            else:
+                model_gpkg = schematisation_filepath
         lm = LayersManager(self.iface, self.uc, model_gpkg)
         if lm in self.workspace_context_manager:
             warn_msg = "Selected schematisation is already loaded. Loading canceled."

threedi_schematisation_editor/custom_widgets/__init__.py

@@ -627,9 +627,9 @@ def __init__(self, uc, parent=None):
         self.working_dir = self.settings.value("threedi/working_dir", "", type=str)
         if self.working_dir:
             self.file_browse_widget.setDefaultRoot(self.working_dir)
-        self.selected_schematisation_gpkg = None
-        self.schematisation_tv.doubleClicked.connect(self.set_schematisation_geopackage_filepath)
-        self.load_pb.clicked.connect(self.set_schematisation_geopackage_filepath)
+        self.selected_schematisation_filepath = None
+        self.schematisation_tv.doubleClicked.connect(self.set_schematisation_filepath)
+        self.load_pb.clicked.connect(self.set_schematisation_filepath)
         self.cancle_pb.clicked.connect(self.reject)
         self.list_working_dir_schematisations()
 
@@ -685,7 +685,7 @@ def list_working_dir_schematisations(self):
             )
             self.schematisation_tv.scrollTo(last_used_schematisation_row_idx)
 
-    def set_schematisation_geopackage_filepath(self):
+    def set_schematisation_filepath(self):
         """Set selected schematisation filepath."""
         if self.load_tab.currentIndex() == 0:
             index = self.schematisation_tv.currentIndex()
@@ -695,11 +695,11 @@ def set_schematisation_geopackage_filepath(self):
             current_row = index.row()
             revision_item = self.schematisation_model.item(current_row, 1)
             revision_gpkg = revision_item.data(Qt.UserRole)
-            self.selected_schematisation_gpkg = revision_gpkg
+            self.selected_schematisation_filepath = revision_gpkg
         else:
             selected_filepath = self.file_browse_widget.filePath()
             if not selected_filepath:
                 self.uc.show_warn("No file selected. Please select schematisation file to continue.", parent=self)
                 return
-            self.selected_schematisation_gpkg = self.file_browse_widget.filePath()
+            self.selected_schematisation_filepath = self.file_browse_widget.filePath()
         self.accept()

threedi_schematisation_editor/custom_widgets/ui/load_schematisation.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>750</width>
-    <height>750</height>
+    <width>748</width>
+    <height>743</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -56,7 +56,7 @@
        <item row="1" column="1">
         <widget class="QgsFileWidget" name="file_browse_widget">
          <property name="filter">
-          <string>*.gpkg</string>
+          <string>*.gpkg *.sqlite</string>
          </property>
         </widget>
        </item>
@@ -76,7 +76,7 @@
        <item row="0" column="1">
         <widget class="QLabel" name="label">
          <property name="text">
-          <string>Path to the schematisation Geopackage file:</string>
+          <string>Path to the schematisation file:</string>
          </property>
         </widget>
        </item>

threedi_schematisation_editor/deps/custom_imports.py

@@ -0,0 +1,27 @@
+# Copyright (C) 2025 by Lutra Consulting
+import os
+import sys
+
+MAIN_DIR = os.path.dirname(os.path.abspath(__file__))
+REQUIRED_3DI_SCHEMA_VERSION = "0.230.0.dev0"
+REQUIRED_3DI_MI_UTILS_VERSION = "0.1.5"
+THREEDI_SCHEMA_WHEEL = os.path.join(MAIN_DIR, f"threedi_schema-{REQUIRED_3DI_SCHEMA_VERSION}-py3-none-any.whl")
+MI_UTILS_WHEEL = os.path.join(MAIN_DIR, f"threedi_mi_utils-{REQUIRED_3DI_MI_UTILS_VERSION}-py3-none-any.whl")
+
+
+def patch_wheel_imports():
+    """
+    Function that tests if extra modules are installed.
+    If modules are not available then it will add missing modules wheels to the Python path.
+    """
+    try:
+        import threedi_schema
+    except ImportError:
+        deps_path = THREEDI_SCHEMA_WHEEL
+        sys.path.append(deps_path)
+
+    try:
+        import threedi_mi_utils
+    except ImportError:
+        deps_path = MI_UTILS_WHEEL
+        sys.path.append(deps_path)

threedi_schematisation_editor/utils.py

@@ -53,14 +53,14 @@
 }
 
 
-def backup_geopackage(filename):
-    """Make a backup of the geopackage."""
+def backup_schematisation_file(filename):
+    """Make a backup of the schematisation file."""
     backup_folder = os.path.join(os.path.dirname(os.path.dirname(filename)), "_backup")
     os.makedirs(backup_folder, exist_ok=True)
     prefix = str(uuid4())[:8]
-    backup_gpkg_path = os.path.join(backup_folder, f"{prefix}_{os.path.basename(filename)}")
-    shutil.copyfile(filename, backup_gpkg_path)
-    return backup_gpkg_path
+    backup_file_path = os.path.join(backup_folder, f"{prefix}_{os.path.basename(filename)}")
+    shutil.copyfile(filename, backup_file_path)
+    return backup_file_path
 
 
 def vector_layer_factory(annotated_model_cls, epsg=4326):
@@ -690,25 +690,24 @@ def modify_raster_style(raster_layer, limits=QgsRasterMinMaxOrigin.MinMax, exten
     raster_layer.setRenderer(renderer)
 
 
-def migrate_geopackage_schema(gpkg_filepath):
-    # TODO: Needs refactoring
+def migrate_schematisation_schema(schematisation_filepath):
     migration_succeed = False
     try:
         from threedi_schema import ThreediDatabase, errors
 
-        threedi_db = ThreediDatabase(gpkg_filepath)
+        threedi_db = ThreediDatabase(schematisation_filepath)
         schema = threedi_db.schema
-        backup_filepath = backup_geopackage(gpkg_filepath)
-        schema.upgrade(backup=False, upgrade_schema_version=True)
-        schema.set_spatial_indexes()
+        backup_filepath = backup_schematisation_file(schematisation_filepath)
+        # schema.upgrade(backup=False, convert_to_geopackage=True)
+        schema.convert_to_geopackage()
         shutil.rmtree(os.path.dirname(backup_filepath))
         migration_succeed = True
         migration_feedback_msg = "Migration succeed."
     except ImportError:
-        migration_feedback_msg = "Missing threedi-schema library. Schema migration failed."
+        migration_feedback_msg = "Missing threedi-schema library (or its dependencies). Schema migration failed."
     except errors.UpgradeFailedError:
         migration_feedback_msg = (
-            "The geopackage database schema cannot be migrated to the current version. "
+            "The schematisation database schema cannot be migrated to the current version. "
             "Please contact the service desk for assistance."
         )
     except Exception as e:
@@ -738,60 +737,6 @@ def bypass_max_path_limit(path, is_file=False):
     return valid_path
 
 
-def ensure_valid_schema(schematisation_gpkg, communication):
-    """Check if schema version is up-to-date and migrate it if needed."""
-    # TODO: Needs refactoring
-    try:
-        from threedi_schema import ThreediDatabase, errors
-    except ImportError:
-        return
-    schematisation_dirname = os.path.dirname(schematisation_gpkg)
-    schematisation_filename = os.path.basename(schematisation_gpkg)
-    backup_folder = os.path.join(schematisation_dirname, "_backup")
-    os.makedirs(bypass_max_path_limit(backup_folder), exist_ok=True)
-    prefix = str(uuid4())[:8]
-    backup_gpkg_path = os.path.join(backup_folder, f"{prefix}_{schematisation_filename}")
-    shutil.copyfile(schematisation_gpkg, bypass_max_path_limit(backup_gpkg_path, is_file=True))
-    threedi_db = ThreediDatabase(schematisation_gpkg)
-    schema = threedi_db.schema
-    try:
-        schema.validate_schema()
-        schema.set_spatial_indexes()
-    except errors.MigrationMissingError:
-        warn_and_ask_msg = (
-            "The selected geopackage cannot be used because its database schema version is out of date. "
-            "Would you like to migrate your geopackage to the current schema version?"
-        )
-        do_migration = communication.ask(None, "Missing migration", warn_and_ask_msg)
-        if not do_migration:
-            return False
-        try:
-            schema.upgrade(backup=False, upgrade_schema_version=True)
-            schema.set_spatial_indexes()
-            shutil.rmtree(backup_folder)
-        except errors.MigrationMissingError as e:
-            if "This tool cannot update versions below 160" in str(e):
-                error_msg = (
-                    "This tool cannot update versions below 160. " "Please contact the service desk for assistance."
-                )
-                communication.show_error(error_msg)
-                return False
-            else:
-                raise e
-    except errors.UpgradeFailedError:
-        error_msg = (
-            "The geopackage database schema cannot be migrated to the current version. "
-            "Please contact the service desk for assistance."
-        )
-        communication.show_error(error_msg)
-        return False
-    except Exception as e:
-        error_msg = f"{e}"
-        communication.show_error(error_msg)
-        return False
-    return True
-
-
 def validation_errors_summary(validation_errors):
     """Create validation summary message grouped by the data model class."""
     summary_per_model = []
@@ -1059,3 +1004,54 @@ def extract_substring(linestring_geometry, start_distance, end_distance):
     before_start_geometry = QgsGeometry(before_start_substring)
     after_end_geometry = QgsGeometry(after_end_substring)
     return substring_geometry, before_start_geometry, after_end_geometry
+
+
+# def ensure_valid_schema(schematisation_filepath, communication):
+#     """Check if schema version is up-to-date and migrate it if needed."""
+#     try:
+#         from threedi_schema import ThreediDatabase, errors
+#     except ImportError:
+#         return
+#     schematisation_dirname = os.path.dirname(schematisation_filepath)
+#     schematisation_filename = os.path.basename(schematisation_filepath)
+#     backup_folder = os.path.join(schematisation_dirname, "_backup")
+#     os.makedirs(bypass_max_path_limit(backup_folder), exist_ok=True)
+#     prefix = str(uuid4())[:8]
+#     backup_filepath = os.path.join(backup_folder, f"{prefix}_{schematisation_filename}")
+#     shutil.copyfile(schematisation_filepath, bypass_max_path_limit(backup_filepath, is_file=True))
+#     threedi_db = ThreediDatabase(schematisation_filepath)
+#     schema = threedi_db.schema
+#     try:
+#         schema.validate_schema()
+#     except errors.MigrationMissingError:
+#         warn_and_ask_msg = (
+#             "The selected geopackage cannot be used because its database schema version is out of date. "
+#             "Would you like to migrate your geopackage to the current schema version?"
+#         )
+#         do_migration = communication.ask(None, "Missing migration", warn_and_ask_msg)
+#         if not do_migration:
+#             return False
+#         try:
+#             schema.upgrade(backup=False, upgrade_schema_version=True)
+#             shutil.rmtree(backup_folder)
+#         except errors.MigrationMissingError as e:
+#             if "This tool cannot update versions below 160" in str(e):
+#                 error_msg = (
+#                     "This tool cannot update versions below 160. " "Please contact the service desk for assistance."
+#                 )
+#                 communication.show_error(error_msg)
+#                 return False
+#             else:
+#                 raise e
+#     except errors.UpgradeFailedError:
+#         error_msg = (
+#             "The geopackage database schema cannot be migrated to the current version. "
+#             "Please contact the service desk for assistance."
+#         )
+#         communication.show_error(error_msg)
+#         return False
+#     except Exception as e:
+#         error_msg = f"{e}"
+#         communication.show_error(error_msg)
+#         return False
+#     return True