Merge pull request #213 from GeoNode/ISSUE_212

Fixes #212 PG_USE_COPY should be optional
GeoNode · Nov 13, 2023 · 2d8ae57 · 2d8ae57
2 parents c2a4c1e + 25489c1
commit 2d8ae57
Show file tree

Hide file tree

Showing 3 changed files with 44 additions and 3 deletions.
diff --git a/README.md b/README.md
@@ -128,6 +128,10 @@ IMPORTER_GLOBAL_RATE_LIMIT= # default 5
 IMPORTER_PUBLISHING_RATE_LIMIT= # default 5
 IMPORTER_RESOURCE_CREATION_RATE_LIMIT= # default 10
 IMPORTER_RESOURCE_COPY_RATE_LIMIT = # default 10
+
+# https://gdal.org/drivers/vector/pg.html#configuration-options
+DISABLE_PG_COPY_OGR2OGR= # If TRUE disable the usage of COPY during the import in postgres. Disable it can affect the performance 
+
 ```
 
 ## Troubleshooting

diff --git a/importer/handlers/common/tests_vector.py b/importer/handlers/common/tests_vector.py
@@ -1,3 +1,4 @@
+import os
 import uuid
 from celery.canvas import Signature
 from celery import group
@@ -225,7 +226,7 @@ def test_import_with_ogr2ogr_without_errors_should_call_the_right_command(
 
         _open.assert_called_once()
         _open.assert_called_with(
-            f'/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:" dbname=\'geonode_data\' host=localhost port=5434 user=\'geonode\' password=\'geonode\' " "{self.valid_files.get("base_file")}" -lco DIM=2 -nln alternate "dataset"',
+            f'/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:" dbname=\'geonode_data\' host=localhost port=5434 user=\'geonode\' password=\'geonode\' " "{self.valid_files.get("base_file")}" -nln alternate "dataset"',
             stdout=-1,
             stderr=-1,
             shell=True,  # noqa
@@ -251,7 +252,39 @@ def test_import_with_ogr2ogr_with_errors_should_raise_exception(self, _open):
 
         _open.assert_called_once()
         _open.assert_called_with(
-            f'/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:" dbname=\'geonode_data\' host=localhost port=5434 user=\'geonode\' password=\'geonode\' " "{self.valid_files.get("base_file")}" -lco DIM=2 -nln alternate "dataset"',
+            f'/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:" dbname=\'geonode_data\' host=localhost port=5434 user=\'geonode\' password=\'geonode\' " "{self.valid_files.get("base_file")}" -nln alternate "dataset"',
+            stdout=-1,
+            stderr=-1,
+            shell=True,  # noqa
+        )
+
+    @patch.dict(os.environ, {"DISABLE_PG_COPY_OGR2OGR": "True"}, clear=True)
+    @patch("importer.handlers.common.vector.Popen")
+    def test_import_with_ogr2ogr_without_errors_should_call_the_right_command_if_copy_is_disabled(
+        self, _open
+    ):
+        _uuid = uuid.uuid4()
+
+        comm = MagicMock()
+        comm.communicate.return_value = b"", b""
+        _open.return_value = comm
+
+        _task, alternate, execution_id = import_with_ogr2ogr(
+            execution_id=str(_uuid),
+            files=self.valid_files,
+            original_name="dataset",
+            handler_module_path=str(self.handler),
+            ovverwrite_layer=False,
+            alternate="alternate",
+        )
+
+        self.assertEqual("ogr2ogr", _task)
+        self.assertEqual(alternate, "alternate")
+        self.assertEqual(str(_uuid), execution_id)
+
+        _open.assert_called_once()
+        _open.assert_called_with(
+            f'/usr/bin/ogr2ogr --config PG_USE_COPY NO -f PostgreSQL PG:" dbname=\'geonode_data\' host=localhost port=5434 user=\'geonode\' password=\'geonode\' " "{self.valid_files.get("base_file")}" -nln alternate "dataset"',
             stdout=-1,
             stderr=-1,
             shell=True,  # noqa

diff --git a/importer/handlers/common/vector.py b/importer/handlers/common/vector.py
@@ -1,3 +1,4 @@
+import ast
 from django.db import connections
 from importer.publisher import DataPublisher
 from importer.utils import call_rollback_function, find_key_recursively
@@ -161,7 +162,10 @@ def create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate):
             db_host, db_port = db_host.split(":")
         db_name = _uri.split("@")[1].split("/")[1]
 
-        options = "--config PG_USE_COPY YES "
+
+        disable_pg_copy = ast.literal_eval(os.getenv("DISABLE_PG_COPY_OGR2OGR", "False"))
+        options = f"--config PG_USE_COPY {'NO' if disable_pg_copy else 'YES'} "
+
         options += (
             "-f PostgreSQL PG:\" dbname='%s' host=%s port=%s user='%s' password='%s' \" "
             % (db_name, db_host, db_port, db_user, db_password)