From 5c3d89ff7aa6da5a0146bc89526632623b991b72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niccol=C3=B2=20Cant=C3=B9?= <niccolo.cantu@nina.no>
Date: Tue, 10 Dec 2024 13:56:39 +0100
Subject: [PATCH] pre-process tables before dumping to parquet

---
 Makefile.dump                   | 2 +-
 queries/dedump_export/radar.sql | 2 ++
 queries/dedump_export/track.sql | 6 ++++++
 shell.nix                       | 1 +
 4 files changed, 10 insertions(+), 1 deletion(-)
 create mode 100644 queries/dedump_export/radar.sql
 create mode 100644 queries/dedump_export/track.sql

diff --git a/Makefile.dump b/Makefile.dump
index 0bcdb4f..fc70483 100644
--- a/Makefile.dump
+++ b/Makefile.dump
@@ -30,7 +30,7 @@ yearly: $(yearly_path)
 
 
 monthly: $(monthly_path)
-	pg_restore -f - $(input) -t $(table_name) | pg_dedump -o $(monthly_path) -d m$(table_name)$(year)$(month).ddb
+	pg_restore -f - $(input) -t $(table_name) | pg_dedump -o $(monthly_path) -d m$(table_name)$(year)$(month).ddb --custom-sql-dir queries/dedump_export -r
 	rm m$(table_name)$(year)$(month).ddb
 
 
diff --git a/queries/dedump_export/radar.sql b/queries/dedump_export/radar.sql
new file mode 100644
index 0000000..e958c91
--- /dev/null
+++ b/queries/dedump_export/radar.sql
@@ -0,0 +1,2 @@
+install spatial; load spatial;
+select * exclude(position), ST_GeomFromHEXEWKB(position) as position from radar
\ No newline at end of file
diff --git a/queries/dedump_export/track.sql b/queries/dedump_export/track.sql
new file mode 100644
index 0000000..ef71040
--- /dev/null
+++ b/queries/dedump_export/track.sql
@@ -0,0 +1,6 @@
+install spatial; load spatial;
+select 
+    * exclude(trajectory), 
+    ST_Force3DZ(ST_GeomFromHEXEWKB(trajectory), -99999) as trajectory,
+    list_transform(st_dump(st_points(ST_GeomFromHEXEWKB(trajectory))), str -> st_m(str.geom)) as m_dimension
+from track
\ No newline at end of file
diff --git a/shell.nix b/shell.nix
index f3ca76e..68f1fb7 100644
--- a/shell.nix
+++ b/shell.nix
@@ -5,6 +5,7 @@ let
   unstable = import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/tarball/b69de56fac8c2b6f8fd27f2eca01dcda8e0a4221) {};
 in pkgs.mkShell rec {
   name = "impurePythonEnv";
+  LD_LIBRARY_PATH = "${stdenv.cc.cc.lib}/lib";
   venvDir = "./.venv";
   buildInputs = [
     unstable.python312Packages.venvShellHook