pre-process tables before dumping to parquet

NINAnor · Dec 10, 2024 · 5c3d89f · 5c3d89f
1 parent 5b0af51
commit 5c3d89f
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 1 deletion.
diff --git a/Makefile.dump b/Makefile.dump
@@ -30,7 +30,7 @@ yearly: $(yearly_path)
 
 
 monthly: $(monthly_path)
-	pg_restore -f - $(input) -t $(table_name) | pg_dedump -o $(monthly_path) -d m$(table_name)$(year)$(month).ddb
+	pg_restore -f - $(input) -t $(table_name) | pg_dedump -o $(monthly_path) -d m$(table_name)$(year)$(month).ddb --custom-sql-dir queries/dedump_export -r
 	rm m$(table_name)$(year)$(month).ddb
 
 

diff --git a/queries/dedump_export/radar.sql b/queries/dedump_export/radar.sql
@@ -0,0 +1,2 @@
+install spatial; load spatial;
+select * exclude(position), ST_GeomFromHEXEWKB(position) as position from radar
diff --git a/queries/dedump_export/track.sql b/queries/dedump_export/track.sql
@@ -0,0 +1,6 @@
+install spatial; load spatial;
+select 
+    * exclude(trajectory), 
+    ST_Force3DZ(ST_GeomFromHEXEWKB(trajectory), -99999) as trajectory,
+    list_transform(st_dump(st_points(ST_GeomFromHEXEWKB(trajectory))), str -> st_m(str.geom)) as m_dimension
+from track
diff --git a/shell.nix b/shell.nix
@@ -5,6 +5,7 @@ let
   unstable = import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/tarball/b69de56fac8c2b6f8fd27f2eca01dcda8e0a4221) {};
 in pkgs.mkShell rec {
   name = "impurePythonEnv";
+  LD_LIBRARY_PATH = "${stdenv.cc.cc.lib}/lib";
   venvDir = "./.venv";
   buildInputs = [
     unstable.python312Packages.venvShellHook
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		install spatial; load spatial;
		select * exclude(position), ST_GeomFromHEXEWKB(position) as position from radar