fix: add support to optionally append to the mbtiles file (#255)

* fix: Actually write the CSV file for CSVDump.py * fix: Add option to optionally append to an existing mbtiles database * fix: Optionally append to an existing mbtiles database * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: Rob Savoye <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
hotosm · May 24, 2024 · 496dd4e · 496dd4e
1 parent 4e7ec02
commit 496dd4e
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 5 deletions.
diff --git a/osm_fieldwork/basemapper.py b/osm_fieldwork/basemapper.py
@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 
-# Copyright (c) 2020, 2021, 2022 Humanitarian OpenStreetMap Team
+# Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team
 #
 # This file is part of OSM-Fieldwork.
 #
@@ -439,6 +439,7 @@ def create_basemap_file(
     zooms="12-17",
     outdir=None,
     source="esri",
+    append: bool = False,
 ) -> None:
     """Create a basemap with given parameters.
 
@@ -453,6 +454,7 @@ def create_basemap_file(
         outdir (str, optional): Output directory name for tile cache.
         source (str, optional): Imagery source, one of
             ["esri", "bing", "topo", "google", "oam"] (default is "esri").
+        append (bool, optional): Whether to append to an existing file
 
     Returns:
         None
@@ -528,7 +530,7 @@ def create_basemap_file(
     log.debug(f"Basemap output format: {suffix}")
 
     if any(substring in suffix for substring in ["sqlite", "mbtiles"]):
-        outf = DataFile(outfile, basemap.getFormat())
+        outf = DataFile(outfile, basemap.getFormat(), append)
         if suffix == ".mbtiles":
             outf.addBounds(basemap.bbox)
         # Create output database and specify image format, png, jpg, or tif
@@ -564,6 +566,7 @@ def main():
     )
     parser.add_argument("-z", "--zooms", default="12-17", help="The Zoom levels")
     parser.add_argument("-d", "--outdir", help="Output directory name for tile cache")
+    parser.add_argument("-a", "--append", action="store_true", default=False, help="Append to an existing database file")
     parser.add_argument(
         "-s",
         "--source",
@@ -617,6 +620,7 @@ def main():
         zooms=args.zooms,
         outdir=args.outdir,
         source=args.source,
+        append=args.append,
     )
 
 

diff --git a/osm_fieldwork/sqlite.py b/osm_fieldwork/sqlite.py
@@ -106,20 +106,22 @@ def __init__(
         self,
         dbname: str = None,
         suffix: str = "jpg",
+        append: bool = False,
     ):
         """Handle the sqlite3 database file.
 
         Args:
             dbname (str): The name of the output sqlite file
             suffix (str): The image suffix, jpg or png usually
+            append (bool): Whether to append to or create the database
 
         Returns:
             (DataFile): An instance of this class
         """
         self.db = None
         self.cursor = None
         if dbname:
-            self.createDB(dbname)
+            self.createDB(dbname, append)
         self.dbname = dbname
         self.metadata = None
         self.toplevel = None
@@ -136,13 +138,14 @@ def addBounds(
         """
         entry = str(bounds)
         entry = entry[1 : len(entry) - 1].replace(" ", "")
-        self.cursor.execute(f"INSERT INTO metadata (name, value) VALUES('bounds', '{entry}')")
+        self.cursor.execute(f"INSERT OR IGNORE INTO metadata (name, value) VALUES('bounds', '{entry}') ")
         # self.cursor.execute(f"INSERT INTO metadata (name, value) VALUES('minzoom', '9')")
         # self.cursor.execute(f"INSERT INTO metadata (name, value) VALUES('maxzoom', '15')")
 
     def createDB(
         self,
         dbname: str,
+        append: bool = False,
     ):
         """Create and sqlitedb in either mbtiles or Osman sqlitedb format.
 
@@ -151,11 +154,17 @@ def createDB(
         """
         suffix = os.path.splitext(dbname)[1]
 
-        if os.path.exists(dbname):
+        if os.path.exists(dbname) and append == False:
             os.remove(dbname)
 
         self.db = sqlite3.connect(dbname)
         self.cursor = self.db.cursor()
+        self.cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='tiles'")
+        exists = self.cursor.fetchone()
+        if exists and append:
+            logging.info("Appending to database file %s" % dbname)
+            return
+
         if suffix == ".mbtiles":
             self.cursor.execute("CREATE TABLE tiles (zoom_level integer, tile_column integer, tile_row integer, tile_data blob)")
             self.cursor.execute("CREATE INDEX tiles_idx on tiles (zoom_level, tile_column, tile_row)")