diff --git a/Tutorial.ipynb b/Tutorial.ipynb index 95be370..8e27639 100644 --- a/Tutorial.ipynb +++ b/Tutorial.ipynb @@ -72,6 +72,7 @@ "\n", "# To silence a deprecation warning.\n", "gt.gdal.UseExceptions()\n", + "generateDataFromScratch=False\n", "# You have have successfully prepared your environment.\n", "print(\"You have successfully prepared your environment.\")" ] @@ -114,24 +115,27 @@ "metadata": {}, "outputs": [], "source": [ - "download_list = \"./download_urls.txt\" # Where the list of download links will be stored\n", - "root_output_folder = \"./files/tif_files/\" # root folder where geotiled will store data\n", - "n_tiles = 4 # Number of tiles that are generated for parameter computation\n", - "dem_tiles_dir_name = \"tiles\" # Folder where downloaded DEM tiles will be saved\n", - "param_tiles_dir_name = (\n", - " \"elevation_tiles\" # Folder where computation tiles will be saved.\n", - ")\n", - "gcs_name = \"gcs.tif\" # Name for the mosaicked DEM\n", - "pcs_name = \"pcs.tif\" # Name for the projected DEM\n", - "shapefile = [\"./files/shape_files/STATEFP_47.shp\"] # Shapefile for Visualization\n", - "region_bounding_box = {\n", - " \"xmin\": -90.4,\n", - " \"ymin\": 34.8,\n", - " \"xmax\": -81.55,\n", - " \"ymax\": 36.8,\n", - "} # For `fetch_dem`. X=Longitude Y=Latitude. Determine bounding coordinates by looking at a map.\n", - "\n", - "print(\"You have successfully initialized the GEOtiled variables.\")" + "if generateDataFromScratch:\n", + " download_list = \"./download_urls.txt\" # Where the list of download links will be stored\n", + " root_output_folder = \"./files/tif_files/\" # root folder where geotiled will store data\n", + " n_tiles = 4 # Number of tiles that are generated for parameter computation\n", + " dem_tiles_dir_name = \"tiles\" # Folder where downloaded DEM tiles will be saved\n", + " param_tiles_dir_name = (\n", + " \"elevation_tiles\" # Folder where computation tiles will be saved.\n", + " )\n", + " gcs_name = \"gcs.tif\" # Name for the mosaicked DEM\n", + " pcs_name = \"pcs.tif\" # Name for the projected DEM\n", + " shapefile = [\"./files/shape_files/STATEFP_47.shp\"] # Shapefile for Visualization\n", + " region_bounding_box = {\n", + " \"xmin\": -90.4,\n", + " \"ymin\": 34.8,\n", + " \"xmax\": -81.55,\n", + " \"ymax\": 36.8,\n", + " } # For `fetch_dem`. X=Longitude Y=Latitude. Determine bounding coordinates by looking at a map.\n", + " \n", + " print(\"You have successfully initialized the GEOtiled variables.\")\n", + "else:\n", + " print('Skipping this cell...')" ] }, { @@ -151,33 +155,37 @@ "metadata": {}, "outputs": [], "source": [ - "# Fetching Data\n", - "tiles_folder = os.path.join(root_output_folder, dem_tiles_dir_name)\n", - "Path(root_output_folder).mkdir(parents=True, exist_ok=True)\n", - "Path(tiles_folder).mkdir(parents=True, exist_ok=True)\n", - "\n", - "# Setting up for parameter computation\n", - "gcs = os.path.join(root_output_folder, gcs_name)\n", - "pcs = os.path.join(root_output_folder, pcs_name)\n", - "elevation_tiles = os.path.join(root_output_folder, param_tiles_dir_name)\n", - "Path(elevation_tiles).mkdir(parents=True, exist_ok=True)\n", - "\n", - "# Computing Parameters\n", - "aspect_tiles = os.path.join(root_output_folder, \"aspect_tiles\")\n", - "hillshading_tiles = os.path.join(root_output_folder, \"hillshading_tiles\")\n", - "slope_tiles = os.path.join(root_output_folder, \"slope_tiles\")\n", - "Path(aspect_tiles).mkdir(parents=True, exist_ok=True)\n", - "Path(hillshading_tiles).mkdir(parents=True, exist_ok=True)\n", - "Path(slope_tiles).mkdir(parents=True, exist_ok=True)\n", - "\n", - "gt.fetch_dem(\n", - " bbox=region_bounding_box,\n", - " txtPath=download_list,\n", - " dataset=\"National Elevation Dataset (NED) 1 arc-second Current\",\n", - ")\n", - "gt.download_files(download_list, tiles_folder)\n", - "\n", - "print(\"You have successfully downloaded the DEMs from USGS.\")" + "if generateDataFromScratch:\n", + "\n", + " # Fetching Data\n", + " tiles_folder = os.path.join(root_output_folder, dem_tiles_dir_name)\n", + " Path(root_output_folder).mkdir(parents=True, exist_ok=True)\n", + " Path(tiles_folder).mkdir(parents=True, exist_ok=True)\n", + " \n", + " # Setting up for parameter computation\n", + " gcs = os.path.join(root_output_folder, gcs_name)\n", + " pcs = os.path.join(root_output_folder, pcs_name)\n", + " elevation_tiles = os.path.join(root_output_folder, param_tiles_dir_name)\n", + " Path(elevation_tiles).mkdir(parents=True, exist_ok=True)\n", + " \n", + " # Computing Parameters\n", + " aspect_tiles = os.path.join(root_output_folder, \"aspect_tiles\")\n", + " hillshading_tiles = os.path.join(root_output_folder, \"hillshading_tiles\")\n", + " slope_tiles = os.path.join(root_output_folder, \"slope_tiles\")\n", + " Path(aspect_tiles).mkdir(parents=True, exist_ok=True)\n", + " Path(hillshading_tiles).mkdir(parents=True, exist_ok=True)\n", + " Path(slope_tiles).mkdir(parents=True, exist_ok=True)\n", + " \n", + " gt.fetch_dem(\n", + " bbox=region_bounding_box,\n", + " txtPath=download_list,\n", + " dataset=\"National Elevation Dataset (NED) 1 arc-second Current\",\n", + " )\n", + " gt.download_files(download_list, tiles_folder)\n", + " \n", + " print(\"You have successfully downloaded the DEMs from USGS.\")\n", + "else:\n", + " print('Skipping this cell...')" ] }, { @@ -197,22 +205,25 @@ "metadata": {}, "outputs": [], "source": [ - "raster_list = glob.glob(tiles_folder + \"/*\")\n", - "\n", - "gt.build_mosaic(raster_list, gcs)\n", - "\n", - "shutil.rmtree(tiles_folder)\n", - "os.remove(\"./merged.vrt\")\n", - "\n", - "gt.reproject(gcs, pcs, \"EPSG:9822\")\n", - "\n", - "os.remove(gcs)\n", - "\n", - "gt.crop_into_tiles(pcs, elevation_tiles, n_tiles)\n", - "\n", - "glob_of_tiles = glob.glob(elevation_tiles + \"/*.tif\")\n", - "\n", - "print(\"You have successfully processed the DEMs using GEOtiled\")" + "if generateDataFromScratch:\n", + " raster_list = glob.glob(tiles_folder + \"/*\")\n", + " \n", + " gt.build_mosaic(raster_list, gcs)\n", + " \n", + " shutil.rmtree(tiles_folder)\n", + " os.remove(\"./merged.vrt\")\n", + " \n", + " gt.reproject(gcs, pcs, \"EPSG:9822\")\n", + " \n", + " os.remove(gcs)\n", + " \n", + " gt.crop_into_tiles(pcs, elevation_tiles, n_tiles)\n", + " \n", + " glob_of_tiles = glob.glob(elevation_tiles + \"/*.tif\")\n", + " \n", + " print(\"You have successfully processed the DEMs using GEOtiled\")\n", + "else:\n", + " print('Skipping this cell...')" ] }, { @@ -232,29 +243,33 @@ "metadata": {}, "outputs": [], "source": [ - "pool = multiprocessing.Pool(processes=n_tiles)\n", - "pool.map(gt.compute_geotiled, sorted(glob.glob(elevation_tiles + \"/*.tif\")))\n", - "\n", - "gt.build_mosaic_filtered(\n", - " sorted(glob.glob(aspect_tiles + \"/*.tif\")),\n", - " os.path.join(root_output_folder, \"aspect.tif\"),\n", - ")\n", - "gt.build_mosaic_filtered(\n", - " sorted(glob.glob(hillshading_tiles + \"/*.tif\")),\n", - " os.path.join(root_output_folder, \"hillshading.tif\"),\n", - ")\n", - "gt.build_mosaic_filtered(\n", - " sorted(glob.glob(slope_tiles + \"/*.tif\")),\n", - " os.path.join(root_output_folder, \"slope.tif\"),\n", - ")\n", - "\n", - "\n", - "shutil.rmtree(aspect_tiles)\n", - "shutil.rmtree(hillshading_tiles)\n", - "shutil.rmtree(slope_tiles)\n", - "shutil.rmtree(elevation_tiles)\n", - "\n", - "print(\"You have successfully computed all the terrain parameters.\")" + "if generateDataFromScratch:\n", + "\n", + " pool = multiprocessing.Pool(processes=n_tiles)\n", + " pool.map(gt.compute_geotiled, sorted(glob.glob(elevation_tiles + \"/*.tif\")))\n", + " \n", + " gt.build_mosaic_filtered(\n", + " sorted(glob.glob(aspect_tiles + \"/*.tif\")),\n", + " os.path.join(root_output_folder, \"aspect.tif\"),\n", + " )\n", + " gt.build_mosaic_filtered(\n", + " sorted(glob.glob(hillshading_tiles + \"/*.tif\")),\n", + " os.path.join(root_output_folder, \"hillshading.tif\"),\n", + " )\n", + " gt.build_mosaic_filtered(\n", + " sorted(glob.glob(slope_tiles + \"/*.tif\")),\n", + " os.path.join(root_output_folder, \"slope.tif\"),\n", + " )\n", + " \n", + " \n", + " shutil.rmtree(aspect_tiles)\n", + " shutil.rmtree(hillshading_tiles)\n", + " shutil.rmtree(slope_tiles)\n", + " shutil.rmtree(elevation_tiles)\n", + " \n", + " print(\"You have successfully computed all the terrain parameters.\")\n", + "else:\n", + " print('Skipping this cell...')" ] }, { @@ -274,57 +289,58 @@ "metadata": {}, "outputs": [], "source": [ - "hill = os.path.join(root_output_folder, \"hillshading.tif\")\n", - "aspect = os.path.join(root_output_folder, \"aspect.tif\")\n", - "slope = os.path.join(root_output_folder, \"slope.tif\")\n", - "\n", - "\n", - "pcs_array = gt.generate_img(\n", - " pcs,\n", - " downsample=5,\n", - " reproject_gcs=True,\n", - " shp_files=shapefile,\n", - " title=\"Elevation Data for TN @ 1 Arc-Second/30m Resolution\",\n", - " zunit=\"Meter\",\n", - " xyunit=\"Degree\",\n", - " ztype=\"Elevation\",\n", - " crop_shp=True,\n", - ")\n", - "aspect_array = gt.generate_img(\n", - " aspect,\n", - " downsample=5,\n", - " reproject_gcs=True,\n", - " shp_files=shapefile,\n", - " title=\"Aspect Data for Rhode TN @ 1 Arc-Second/30m Resolution\",\n", - " zunit=\"Degree\",\n", - " xyunit=\"Degree\",\n", - " ztype=\"Aspect\",\n", - " crop_shp=True,\n", - ")\n", - "hill_array = gt.generate_img(\n", - " hill,\n", - " downsample=5,\n", - " reproject_gcs=True,\n", - " shp_files=shapefile,\n", - " title=\"Hillshading Data for TN @ 1 Arc-Second/30m Resolution\",\n", - " zunit=\"Level\",\n", - " xyunit=\"Degree\",\n", - " ztype=\"Hillshading\",\n", - " crop_shp=True,\n", - ")\n", - "slope_array = gt.generate_img(\n", - " slope,\n", - " downsample=5,\n", - " reproject_gcs=True,\n", - " shp_files=shapefile,\n", - " title=\"Slope Data for TN @ 1 Arc-Second/30m Resolution\",\n", - " zunit=\"Degree\",\n", - " xyunit=\"Degree\",\n", - " ztype=\"Slope\",\n", - " crop_shp=True,\n", - ")\n", - "\n", - "print(\"You have successfully visualized the 4 terrain parameters using GEOtiled\")" + "if generateDataFromScratch:\n", + " hill = os.path.join(root_output_folder, \"hillshading.tif\")\n", + " aspect = os.path.join(root_output_folder, \"aspect.tif\")\n", + " slope = os.path.join(root_output_folder, \"slope.tif\")\n", + " pcs_array = gt.generate_img(\n", + " pcs,\n", + " downsample=5,\n", + " reproject_gcs=True,\n", + " shp_files=shapefile,\n", + " title=\"Elevation Data for TN @ 1 Arc-Second/30m Resolution\",\n", + " zunit=\"Meter\",\n", + " xyunit=\"Degree\",\n", + " ztype=\"Elevation\",\n", + " crop_shp=True,\n", + " )\n", + " aspect_array = gt.generate_img(\n", + " aspect,\n", + " downsample=5,\n", + " reproject_gcs=True,\n", + " shp_files=shapefile,\n", + " title=\"Aspect Data for Rhode TN @ 1 Arc-Second/30m Resolution\",\n", + " zunit=\"Degree\",\n", + " xyunit=\"Degree\",\n", + " ztype=\"Aspect\",\n", + " crop_shp=True,\n", + " )\n", + " hill_array = gt.generate_img(\n", + " hill,\n", + " downsample=5,\n", + " reproject_gcs=True,\n", + " shp_files=shapefile,\n", + " title=\"Hillshading Data for TN @ 1 Arc-Second/30m Resolution\",\n", + " zunit=\"Level\",\n", + " xyunit=\"Degree\",\n", + " ztype=\"Hillshading\",\n", + " crop_shp=True,\n", + " )\n", + " slope_array = gt.generate_img(\n", + " slope,\n", + " downsample=5,\n", + " reproject_gcs=True,\n", + " shp_files=shapefile,\n", + " title=\"Slope Data for TN @ 1 Arc-Second/30m Resolution\",\n", + " zunit=\"Degree\",\n", + " xyunit=\"Degree\",\n", + " ztype=\"Slope\",\n", + " crop_shp=True,\n", + " )\n", + " \n", + " print(\"You have successfully visualized the 4 terrain parameters using GEOtiled\")\n", + "else:\n", + " print('Skipping this cell...')" ] }, { @@ -595,7 +611,7 @@ "source": [ "filename = \"idx_data/Tennessee_terrain_parameters.idx\"\n", "db = ov.LoadDataset(filename)\n", - "\n", + "loadDataFromSeal=False\n", "print(\"You have successfully loaded the IDX file with the four terrain parmaters.\")" ] }, @@ -753,21 +769,24 @@ "metadata": {}, "outputs": [], "source": [ - "filename = \"Tennessee_terrain_parameters.idx\"\n", - "HOME_DIR = \"s3://utah/nsdf/somospie/\" # DONOT change this line\n", - "data_dir = \"terrain_tennessee/\"\n", - "upload_dir = HOME_DIR + data_dir\n", - "s3_path = upload_dir.split(\"://\")[1]\n", - "s3_path += filename\n", - "remote_dir = (\n", - " \"https://maritime.sealstorage.io/api/v0/s3/\"\n", - " + s3_path\n", - " + \"?access_key=any&secret_key=any&endpoint_url=https://maritime.sealstorage.io/api/v0/s3&cached=arco\"\n", - ")\n", - "\n", - "db = ov.LoadDataset(remote_dir)\n", - "\n", - "print(\"You have successfully downloaded the IDX files from Seal Storage.\")" + "if loadDataFromSeal:\n", + " filename = \"Tennessee_terrain_parameters.idx\"\n", + " HOME_DIR = \"s3://utah/nsdf/somospie/\" # DONOT change this line\n", + " data_dir = \"terrain_tennessee/\"\n", + " upload_dir = HOME_DIR + data_dir\n", + " s3_path = upload_dir.split(\"://\")[1]\n", + " s3_path += filename\n", + " remote_dir = (\n", + " \"https://maritime.sealstorage.io/api/v0/s3/\"\n", + " + s3_path\n", + " + \"?access_key=any&secret_key=any&endpoint_url=https://maritime.sealstorage.io/api/v0/s3&cached=arco\"\n", + " )\n", + " \n", + " db = ov.LoadDataset(remote_dir)\n", + " \n", + " print(\"You have successfully downloaded the IDX files from Seal Storage.\")\n", + "else:\n", + " print('Data Loaded Locally earlier...')" ] }, { @@ -921,7 +940,7 @@ "source": [ "PORT = \"8989\" # Dont change this since this is the forwarded port.\n", "ADDRESS = \"0.0.0.0\" # Local to the server; 0.0.0.0 or 127.0.0.1\n", - "\n", + "loadSeal=False\n", "print(\"You have successfully set the port for the dashboard.\")" ] }, @@ -973,9 +992,11 @@ "metadata": {}, "outputs": [], "source": [ - "URL = \"https://maritime.sealstorage.io/api/v0/s3/utah/nsdf/somospie/tennessee_30m_terrain/Tennessee_terrain_parameters.idx?access_key=any&secret_key=any&endpoint_url=https://maritime.sealstorage.io/api/v0/s3&cached=arco\"\n", - "\n", - "print(\"You have successfully loaded the IDX Tennessee data from remote storage.\")" + "if loadSeal:\n", + " URL = \"https://maritime.sealstorage.io/api/v0/s3/utah/nsdf/somospie/tennessee_30m_terrain/Tennessee_terrain_parameters.idx?access_key=any&secret_key=any&endpoint_url=https://maritime.sealstorage.io/api/v0/s3&cached=arco\" \n", + " print(\"You have successfully loaded the IDX Tennessee data from remote storage.\")\n", + "else:\n", + " print('Data loaded locally earlier; Change loadSeal to `True` to load data from sealstorage')" ] }, { @@ -993,6 +1014,7 @@ "metadata": {}, "outputs": [], "source": [ + "%%capture\n", "print(\"Visit http://0.0.0.0:8989 to explore the dashboard\")\n", "!python -m panel serve openvisuspy/src/openvisuspy/dashboards --log-file \"files/log.log\" --dev --allow-websocket-origin='*' --address=\"{ADDRESS}\" --port \"{PORT}\" --args \"{URL}\"" ] @@ -1025,6 +1047,7 @@ "metadata": {}, "outputs": [], "source": [ + "%%capture\n", "print(\"Visit http://0.0.0.0:8989 to explore the dashboard\")\n", "!python -m panel serve openvisuspy/src/openvisuspy/dashboards --log-file \"files/log.log\" --dev --allow-websocket-origin='*' --address=\"{ADDRESS}\" --port \"{PORT}\" --args \"{URL}\" " ] @@ -1097,7 +1120,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.3" + "version": "3.9.7" }, "vscode": { "interpreter": {