From fded9a4d9b1a72a04ea8de2fbd2bf27121fcf502 Mon Sep 17 00:00:00 2001 From: ghiggi Date: Fri, 12 May 2023 12:46:56 +0200 Subject: [PATCH] Fix docstring warnings --- disdrodb/l0/check_metadata.py | 8 +- disdrodb/l0/check_readers.py | 1 - disdrodb/l0/l0_processing.py | 103 ++++++++++-------- disdrodb/l0/l0_reader.py | 25 +++-- disdrodb/l0/scripts/run_disdrodb_l0.py | 12 +- .../l0/scripts/run_disdrodb_l0_station.py | 9 +- disdrodb/l0/scripts/run_disdrodb_l0a.py | 8 +- disdrodb/l0/scripts/run_disdrodb_l0b.py | 4 +- docs/requirements.txt | 1 + docs/source/authors.rst | 3 +- docs/source/conf.py | 1 + docs/source/data.rst | 12 +- docs/source/index.rst | 2 +- docs/source/sensor_configs.rst | 7 +- 14 files changed, 107 insertions(+), 89 deletions(-) diff --git a/disdrodb/l0/check_metadata.py b/disdrodb/l0/check_metadata.py index 9bb0d0a3..81bdffc4 100644 --- a/disdrodb/l0/check_metadata.py +++ b/disdrodb/l0/check_metadata.py @@ -123,6 +123,7 @@ def identify_empty_metadata_keys(metadata_fpaths: list, keys: Union[str, list]) def get_archive_metadata_key_value(disdrodb_dir: str, key: str, return_tuple: bool = True): """Return the values of a metadata key for all the archive. + Parameters ---------- disdrodb_dir : str @@ -130,14 +131,15 @@ def get_archive_metadata_key_value(disdrodb_dir: str, key: str, return_tuple: bo key : str Metadata key. return_tuple : bool, optional - if True, returns a tuple of values with station, campaign and data source name (default is True) - if False, returns a list of values without station, campaign and data source name + If True, returns a tuple of values with station, campaign and data source name. + If False, returns a list of values without station, campaign and data source name. + The default is True. + Returns ------- list or tuple List or tuple of values of the metadata key. """ - list_metadata_paths = get_list_metadata( disdrodb_dir, data_sources=None, campaign_names=None, station_names=None, with_stations_data=False ) diff --git a/disdrodb/l0/check_readers.py b/disdrodb/l0/check_readers.py index 986b7686..9497f7ae 100644 --- a/disdrodb/l0/check_readers.py +++ b/disdrodb/l0/check_readers.py @@ -140,7 +140,6 @@ def check_all_readers() -> None: """Test all readers that have data samples and ground truth. Raises - ------ Exception If the reader validation has failed. diff --git a/disdrodb/l0/l0_processing.py b/disdrodb/l0/l0_processing.py index 65911fbc..5734c42c 100644 --- a/disdrodb/l0/l0_processing.py +++ b/disdrodb/l0/l0_processing.py @@ -351,31 +351,33 @@ def run_l0a( ---------- raw_dir : str The directory path where all the raw content of a specific campaign is stored. - The path must have the following structure: - <...>/DISDRODB/Raw//'. + The path must have the following structure: ``<...>/DISDRODB/Raw//``. Inside the raw_dir directory, it is required to adopt the following structure: - - /data// - - /metadata/.yaml + + - /data// + - /metadata/.yaml + Important points: - - For each there must be a corresponding YAML file in the metadata subfolder. - - The must semantically match between: - - the raw_dir and processed_dir directory paths; - - with the key 'campaign_name' within the metadata YAML files. - - The campaign_name are expected to be UPPER CASE. + - For each there must be a corresponding YAML file in the metadata subfolder. + - The campaign_name are expected to be UPPER CASE. + - The must semantically match between: + + - the raw_dir and processed_dir directory paths; + - with the key 'campaign_name' within the metadata YAML files. + processed_dir : str The desired directory path for the processed DISDRODB L0A and L0B products. - The path should have the following structure: - <...>/DISDRODB/Processed//' + The path should have the following structure: ``<...>/DISDRODB/Processed//`` For testing purpose, this function exceptionally accept also a directory path simply ending with (i.e. /tmp/). station_name : str Station name glob_patterns: str - Glob pattern to search data files in /data/ + Glob pattern to search data files in ``/data/`` column_names : list Columns names of the raw text file. reader_kwargs : dict - Pandas `read_csv` arguments to open the text file. + Pandas `read_csv` arguments to open the text file. df_sanitizer_fun : object, optional Sanitizer function to format the datafame into DISDRODB L0A standard. parallel : bool @@ -484,21 +486,23 @@ def run_l0b( ---------- raw_dir : str The directory path where all the raw content of a specific campaign is stored. - The path must have the following structure: - <...>/DISDRODB/Raw//'. + The path must have the following structure: ``<...>/DISDRODB/Raw//``. Inside the raw_dir directory, it is required to adopt the following structure: - - /data// - - /metadata/.yaml + + - /data// + - /metadata/.yaml + Important points: - - For each there must be a corresponding YAML file in the metadata subfolder. - - The must semantically match between: - - the raw_dir and processed_dir directory paths; - - with the key 'campaign_name' within the metadata YAML files. - - The campaign_name are expected to be UPPER CASE. + + - For each there must be a corresponding YAML file in the metadata subfolder. + - The campaign_name are expected to be UPPER CASE. + - The must semantically match between: + - the raw_dir and processed_dir directory paths; + - with the key 'campaign_name' within the metadata YAML files. + processed_dir : str The desired directory path for the processed DISDRODB L0A and L0B products. - The path should have the following structure: - <...>/DISDRODB/Processed//' + The path should have the following structure: ``<...>/DISDRODB/Processed//``. For testing purpose, this function exceptionally accept also a directory path simply ending with (i.e. /tmp/). station_name : str @@ -618,29 +622,32 @@ def run_l0b_from_nc( Parameters ---------- + raw_dir : str The directory path where all the raw content of a specific campaign is stored. - The path must have the following structure: - <...>/DISDRODB/Raw//'. + The path must have the following structure: ``<...>/DISDRODB/Raw//``. Inside the raw_dir directory, it is required to adopt the following structure: - - /data// - - /metadata/.yaml + + - /data// + - /metadata/.yaml + Important points: - - For each there must be a corresponding YAML file in the metadata subfolder. - - The must semantically match between: - - the raw_dir and processed_dir directory paths; - - with the key 'campaign_name' within the metadata YAML files. - - The campaign_name are expected to be UPPER CASE. + - For each there must be a corresponding YAML file in the metadata subfolder. + - The campaign_name are expected to be UPPER CASE. + - The must semantically match between: + + - the raw_dir and processed_dir directory paths; + - with the key 'campaign_name' within the metadata YAML files. + processed_dir : str The desired directory path for the processed DISDRODB L0B products. - The path should have the following structure: - <...>/DISDRODB/Processed//' + The path should have the following structure: ``<...>/DISDRODB/Processed//`` For testing purpose, this function exceptionally accept also a directory path simply ending with (i.e. /tmp/). station_name : str Station name glob_patterns: str - Glob pattern to search data files in /data/. + Glob pattern to search data files in ``/data/``. Example: glob_patterns = "*.nc" dict_names : dict Dictionary mapping raw netCDF variables/coordinates/dimension names @@ -863,11 +870,11 @@ def run_disdrodb_l0_station( station_name : str Station name l0a_processing : bool - Whether to launch processing to generate L0A Apache Parquet file(s) from raw data. - The default is True. + Whether to launch processing to generate L0A Apache Parquet file(s) from raw data. + The default is True. l0b_processing : bool - Whether to launch processing to generate L0B netCDF4 file(s) from L0A data. - The default is True. + Whether to launch processing to generate L0B netCDF4 file(s) from L0A data. + The default is True. l0b_concat : bool Whether to concatenate all raw files into a single L0B netCDF file. If l0b_concat=True, all raw files will be saved into a single L0B netCDF file. @@ -877,8 +884,8 @@ def run_disdrodb_l0_station( Whether to keep the L0A files after having generated the L0B netCDF products. The default is False. remove_l0b : bool - Whether to remove the L0B files after having concatenated all L0B netCDF files. - It takes places only if l0b_concat=True + Whether to remove the L0B files after having concatenated all L0B netCDF files. + It takes places only if l0b_concat=True The default is False. force : bool If True, overwrite existing data into destination directories. @@ -1016,11 +1023,11 @@ def run_disdrodb_l0( Station names to process. The default is None l0a_processing : bool - Whether to launch processing to generate L0A Apache Parquet file(s) from raw data. - The default is True. + Whether to launch processing to generate L0A Apache Parquet file(s) from raw data. + The default is True. l0b_processing : bool - Whether to launch processing to generate L0B netCDF4 file(s) from L0A data. - The default is True. + Whether to launch processing to generate L0B netCDF4 file(s) from L0A data. + The default is True. l0b_concat : bool Whether to concatenate all raw files into a single L0B netCDF file. If l0b_concat=True, all raw files will be saved into a single L0B netCDF file. @@ -1030,8 +1037,8 @@ def run_disdrodb_l0( Whether to keep the L0A files after having generated the L0B netCDF products. The default is False. remove_l0b : bool - Whether to remove the L0B files after having concatenated all L0B netCDF files. - It takes places only if l0b_concat = True + Whether to remove the L0B files after having concatenated all L0B netCDF files. + It takes places only if l0b_concat = True The default is False. force : bool If True, overwrite existing data into destination directories. diff --git a/disdrodb/l0/l0_reader.py b/disdrodb/l0/l0_reader.py index 133bf9c9..ee1f3003 100644 --- a/disdrodb/l0/l0_reader.py +++ b/disdrodb/l0/l0_reader.py @@ -381,21 +381,24 @@ def reader_generic_docstring(): ---------- raw_dir : str The directory path where all the raw content of a specific campaign is stored. - The path must have the following structure: - <...>/DISDRODB/Raw//'. + The path must have the following structure <...>/DISDRODB/Raw//'. Inside the raw_dir directory, it is required to adopt the following structure: - - /data// - - /metadata/.yaml + + - /data// + - /metadata/.yaml + Important points: - - For each there must be a corresponding YAML file in the metadata subfolder. - - The must semantically match between: - - the raw_dir and processed_dir directory paths; - - with the key 'campaign_name' within the metadata YAML files. - - The campaign_name are expected to be UPPER CASE. + + - For each there must be a corresponding YAML file in the metadata subfolder. + - The campaign_name are expected to be UPPER CASE. + - The must semantically match between: + + - the raw_dir and processed_dir directory paths; + - with the key 'campaign_name' within the metadata YAML files. + processed_dir : str The desired directory path for the processed DISDRODB L0A and L0B products. - The path should have the following structure: - <...>/DISDRODB/Processed//' + The path should have the following structure <...>/DISDRODB/Processed//' For testing purpose, this function exceptionally accept also a directory path simply ending with (i.e. /tmp/). station_name : str diff --git a/disdrodb/l0/scripts/run_disdrodb_l0.py b/disdrodb/l0/scripts/run_disdrodb_l0.py index 1813c448..972156f0 100644 --- a/disdrodb/l0/scripts/run_disdrodb_l0.py +++ b/disdrodb/l0/scripts/run_disdrodb_l0.py @@ -51,7 +51,8 @@ def run_disdrodb_l0( parallel: bool = True, debugging_mode: bool = False, ): - """Run the L0 processing of DISDRODB stations. + """ + Run the L0 processing of DISDRODB stations. This function enable to launch the processing of many DISDRODB stations with a single command. From the list of all available DISDRODB stations, it runs the processing of the @@ -59,6 +60,7 @@ def run_disdrodb_l0( Parameters ---------- + disdrodb_dir : str Base directory of DISDRODB Format: <...>/DISDRODB @@ -75,11 +77,11 @@ def run_disdrodb_l0( Station names. To specify multiple stations, write i.e.: --station_names 'station1 station2' l0a_processing : bool - Whether to launch processing to generate L0A Apache Parquet file(s) from raw data. - The default is True. + Whether to launch processing to generate L0A Apache Parquet file(s) from raw data. + The default is True. l0b_processing : bool - Whether to launch processing to generate L0B netCDF4 file(s) from L0A data. - The default is True. + Whether to launch processing to generate L0B netCDF4 file(s) from L0A data. + The default is True. l0b_concat : bool Whether to concatenate all raw files into a single L0B netCDF file. If l0b_concat=True, all raw files will be saved into a single L0B netCDF file. diff --git a/disdrodb/l0/scripts/run_disdrodb_l0_station.py b/disdrodb/l0/scripts/run_disdrodb_l0_station.py index 1dd82ce2..173f87b5 100644 --- a/disdrodb/l0/scripts/run_disdrodb_l0_station.py +++ b/disdrodb/l0/scripts/run_disdrodb_l0_station.py @@ -69,12 +69,11 @@ def run_disdrodb_l0_station( station_name : str \n Station name \n l0a_processing : bool - Whether to launch processing to generate L0A Apache Parquet file(s) from raw data. - The default is True.\n + Whether to launch processing to generate L0A Apache Parquet file(s) from raw data. + The default is True.\n l0b_processing : bool \n - Whether to launch processing to generate L0B netCDF4 file(s) from L0A data.\n - The default is True.\n - + Whether to launch processing to generate L0B netCDF4 file(s) from L0A data.\n + he default is True.\n l0b_concat : bool \n Whether to concatenate all raw files into a single L0B netCDF file.\n If l0b_concat=True, all raw files will be saved into a single L0B netCDF file.\n diff --git a/disdrodb/l0/scripts/run_disdrodb_l0a.py b/disdrodb/l0/scripts/run_disdrodb_l0a.py index a1a6ddcf..7a404f0d 100644 --- a/disdrodb/l0/scripts/run_disdrodb_l0a.py +++ b/disdrodb/l0/scripts/run_disdrodb_l0a.py @@ -44,14 +44,16 @@ def run_disdrodb_l0a( parallel: bool = True, debugging_mode: bool = False, ): - """Run the L0A processing of DISDRODB stations. + """ + Run the L0A processing of DISDRODB stations. This function enable to launch the processing of many DISDRODB stations with a single command. - From the list of all available DISDRODB stations, it runs the processing of the - stations matching the provided data_sources, campaign_names and station_names. + From the list of all available DISDRODB stations, it runs the processing + of the stations matching the provided data_sources, campaign_names and station_names. Parameters ---------- + disdrodb_dir : str Base directory of DISDRODB Format: <...>/DISDRODB diff --git a/disdrodb/l0/scripts/run_disdrodb_l0b.py b/disdrodb/l0/scripts/run_disdrodb_l0b.py index 60d70203..cae32de5 100644 --- a/disdrodb/l0/scripts/run_disdrodb_l0b.py +++ b/disdrodb/l0/scripts/run_disdrodb_l0b.py @@ -44,7 +44,8 @@ def run_disdrodb_l0b( parallel: bool = True, debugging_mode: bool = False, ): - """Run the L0B processing of DISDRODB stations. + """ + Run the L0B processing of DISDRODB stations. This function enable to launch the processing of many DISDRODB stations with a single command. From the list of all available DISDRODB stations, it runs the processing of the @@ -52,6 +53,7 @@ def run_disdrodb_l0b( Parameters ---------- + disdrodb_dir : str Base directory of DISDRODB Format: <...>/DISDRODB diff --git a/docs/requirements.txt b/docs/requirements.txt index ea5c928b..e1e3ed82 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -34,5 +34,6 @@ requests-mock==1.10.0 requests==2.28.2 sphinx==5.1.1 sphinx_rtd_theme==1.0.0 +sphinx-mdinclude==0.5.3 tqdm==4.65.0 disdrodb==0.0.17 diff --git a/docs/source/authors.rst b/docs/source/authors.rst index ab68d0d9..dcd86d9f 100644 --- a/docs/source/authors.rst +++ b/docs/source/authors.rst @@ -1,3 +1,4 @@ -.. _list_authors: +List of contributors +===================== .. mdinclude:: ../../AUTHORS.md diff --git a/docs/source/conf.py b/docs/source/conf.py index ad5ef01e..0736b850 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -46,6 +46,7 @@ "sphinx.ext.viewcode", "sphinx.ext.autosectionlabel", "nbsphinx", + "sphinx_mdinclude", ] diff --git a/docs/source/data.rst b/docs/source/data.rst index aea2fa14..7b2cb44a 100644 --- a/docs/source/data.rst +++ b/docs/source/data.rst @@ -1,6 +1,6 @@ -========================= +===== Data -========================= +===== Users can make their own data accessible to the community. DISDRODB @@ -29,7 +29,7 @@ Data transfer upload and download schema: .. image:: /static/transfer.png Download the DISDRODB metadata archive -------------------------------------------------- +----------------------------------------- You can clone the disdrodb-data repository with: @@ -42,7 +42,7 @@ fork the repository on your GitHub account and then clone the forked repository. Update the DISDRODB metadata archive ----------------------------------------------- +---------------------------------------- Do you want to contribute to the project with your own data? Great! Just follow these steps: @@ -77,7 +77,7 @@ follow these steps: 9. If you struggle with this process, don’t hesitate to raise an `issue `__ so we can help! Download the DISDRODB raw data archive --------------------------------------- +--------------------------------------- Prerequisite: First clone the disdrodb-data repository as described above to get the folder structure, metadatas and issues. @@ -111,7 +111,7 @@ the data sources or campaigns you require. For example, ``"EPFL NASA"``. Add new stations raw data to the DISDRODB archive (using Zenodo) -------------------------------------- +----------------------------------------------------------------- We provide users with a code to upload their station’s raw data to Zenodo. diff --git a/docs/source/index.rst b/docs/source/index.rst index c5a6d421..712882fb 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -32,7 +32,7 @@ difficult to share, analyse, compare and re-use. Additionally, very limited software is currently publicly available for DSD processing. This software aims to define a standard format to save disdrometer data and to create a decentralized archive to promote the exchange of data across the scientific community. -Currently, disdrodb enables to process data acquired from the OTT Parsivel (``OTT_Parsivel``), OTT Parsivel2 (``OTT_Parsivel2``), Thies Laser Precipitation Monitor (``ThiesLPM``) and RD-80 (``RD_80``) disdrometers. +Currently, disdrodb enables to process data acquired from the OTT Parsivel (``OTT_Parsivel``), OTT Parsivel2 (``OTT_Parsivel2``), Thies Laser Precipitation Monitor (``Thies_LPM``) and RD-80 (``RD_80``) disdrometers. Documentation diff --git a/docs/source/sensor_configs.rst b/docs/source/sensor_configs.rst index 2ffdced4..6a47c783 100644 --- a/docs/source/sensor_configs.rst +++ b/docs/source/sensor_configs.rst @@ -5,7 +5,7 @@ Add new sensor configs DISDRODB tailor the processing of the disdrometer measurements according to the instrument type and characteristics. -Currently, disdrodb enables to process data acquired from the OTT Parsivel (``OTT_Parsivel``), OTT Parsivel2 (``OTT_Parsivel2``), Thies Laser Precipitation Monitor (``ThiesLPM``) and RD-80 (``RD_80``) disdrometers. +Currently, disdrodb enables to process data acquired from the OTT Parsivel (``OTT_Parsivel``), OTT Parsivel2 (``OTT_Parsivel2``), Thies Laser Precipitation Monitor (``Thies_LPM``) and RD-80 (``RD_80``) disdrometers. The sensor configurations already implemented can be listed by typing the command: @@ -120,7 +120,7 @@ During the DISDRODB L0 processing: The ``n_digits``, ``n_characters``, ``n_decimals`` and ``n_naturals`` information is used to infer the raw files header when this is unknown. See usage of the ``infer_column_names`` function in the -`reader_preparation.ipynb`_ Jupyter Notebook. +`reader_preparation.ipynb `_ Jupyter Notebook. For the variables which values do not depend only from the time dimension, it is necessary to specify 2 additional keys: ``n_values`` and ``dimension_order`` @@ -168,6 +168,5 @@ variable_long_name.yml file This file specify the long_name for each variable. The ``long_name`` will be attached as a variable attribute to the DISDRODB L0B netCDF. -See the `CF Conventions guidelines for long_name -`_ +See the `CF Conventions guidelines for long_name `_ for more information.