From 204bd0e8c23ab5ea05ec7e6e0b29028d01f585d9 Mon Sep 17 00:00:00 2001 From: kamleshcodes Date: Tue, 21 Nov 2023 12:12:15 -0500 Subject: [PATCH 1/2] high-flow commit 2 --- environment.yml | 468 ++++++++++++-------------------- example/StreamflowExample-DI.py | 8 +- hydroDL/model/dropout.py | 3 + new_env.yml | 177 ------------ 4 files changed, 177 insertions(+), 479 deletions(-) delete mode 100644 new_env.yml diff --git a/environment.yml b/environment.yml index a27f6b9..bd92454 100644 --- a/environment.yml +++ b/environment.yml @@ -1,304 +1,176 @@ -name: mhpihydrodl +name: hydrodl_new channels: - pytorch + - nvidia + - conda-forge + - anaconda - defaults dependencies: - - _ipyw_jlab_nb_ext_conf=0.1.0=py36_0 - - alabaster=0.7.12=py36_0 - - anaconda-client=1.7.2=py36_0 - - anaconda-navigator=1.9.7=py36_0 - - anaconda-project=0.8.2=py36_0 - - asn1crypto=0.24.0=py36_0 - - astroid=2.2.5=py36_0 - - astropy=3.1.2=py36h7b6447c_0 - - atomicwrites=1.3.0=py36_1 - - attrs=19.1.0=py36_1 - - babel=2.6.0=py36_0 - - backcall=0.1.0=py36_0 - - backports=1.0=py36_1 - - backports.os=0.1.1=py36_0 - - backports.shutil_get_terminal_size=1.0.0=py36_2 - - basemap=1.2.0=py36h705c2d8_0 - - beautifulsoup4=4.7.1=py36_1 - - bitarray=0.9.0=py36h7b6447c_0 - - bkcharts=0.2=py36_0 + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - basemap=1.3.2=py310hb079758_1 + - basemap-data=1.3.6=py310h06a4308_1 - blas=1.0=mkl - - blaze=0.11.3=py36_0 - - bleach=3.1.0=py36_0 - - blosc=1.15.0=hd408876_0 - - bokeh=1.1.0=py36_0 - - boto=2.49.0=py36_0 - - bottleneck=1.2.1=py36h035aef0_1 - - bzip2=1.0.6=h14c3975_5 - - ca-certificates=2019.1.23=0 - - cairo=1.14.12=h8948797_3 - - certifi=2019.3.9=py36_0 - - cffi=1.12.3=py36h2e261b9_0 - - chardet=3.0.4=py36_1 - - click=7.0=py36_0 - - cloudpickle=0.8.1=py_0 - - clyent=1.2.2=py36_1 - - colorama=0.4.1=py36_0 - - conda=4.6.14=py36_0 - - conda-build=3.17.8=py36_0 - - conda-env=2.6.0=1 - - conda-verify=3.1.1=py36_0 - - contextlib2=0.5.5=py36_0 - - cryptography=2.6.1=py36h1ba5d50_0 - - cuda80=1.0=h205658b_0 - - cudatoolkit=10.0.130=0 - - cudnn=7.3.1=cuda10.0_0 - - curl=7.64.1=hbc83047_0 - - cycler=0.10.0=py36_0 - - cython=0.29.7=py36he6710b0_0 - - cytoolz=0.9.0.1=py36h14c3975_1 - - dask=1.2.0=py_0 - - dask-core=1.2.0=py_0 - - datashape=0.5.4=py36_1 - - dbus=1.13.6=h746ee38_0 - - decorator=4.4.0=py36_1 - - defusedxml=0.6.0=py_0 - - distributed=1.27.1=py36_0 - - docutils=0.14=py36_0 - - entrypoints=0.3=py36_0 - - et_xmlfile=1.0.1=py36_0 - - expat=2.2.6=he6710b0_0 - - fastcache=1.0.2=py36h14c3975_2 - - filelock=3.0.10=py36_0 - - flask=1.0.2=py36_1 - - flask-cors=3.0.7=py36_0 - - fontconfig=2.13.0=h9420a91_0 - - freetype=2.9.1=h8a8886c_1 - - fribidi=1.0.5=h7b6447c_0 - - future=0.17.1=py36_0 - - geos=3.6.2=heeff764_2 - - get_terminal_size=1.0.0=haa9412d_0 - - gevent=1.4.0=py36h7b6447c_0 - - glib=2.56.2=hd408876_0 - - glob2=0.6=py36_1 - - gmp=6.1.2=h6c8ec71_1 - - gmpy2=2.0.8=py36h10f8cd9_2 - - graphite2=1.3.13=h23475e2_0 - - greenlet=0.4.15=py36h7b6447c_0 - - gst-plugins-base=1.14.0=hbbd80ab_1 - - gstreamer=1.14.0=hb453b48_1 - - h5py=2.9.0=py36h7918eee_0 - - harfbuzz=1.8.8=hffaf4a1_0 - - hdf5=1.10.4=hb1b8bf9_0 - - heapdict=1.0.0=py36_2 - - html5lib=1.0.1=py36_0 - - icu=58.2=h9c2bf20_1 - - idna=2.8=py36_0 - - imageio=2.5.0=py36_0 - - imagesize=1.1.0=py36_0 - - importlib_metadata=0.9=py36_0 - - intel-openmp=2019.3=199 - - ipykernel=5.1.0=py36h39e3cac_0 - - ipython=7.5.0=py36h39e3cac_0 - - ipython_genutils=0.2.0=py36_0 - - ipywidgets=7.4.2=py36_0 - - isort=4.3.17=py36_0 - - itsdangerous=1.1.0=py36_0 - - jbig=2.1=hdba287a_0 - - jdcal=1.4.1=py_0 - - jedi=0.13.3=py36_0 - - jeepney=0.4=py36_0 - - jinja2=2.10.1=py36_0 - - jpeg=9b=h024ee3a_2 - - jsonschema=3.0.1=py36_0 - - jupyter=1.0.0=py36_7 - - jupyter_client=5.2.4=py36_0 - - jupyter_console=6.0.0=py36_0 - - jupyter_core=4.4.0=py36_0 - - jupyterlab=0.35.5=py36hf63ae98_0 - - jupyterlab_launcher=0.13.1=py36_0 - - jupyterlab_server=0.2.0=py36_0 - - keyring=18.0.0=py36_0 - - kiwisolver=1.1.0=py36he6710b0_0 - - krb5=1.16.1=h173b8e3_7 - - lazy-object-proxy=1.3.1=py36h14c3975_2 - - libarchive=3.3.3=h5d8350f_5 - - libcurl=7.64.1=h20c2e04_0 - - libedit=3.1.20181209=hc058e9b_0 - - libffi=3.2.1=hd88cf55_4 - - libgcc-ng=8.2.0=hdf63c60_1 - - libgfortran-ng=7.3.0=hdf63c60_0 - - liblief=0.9.0=h7725739_2 - - libpng=1.6.37=hbc83047_0 - - libsodium=1.0.16=h1bed415_0 - - libssh2=1.8.2=h1ba5d50_0 - - libstdcxx-ng=8.2.0=hdf63c60_1 - - libtiff=4.0.10=h2733197_2 - - libtool=2.4.6=h7b6447c_5 - - libuuid=1.0.3=h1bed415_2 - - libxcb=1.13=h1bed415_1 - - libxml2=2.9.9=he19cac6_0 - - libxslt=1.1.33=h7d1a2b0_0 - - llvmlite=0.28.0=py36hd408876_0 - - locket=0.2.0=py36_1 - - lxml=4.3.3=py36hefd8a0e_0 - - lz4-c=1.8.1.2=h14c3975_0 - - lzo=2.10=h49e0be7_2 - - markupsafe=1.1.1=py36h7b6447c_0 - - matplotlib=3.0.3=py36h5429711_0 - - mccabe=0.6.1=py36_1 - - mistune=0.8.4=py36h7b6447c_0 - - mkl=2019.3=199 - - mkl-service=1.1.2=py36he904b0f_5 - - mkl_fft=1.0.12=py36ha843d7b_0 - - mkl_random=1.0.2=py36hd81dba3_0 - - more-itertools=7.0.0=py36_0 + - bottleneck=1.3.5=py310ha9d4c09_0 + - brotli=1.0.9=h5eee18b_7 + - brotli-bin=1.0.9=h5eee18b_7 + - brotli-python=1.0.9=py310h6a678d5_7 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.1=h5eee18b_0 + - ca-certificates=2023.08.22=h06a4308_0 + - certifi=2023.7.22=py310h06a4308_0 + - cffi=1.16.0=py310h5eee18b_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - colorama=0.4.6=pyhd8ed1ab_0 + - contourpy=1.0.5=py310hdb19cb5_0 + - cryptography=41.0.3=py310hdda0065_0 + - cuda-cudart=12.1.105=0 + - cuda-cupti=12.1.105=0 + - cuda-libraries=12.1.0=0 + - cuda-nvrtc=12.1.105=0 + - cuda-nvtx=12.1.105=0 + - cuda-opencl=12.3.101=0 + - cuda-runtime=12.1.0=0 + - cycler=0.11.0=pyhd3eb1b0_0 + - cyrus-sasl=2.1.28=h52b45da_1 + - dbus=1.13.18=hb2f20db_0 + - expat=2.5.0=h6a678d5_0 + - ffmpeg=4.3=hf484d3e_0 + - filelock=3.9.0=py310h06a4308_0 + - fontconfig=2.14.1=h4c34cd2_2 + - fonttools=4.25.0=pyhd3eb1b0_0 + - freetype=2.12.1=h4a9f257_0 + - geos=3.8.0=he6710b0_0 + - giflib=5.2.1=h5eee18b_3 + - glib=2.69.1=he621ea3_2 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py310heeb90bb_0 + - gnutls=3.6.15=he1e5248_0 + - gst-plugins-base=1.14.1=h6a678d5_1 + - gstreamer=1.14.1=h5eee18b_1 + - icu=73.1=h6a678d5_0 + - idna=3.4=py310h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - jinja2=3.1.2=py310h06a4308_0 + - jpeg=9e=h5eee18b_1 + - kiwisolver=1.4.4=py310h6a678d5_0 + - krb5=1.20.1=h143b758_1 + - lame=3.100=h7b6447c_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libbrotlicommon=1.0.9=h5eee18b_7 + - libbrotlidec=1.0.9=h5eee18b_7 + - libbrotlienc=1.0.9=h5eee18b_7 + - libclang=14.0.6=default_hc6dbbc7_1 + - libclang13=14.0.6=default_he11475f_1 + - libcublas=12.1.0.26=0 + - libcufft=11.0.2.4=0 + - libcufile=1.8.1.2=0 + - libcups=2.4.2=h2d74bed_1 + - libcurand=10.3.4.101=0 + - libcurl=7.88.1=h251f7ec_2 + - libcusolver=11.4.4.55=0 + - libcusparse=12.0.2.55=0 + - libdeflate=1.17=h5eee18b_1 + - libedit=3.1.20221030=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgfortran-ng=11.2.0=h00389a5_1 + - libgfortran5=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libiconv=1.16=h7f8727e_2 + - libidn2=2.3.4=h5eee18b_0 + - libjpeg-turbo=2.0.0=h9bf148f_0 + - libllvm14=14.0.6=hdb19cb5_3 + - libnghttp2=1.57.0=h2d74bed_0 + - libnpp=12.0.2.50=0 + - libnvjitlink=12.1.105=0 + - libnvjpeg=12.1.1.14=0 + - libpng=1.6.39=h5eee18b_0 + - libpq=12.15=hdbd6064_1 + - libssh2=1.10.0=hdbd6064_2 + - libstdcxx-ng=11.2.0=h1234567_1 + - libtasn1=4.19.0=h5eee18b_0 + - libtiff=4.5.1=h6a678d5_0 + - libunistring=0.9.10=h27cfd23_0 + - libuuid=1.41.5=h5eee18b_0 + - libwebp=1.3.2=h11a3e52_0 + - libwebp-base=1.3.2=h5eee18b_0 + - libxcb=1.15=h7f8727e_0 + - libxkbcommon=1.0.1=h5eee18b_1 + - libxml2=2.10.4=hf1b16e4_1 + - llvm-openmp=14.0.6=h9e868ea_0 + - lz4-c=1.9.4=h6a678d5_0 + - markupsafe=2.1.1=py310h7f8727e_0 + - matplotlib=3.8.0=py310h06a4308_0 + - matplotlib-base=3.8.0=py310h1128e8f_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py310h5eee18b_1 + - mkl_fft=1.3.8=py310h5eee18b_0 + - mkl_random=1.2.4=py310hdb19cb5_0 - mpc=1.1.0=h10f8cd9_1 - - mpfr=4.0.1=hdf1c602_3 - - mpmath=1.1.0=py36_0 - - msgpack-python=0.6.1=py36hfd86e86_1 - - multipledispatch=0.6.0=py36_0 - - navigator-updater=0.2.1=py36_0 - - nbconvert=5.5.0=py_0 - - nbformat=4.4.0=py36_0 - - ncurses=6.1=he6710b0_1 - - networkx=2.3=py_0 - - ninja=1.9.0=py36hfd86e86_0 - - nltk=3.4.1=py36_0 - - nose=1.3.7=py36_2 - - notebook=5.7.8=py36_0 - - numba=0.43.1=py36h962f231_0 - - numexpr=2.6.9=py36h9e4a6bb_0 - - numpy=1.16.3=py36h7e9f1db_0 - - numpy-base=1.16.3=py36hde5b4d6_0 - - numpydoc=0.9.1=py_0 - - odo=0.5.1=py36_0 - - olefile=0.46=py36_0 - - openpyxl=2.6.2=py_0 - - openssl=1.1.1b=h7b6447c_1 - - packaging=19.0=py36_0 - - pandas=0.24.2=py36he6710b0_0 - - pandoc=2.2.3.2=0 - - pandocfilters=1.4.2=py36_1 - - pango=1.42.4=h049681c_0 - - parso=0.4.0=py_0 - - partd=0.3.10=py36_1 - - patchelf=0.9=he6710b0_3 - - path.py=12.0.1=py_0 - - pathlib2=2.3.3=py36_0 - - patsy=0.5.1=py36_0 - - pcre=8.43=he6710b0_0 - - pep8=1.7.1=py36_0 - - pexpect=4.7.0=py36_0 - - pickleshare=0.7.5=py36_0 - - pillow=6.0.0=py36h34e0f95_0 - - pip=19.1=py36_0 - - pixman=0.38.0=h7b6447c_0 - - pkginfo=1.5.0.1=py36_0 - - pluggy=0.9.0=py36_0 - - ply=3.11=py36_0 - - proj4=5.2.0=he6710b0_1 - - prometheus_client=0.6.0=py36_0 - - prompt_toolkit=2.0.9=py36_0 - - psutil=5.6.2=py36h7b6447c_0 - - ptyprocess=0.6.0=py36_0 - - py=1.8.0=py36_0 - - py-lief=0.9.0=py36h7725739_2 - - pycodestyle=2.5.0=py36_0 - - pycosat=0.6.3=py36h14c3975_0 - - pycparser=2.19=py36_0 - - pycrypto=2.6.1=py36h14c3975_9 - - pycurl=7.43.0.2=py36h1ba5d50_0 - - pyflakes=2.1.1=py36_0 - - pygments=2.3.1=py36_0 - - pylint=2.3.1=py36_0 - - pyodbc=4.0.26=py36he6710b0_0 - - pyopenssl=19.0.0=py36_0 - - pyparsing=2.4.0=py_0 - - pyproj=1.9.6=py36h14380d9_0 - - pyqt=5.9.2=py36h05f1152_2 - - pyrsistent=0.14.11=py36h7b6447c_0 - - pyshp=2.1.0=py_0 - - pysocks=1.6.8=py36_0 - - pytables=3.5.1=py36h71ec239_0 - - pytest=4.4.1=py36_0 - - pytest-arraydiff=0.3=py36h39e3cac_0 - - pytest-astropy=0.5.0=py36_0 - - pytest-doctestplus=0.3.0=py36_0 - - pytest-openfiles=0.3.2=py36_0 - - pytest-remotedata=0.3.1=py36_0 - - python=3.7.13=h12debd9_0 - - python-dateutil=2.8.0=py36_0 - - python-libarchive-c=2.8=py36_6 - - pytorch=1.0.1=cuda100py36he554f03_0 - - pytz=2019.1=py_0 - - pywavelets=1.0.3=py36hdd07704_1 - - pyyaml=5.1=py36h7b6447c_0 - - pyzmq=18.0.0=py36he6710b0_0 - - qt=5.9.7=h5867ecd_1 - - qtawesome=0.5.7=py36_1 - - qtconsole=4.4.3=py36_0 - - qtpy=1.7.0=py36_1 - - readline=7.0=h7b6447c_5 - - requests=2.21.0=py36_0 - - rope=0.14.0=py_0 - - ruamel_yaml=0.15.46=py36h14c3975_0 - - scikit-image=0.15.0=py36he6710b0_0 - - scikit-learn=0.20.3=py36hd81dba3_0 - - scipy=1.2.1=py36h7c811a0_0 - - seaborn=0.9.0=py36_0 - - secretstorage=3.1.1=py36_0 - - send2trash=1.5.0=py36_0 - - setuptools=41.0.1=py36_0 - - simplegeneric=0.8.1=py36_2 - - singledispatch=3.4.0.3=py36_0 - - sip=4.19.8=py36hf484d3e_0 - - six=1.12.0=py36_0 - - snappy=1.1.7=hbae5bb6_3 - - snowballstemmer=1.2.1=py36_0 - - sortedcollections=1.1.2=py36_0 - - sortedcontainers=2.1.0=py36_0 - - soupsieve=1.8=py36_0 - - sphinx=2.0.1=py_0 - - sphinxcontrib=1.0=py36_1 - - sphinxcontrib-applehelp=1.0.1=py_0 - - sphinxcontrib-devhelp=1.0.1=py_0 - - sphinxcontrib-htmlhelp=1.0.2=py_0 - - sphinxcontrib-jsmath=1.0.1=py_0 - - sphinxcontrib-qthelp=1.0.2=py_0 - - sphinxcontrib-serializinghtml=1.1.3=py_0 - - sphinxcontrib-websupport=1.1.0=py36_1 - - spyder=3.3.4=py36_0 - - spyder-kernels=0.4.4=py36_0 - - sqlalchemy=1.3.3=py36h7b6447c_0 - - sqlite=3.28.0=h7b6447c_0 - - statsmodels=0.9.0=py36h035aef0_0 - - sympy=1.4=py36_0 - - tblib=1.3.2=py36_0 - - terminado=0.8.2=py36_0 - - testpath=0.4.2=py36_0 - - tk=8.6.8=hbc83047_0 - - toolz=0.9.0=py36_0 - - torchvision=0.2.1=py36_0 - - tornado=6.0.2=py36h7b6447c_0 - - tqdm=4.31.1=py36_1 - - traitlets=4.3.2=py36_0 - - typed-ast=1.3.4=py36h7b6447c_0 - - typing=3.6.4=py36_0 - - unicodecsv=0.14.1=py36_0 - - unixodbc=2.3.7=h14c3975_0 - - urllib3=1.24.2=py36_0 - - wcwidth=0.1.7=py36_0 - - webencodings=0.5.1=py36_1 - - werkzeug=0.15.2=py_0 - - wheel=0.33.1=py36_0 - - widgetsnbextension=3.4.2=py36_0 - - wrapt=1.11.1=py36h7b6447c_0 - - wurlitzer=1.0.2=py36_0 - - xlrd=1.2.0=py36_0 - - xlsxwriter=1.1.7=py_0 - - xlwt=1.3.0=py36_0 - - xz=5.2.4=h14c3975_4 - - yaml=0.1.7=had09818_2 - - zeromq=4.3.1=he6710b0_3 - - zict=0.1.4=py36_0 - - zipp=0.3.3=py36_1 - - zlib=1.2.11=h7b6447c_3 - - zstd=1.3.7=h0b5b093_0 - - pip: - - dictdiffer==0.8.0 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py310h06a4308_0 + - munkres=1.1.4=py_0 + - mysql=5.7.24=h721c034_2 + - ncurses=6.4=h6a678d5_0 + - nettle=3.7.3=hbbd107a_1 + - networkx=3.1=py310h06a4308_0 + - numexpr=2.8.7=py310h85018f9_0 + - numpy=1.26.0=py310h5f9d8c6_0 + - numpy-base=1.26.0=py310hb5e798b_0 + - openh264=2.1.1=h4ff587b_0 + - openjpeg=2.4.0=h3ad879b_0 + - openssl=3.0.12=h7f8727e_0 + - packaging=23.1=py310h06a4308_0 + - pandas=2.1.1=py310h1128e8f_0 + - patsy=0.5.3=py310h06a4308_0 + - pcre=8.45=h295c915_0 + - pillow=10.0.1=py310ha6cbd5a_0 + - pip=23.3=py310h06a4308_0 + - ply=3.11=py310h06a4308_0 + - proj=8.2.1=ha227179_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyopenssl=23.2.0=py310h06a4308_0 + - pyparsing=3.0.9=py310h06a4308_0 + - pyproj=3.3.0=py310h162314d_0 + - pyqt=5.15.10=py310h6a678d5_0 + - pyqt5-sip=12.13.0=py310h5eee18b_0 + - pyshp=2.1.3=pyhd3eb1b0_0 + - pysocks=1.7.1=py310h06a4308_0 + - python=3.10.13=h955ad1f_0 + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - pytorch=2.1.1=py3.10_cuda12.1_cudnn8.9.2_0 + - pytorch-cuda=12.1=ha16c6d3_5 + - pytorch-mutex=1.0=cuda + - pytz=2023.3.post1=py310h06a4308_0 + - pyyaml=6.0.1=py310h5eee18b_0 + - qt-main=5.15.2=h53bd1ea_10 + - readline=8.2=h5eee18b_0 + - requests=2.31.0=py310h06a4308_0 + - scipy=1.11.3=py310h5f9d8c6_0 + - setuptools=68.0.0=py310h06a4308_0 + - sip=6.7.12=py310h6a678d5_0 + - six=1.15.0=pyhd3eb1b0_0 + - sqlite=3.41.2=h5eee18b_0 + - statsmodels=0.14.0=py310ha9d4c09_0 + - sympy=1.11.1=py310h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tk=8.6.12=h1ccaba5_0 + - tomli=2.0.1=py310h06a4308_0 + - torchaudio=2.1.1=py310_cu121 + - torchtriton=2.1.0=py310 + - torchvision=0.16.1=py310_cu121 + - tornado=6.3.3=py310h5eee18b_0 + - tqdm=4.66.1=pyhd8ed1ab_0 + - typing_extensions=4.7.1=py310h06a4308_0 + - tzdata=2023c=h04d1e81_0 + - urllib3=1.26.18=py310h06a4308_0 + - wheel=0.41.2=py310h06a4308_0 + - xz=5.4.2=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - zlib=1.2.13=h5eee18b_0 + - zstd=1.5.5=hc292b87_0 diff --git a/example/StreamflowExample-DI.py b/example/StreamflowExample-DI.py index c2134b2..c70b488 100644 --- a/example/StreamflowExample-DI.py +++ b/example/StreamflowExample-DI.py @@ -42,11 +42,10 @@ # 1: train DI model # 0,1: do both base and DI model # 2: test trained models -Action = [2] +Action = [0] gpuid = 6 torch.cuda.set_device(gpuid) - # Set hyperparameters EPOCH = 300 BATCH_SIZE = 100 @@ -57,6 +56,7 @@ forType = 'daymet' trainBuff = 365 loadTrain = True +subset_train = 'All' #give the list of basins to train on or else fix 'All' to use all # Fix random seed seedid = 111111 @@ -104,7 +104,7 @@ 'geol_2nd_class', 'glim_2nd_class_frac', 'carbonate_rocks_frac', 'geol_porostiy', 'geol_permeability'] optData = default.optDataCamels optData = default.update( - optData, varT=varF, varC=attrLst, tRange=Ttrain, forType=forType) # Update the training period + optData, varT=varF, varC=attrLst, tRange=Ttrain, forType=forType, subset=subset_train) # Update the training period # if (interfaceOpt == 1) and (2 not in Action): if (interfaceOpt == 1) and (loadTrain is True): @@ -171,7 +171,7 @@ ) # define output folder for model results -exp_name = f"CAMELSDemo-torch={torch.__version__}" +exp_name = f"CAMELSDemo" exp_disp = "TestRun" save_path = os.path.join( exp_name, diff --git a/hydroDL/model/dropout.py b/hydroDL/model/dropout.py index 845913d..cfb154e 100644 --- a/hydroDL/model/dropout.py +++ b/hydroDL/model/dropout.py @@ -2,7 +2,10 @@ import torch.nn + def createMask(x, dr): + generator = torch.Generator(device="cuda") + generator.manual_seed(42) mask = x.new().resize_as_(x).bernoulli_(1 - dr).div_(1 - dr).detach_() # print('droprate='+str(dr)) return mask diff --git a/new_env.yml b/new_env.yml deleted file mode 100644 index 201e4f3..0000000 --- a/new_env.yml +++ /dev/null @@ -1,177 +0,0 @@ -name: hydrodl_new -channels: - - pytorch - - nvidia - - conda-forge - - anaconda - - defaults -dependencies: - - _libgcc_mutex=0.1=main - - _openmp_mutex=5.1=1_gnu - - basemap=1.3.2=py310hb079758_1 - - basemap-data=1.3.6=py310h06a4308_1 - - blas=1.0=mkl - - bottleneck=1.3.5=py310ha9d4c09_0 - - brotli=1.0.9=h5eee18b_7 - - brotli-bin=1.0.9=h5eee18b_7 - - brotli-python=1.0.9=py310h6a678d5_7 - - bzip2=1.0.8=h7b6447c_0 - - c-ares=1.19.1=h5eee18b_0 - - ca-certificates=2023.08.22=h06a4308_0 - - certifi=2023.7.22=py310h06a4308_0 - - cffi=1.16.0=py310h5eee18b_0 - - charset-normalizer=2.0.4=pyhd3eb1b0_0 - - colorama=0.4.6=pyhd8ed1ab_0 - - contourpy=1.0.5=py310hdb19cb5_0 - - cryptography=41.0.3=py310hdda0065_0 - - cuda-cudart=12.1.105=0 - - cuda-cupti=12.1.105=0 - - cuda-libraries=12.1.0=0 - - cuda-nvrtc=12.1.105=0 - - cuda-nvtx=12.1.105=0 - - cuda-opencl=12.3.101=0 - - cuda-runtime=12.1.0=0 - - cycler=0.11.0=pyhd3eb1b0_0 - - cyrus-sasl=2.1.28=h52b45da_1 - - dbus=1.13.18=hb2f20db_0 - - expat=2.5.0=h6a678d5_0 - - ffmpeg=4.3=hf484d3e_0 - - filelock=3.9.0=py310h06a4308_0 - - fontconfig=2.14.1=h4c34cd2_2 - - fonttools=4.25.0=pyhd3eb1b0_0 - - freetype=2.12.1=h4a9f257_0 - - geos=3.8.0=he6710b0_0 - - giflib=5.2.1=h5eee18b_3 - - glib=2.69.1=he621ea3_2 - - gmp=6.2.1=h295c915_3 - - gmpy2=2.1.2=py310heeb90bb_0 - - gnutls=3.6.15=he1e5248_0 - - gst-plugins-base=1.14.1=h6a678d5_1 - - gstreamer=1.14.1=h5eee18b_1 - - icu=73.1=h6a678d5_0 - - idna=3.4=py310h06a4308_0 - - intel-openmp=2023.1.0=hdb19cb5_46306 - - jinja2=3.1.2=py310h06a4308_0 - - jpeg=9e=h5eee18b_1 - - kiwisolver=1.4.4=py310h6a678d5_0 - - krb5=1.20.1=h143b758_1 - - lame=3.100=h7b6447c_0 - - lcms2=2.12=h3be6417_0 - - ld_impl_linux-64=2.38=h1181459_1 - - lerc=3.0=h295c915_0 - - libbrotlicommon=1.0.9=h5eee18b_7 - - libbrotlidec=1.0.9=h5eee18b_7 - - libbrotlienc=1.0.9=h5eee18b_7 - - libclang=14.0.6=default_hc6dbbc7_1 - - libclang13=14.0.6=default_he11475f_1 - - libcublas=12.1.0.26=0 - - libcufft=11.0.2.4=0 - - libcufile=1.8.1.2=0 - - libcups=2.4.2=h2d74bed_1 - - libcurand=10.3.4.101=0 - - libcurl=7.88.1=h251f7ec_2 - - libcusolver=11.4.4.55=0 - - libcusparse=12.0.2.55=0 - - libdeflate=1.17=h5eee18b_1 - - libedit=3.1.20221030=h5eee18b_0 - - libev=4.33=h7f8727e_1 - - libffi=3.4.4=h6a678d5_0 - - libgcc-ng=11.2.0=h1234567_1 - - libgfortran-ng=11.2.0=h00389a5_1 - - libgfortran5=11.2.0=h1234567_1 - - libgomp=11.2.0=h1234567_1 - - libiconv=1.16=h7f8727e_2 - - libidn2=2.3.4=h5eee18b_0 - - libjpeg-turbo=2.0.0=h9bf148f_0 - - libllvm14=14.0.6=hdb19cb5_3 - - libnghttp2=1.57.0=h2d74bed_0 - - libnpp=12.0.2.50=0 - - libnvjitlink=12.1.105=0 - - libnvjpeg=12.1.1.14=0 - - libpng=1.6.39=h5eee18b_0 - - libpq=12.15=hdbd6064_1 - - libssh2=1.10.0=hdbd6064_2 - - libstdcxx-ng=11.2.0=h1234567_1 - - libtasn1=4.19.0=h5eee18b_0 - - libtiff=4.5.1=h6a678d5_0 - - libunistring=0.9.10=h27cfd23_0 - - libuuid=1.41.5=h5eee18b_0 - - libwebp=1.3.2=h11a3e52_0 - - libwebp-base=1.3.2=h5eee18b_0 - - libxcb=1.15=h7f8727e_0 - - libxkbcommon=1.0.1=h5eee18b_1 - - libxml2=2.10.4=hf1b16e4_1 - - llvm-openmp=14.0.6=h9e868ea_0 - - lz4-c=1.9.4=h6a678d5_0 - - markupsafe=2.1.1=py310h7f8727e_0 - - matplotlib=3.8.0=py310h06a4308_0 - - matplotlib-base=3.8.0=py310h1128e8f_0 - - mkl=2023.1.0=h213fc3f_46344 - - mkl-service=2.4.0=py310h5eee18b_1 - - mkl_fft=1.3.8=py310h5eee18b_0 - - mkl_random=1.2.4=py310hdb19cb5_0 - - mpc=1.1.0=h10f8cd9_1 - - mpfr=4.0.2=hb69a4c5_1 - - mpmath=1.3.0=py310h06a4308_0 - - munkres=1.1.4=py_0 - - mysql=5.7.24=h721c034_2 - - ncurses=6.4=h6a678d5_0 - - nettle=3.7.3=hbbd107a_1 - - networkx=3.1=py310h06a4308_0 - - numexpr=2.8.7=py310h85018f9_0 - - numpy=1.26.0=py310h5f9d8c6_0 - - numpy-base=1.26.0=py310hb5e798b_0 - - openh264=2.1.1=h4ff587b_0 - - openjpeg=2.4.0=h3ad879b_0 - - openssl=3.0.12=h7f8727e_0 - - packaging=23.1=py310h06a4308_0 - - pandas=2.1.1=py310h1128e8f_0 - - patsy=0.5.3=py310h06a4308_0 - - pcre=8.45=h295c915_0 - - pillow=10.0.1=py310ha6cbd5a_0 - - pip=23.3=py310h06a4308_0 - - ply=3.11=py310h06a4308_0 - - proj=8.2.1=ha227179_0 - - pycparser=2.21=pyhd3eb1b0_0 - - pyopenssl=23.2.0=py310h06a4308_0 - - pyparsing=3.0.9=py310h06a4308_0 - - pyproj=3.3.0=py310h162314d_0 - - pyqt=5.15.10=py310h6a678d5_0 - - pyqt5-sip=12.13.0=py310h5eee18b_0 - - pyshp=2.1.3=pyhd3eb1b0_0 - - pysocks=1.7.1=py310h06a4308_0 - - python=3.10.13=h955ad1f_0 - - python-dateutil=2.8.2=pyhd3eb1b0_0 - - python-tzdata=2023.3=pyhd3eb1b0_0 - - pytorch=2.1.1=py3.10_cuda12.1_cudnn8.9.2_0 - - pytorch-cuda=12.1=ha16c6d3_5 - - pytorch-mutex=1.0=cuda - - pytz=2023.3.post1=py310h06a4308_0 - - pyyaml=6.0.1=py310h5eee18b_0 - - qt-main=5.15.2=h53bd1ea_10 - - readline=8.2=h5eee18b_0 - - requests=2.31.0=py310h06a4308_0 - - scipy=1.11.3=py310h5f9d8c6_0 - - setuptools=68.0.0=py310h06a4308_0 - - sip=6.7.12=py310h6a678d5_0 - - six=1.15.0=pyhd3eb1b0_0 - - sqlite=3.41.2=h5eee18b_0 - - statsmodels=0.14.0=py310ha9d4c09_0 - - sympy=1.11.1=py310h06a4308_0 - - tbb=2021.8.0=hdb19cb5_0 - - tk=8.6.12=h1ccaba5_0 - - tomli=2.0.1=py310h06a4308_0 - - torchaudio=2.1.1=py310_cu121 - - torchtriton=2.1.0=py310 - - torchvision=0.16.1=py310_cu121 - - tornado=6.3.3=py310h5eee18b_0 - - tqdm=4.66.1=pyhd8ed1ab_0 - - typing_extensions=4.7.1=py310h06a4308_0 - - tzdata=2023c=h04d1e81_0 - - urllib3=1.26.18=py310h06a4308_0 - - wheel=0.41.2=py310h06a4308_0 - - xz=5.4.2=h5eee18b_0 - - yaml=0.2.5=h7b6447c_0 - - zlib=1.2.13=h5eee18b_0 - - zstd=1.5.5=hc292b87_0 -prefix: /data/kas7897/software_install/anaconda3/envs/hydrodl_new From 6cda80839063bd43f6e5a073ea36c60a56368834 Mon Sep 17 00:00:00 2001 From: kamleshcodes Date: Wed, 22 Nov 2023 19:37:52 -0500 Subject: [PATCH 2/2] high-flow commit 3 --- example/StreamflowExample-DI.py | 4 ++-- hydroDL/data/camels.py | 2 +- hydroDL/model/dropout.py | 4 ++-- hydroDL/model/rnn/CudnnLstm.py | 7 ++++--- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/example/StreamflowExample-DI.py b/example/StreamflowExample-DI.py index c70b488..f70c438 100644 --- a/example/StreamflowExample-DI.py +++ b/example/StreamflowExample-DI.py @@ -42,7 +42,7 @@ # 1: train DI model # 0,1: do both base and DI model # 2: test trained models -Action = [0] +Action = [2] gpuid = 6 torch.cuda.set_device(gpuid) @@ -199,7 +199,7 @@ ny = yTrain.shape[-1] # load model for training if torch.cuda.is_available(): - model = CudnnLstmModel(nx=nx, ny=ny, hiddenSize=HIDDENSIZE, warmUpDay=trainBuff) + model = CudnnLstmModel(nx=nx, ny=ny, hiddenSize=HIDDENSIZE) else: model = CpuLstmModel(nx=nx, ny=ny, hiddenSize=HIDDENSIZE) optModel = default.update(optModel, nx=nx, ny=ny) diff --git a/hydroDL/data/camels.py b/hydroDL/data/camels.py index 01f1237..bc0101c 100644 --- a/hydroDL/data/camels.py +++ b/hydroDL/data/camels.py @@ -646,7 +646,7 @@ def __init__(self, *, subset="All", tRange, forType='nldas'): elif type(subset) is list: self.usgsId = np.array(subset) crd = np.zeros([len(self.usgsId), 2]) - ind = np.full(len(self.usgsId), np.nan).astype(int) + ind = np.full(len(self.usgsId), -1, dtype=int) for ii in range(len(self.usgsId)): tempind = np.where(gageDict["id"] == self.usgsId[ii]) ind[ii] = tempind[0][0] diff --git a/hydroDL/model/dropout.py b/hydroDL/model/dropout.py index cfb154e..7d748d6 100644 --- a/hydroDL/model/dropout.py +++ b/hydroDL/model/dropout.py @@ -3,9 +3,9 @@ -def createMask(x, dr): +def createMask(x, dr, seed): generator = torch.Generator(device="cuda") - generator.manual_seed(42) + generator.manual_seed(seed) mask = x.new().resize_as_(x).bernoulli_(1 - dr).div_(1 - dr).detach_() # print('droprate='+str(dr)) return mask diff --git a/hydroDL/model/rnn/CudnnLstm.py b/hydroDL/model/rnn/CudnnLstm.py index 46652e8..a03832b 100644 --- a/hydroDL/model/rnn/CudnnLstm.py +++ b/hydroDL/model/rnn/CudnnLstm.py @@ -10,7 +10,7 @@ class CudnnLstm(nn.Module): - def __init__(self, *, inputSize, hiddenSize, dr=0.5, drMethod="drW", gpu=0): + def __init__(self, *, inputSize, hiddenSize, dr=0.5, drMethod="drW", gpu=0, seed=42): super(CudnnLstm, self).__init__() self.inputSize = inputSize self.hiddenSize = hiddenSize @@ -23,6 +23,7 @@ def __init__(self, *, inputSize, hiddenSize, dr=0.5, drMethod="drW", gpu=0): self._all_weights = [["w_ih", "w_hh", "b_ih", "b_hh"]] self.cuda() self.name = "CudnnLstm" + self.seed = seed self.is_legacy = True self.reset_mask() @@ -42,8 +43,8 @@ def __setstate__(self, d): self._all_weights = [["w_ih", "w_hh", "b_ih", "b_hh"]] def reset_mask(self): - self.maskW_ih = createMask(self.w_ih, self.dr) - self.maskW_hh = createMask(self.w_hh, self.dr) + self.maskW_ih = createMask(self.w_ih, self.dr, self.seed) + self.maskW_hh = createMask(self.w_hh, self.dr, self.seed) def reset_parameters(self): stdv = 1.0 / math.sqrt(self.hiddenSize)