Merge branch 'development' into plot_max_level

.github/workflows/dependencies_hip.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+#
+# Copyright 2020-2022 The AMReX Community
+#
+# License: BSD-3-Clause-LBNL
+# Authors: Axel Huebl
+
+set -eu -o pipefail
+
+
+# Ref.: https://rocmdocs.amd.com/en/latest/Installation_Guide/Installation-Guide.html#ubuntu
+curl -O https://repo.radeon.com/rocm/rocm.gpg.key
+sudo apt-key add rocm.gpg.key
+echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/debian/ ubuntu main' \
+  | sudo tee /etc/apt/sources.list.d/rocm.list
+echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' \
+  | sudo tee -a /etc/profile.d/rocm.sh
+
+# we should not need to export HIP_PATH=/opt/rocm/hip with those installs
+
+sudo apt-get update
+
+# Ref.: https://rocmdocs.amd.com/en/latest/Installation_Guide/Installation-Guide.html#installing-development-packages-for-cross-compilation
+# meta-package: rocm-dkms
+# OpenCL: rocm-opencl
+# other: rocm-dev rocm-utils
+sudo apt-get install -y --no-install-recommends \
+    build-essential \
+    gfortran        \
+    libnuma-dev     \
+    libopenmpi-dev  \
+    openmpi-bin     \
+    rocm-dev        \
+    roctracer-dev   \
+    rocprofiler-dev \
+    rocrand-dev     \
+    rocprim-dev
+
+# activate
+#
+source /etc/profile.d/rocm.sh
+hipcc --version
+which clang
+which clang++
+which flang

.github/workflows/exact_riemann.yml

@@ -0,0 +1,40 @@
+name: exact Riemann
+
+on: [pull_request]
+jobs:
+  exact_riemann:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Get submodules
+        run: |
+          git submodule update --init
+          cd external/Microphysics
+          git fetch; git checkout development
+          cd ../amrex
+          git fetch; git checkout development
+          cd ../..
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update -y -qq
+          sudo apt-get -qq -y install curl cmake jq clang g++>=9.3.0
+
+      - name: Compile exact_riemann
+        run: |
+          cd Util/exact_riemann
+          make -j 4
+
+      - name: Run test1
+        run: |
+          cd Util/exact_riemann
+          ./Castro3d.gnu.ex inputs.test1.helm
+
+      - name: Compare to stored output
+        run: |
+          cd Util/exact_riemann
+          diff riemann.out ci-benchmarks/test1-riemann.out
+

.github/workflows/hip.yml

@@ -0,0 +1,32 @@
+name: hip
+
+on: [push, pull_request]
+
+concurrency:
+  group: ${{ github.ref }}-${{ github.head_ref }}-hip
+  cancel-in-progress: true
+
+jobs:
+  hip-compile:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Get submodules
+        run: |
+          git submodule update --init
+          cd external/Microphysics
+          git fetch; git checkout development
+          cd ../amrex
+          git fetch; git checkout development
+          cd ../..
+
+      - name: Dependencies
+        run: .github/workflows/dependencies_hip.sh
+
+      - name: compile flame_wave
+        run: |
+          cd Exec/science/flame_wave
+          make COMP=gnu USE_HIP=TRUE USE_MPI=FALSE USE_OMP=FALSE USE_CUDA=FALSE -j 2

CHANGES.md

@@ -1,3 +1,9 @@
+# 22.12
+
+   * castro.lin_limit_state_interp now can be set to 2; this new
+     interpolater both prevents new extrema from being generated
+     and preserves linear combinations of state variables. (#2306)
+
 # 22.11
 
    * We now output the location where the timestep is set (#2273)

Diagnostics/timestep_limiter/limiter_util.F90

@@ -123,7 +123,6 @@ subroutine find_timestep_limiter_burning(lo, hi, state, slo, shi, nc_s, &
     use eos_type_module, only: eos_t, eos_input_rt
     use burner_module, only: ok_to_burn
     use burn_type_module, only : burn_t, net_ienuc, burn_to_eos, eos_to_burn
-    use temperature_integration_module, only: self_heat
     use network, only: nspec, aion
     use meth_params_module, only : dtnuc_e, dtnuc_X, dtnuc_X_threshold
     use extern_probin_module, only: small_x
@@ -178,8 +177,6 @@ subroutine find_timestep_limiter_burning(lo, hi, state, slo, shi, nc_s, &
 
                 state_new % dx = minval(dx(1:AMREX_SPACEDIM))
 
-                state_new % self_heat = .false.
-
                 call actual_rhs(state_new)
 
                 dedt = state_new % ydot(net_ienuc)
@@ -276,4 +273,4 @@ subroutine find_timestep_limiter_diffusion(lo, hi, state, slo, shi, nc_s, &
     enddo
 
 end subroutine find_timestep_limiter_diffusion
-#endif
+#endif

Exec/science/nova/inputs_nova_t7

@@ -2,11 +2,11 @@
 max_step = 100000
 
 # PROBLEM SIZE & GEOMETRY
-geometry.is_periodic = 1      0
+geometry.is_periodic = 1          0
 geometry.coord_sys   = 0                  # 0 => cart, 1 => RZ  2=>spherical
-geometry.prob_lo     = 0      0
-geometry.prob_hi     = 8.192e7 4.096e7
-amr.n_cell           = 256    128
+geometry.prob_lo     = 0          0
+geometry.prob_hi     = 12.192e7   6.096e7
+amr.n_cell           = 256        128
 
 # >>>>>>>>>>>>>  BC FLAGS <<<<<<<<<<<<<<<<
 # 0 = Interior           3 = Symmetry
@@ -33,22 +33,23 @@ castro.do_sponge = 0          #using sponge
 castro.ppm_type = 1
 castro.grav_source_type = 2
 castro.use_pslope = 1
-castro.pslope_cutoff_density = 1.0e2
+castro.pslope_cutoff_density = 6.0e1
 
 castro.use_flattening = 1
 
-castro.small_temp = 1.0e5
+castro.small_temp = 5.0e5
 castro.small_dens = 1.0e-6
-#castro.retry_small_density_cutoff = 1.0
+castro.retry_small_density_cutoff = 2.0e-5
+castro.limit_fluxes_on_small_dens = 1
 
 gravity.gravity_type = ConstantGrav
 gravity.const_grav   = -7.06e8
 
-castro.react_rho_min = 1.0e2
-castro.react_T_min = 1.0e7
+castro.react_rho_min = 6.0e1
+castro.react_T_min = 5.0e6
 
 # TIME STEP CONTROL
-castro.cfl            = 0.8     # cfl number for hyperbolic system
+castro.cfl            = 0.5     # cfl number for hyperbolic system
 castro.init_shrink    = 0.1     # scale back initial timestep
 castro.change_max     = 1.1     # max time step growth
 
@@ -80,7 +81,7 @@ amr.data_log = log
 
 # problem initialization
 
-problem.model_name =  "glasner_T73800.hse"
+problem.model_name =  "glasner_T7_1e+04cm.hse"
 
 problem.apply_vel_field = 0
 problem.apply_temp_field = 1

Exec/science/nova/inputs_nova_t9

@@ -2,21 +2,21 @@
 max_step = 100000
 
 # PROBLEM SIZE & GEOMETRY
-geometry.is_periodic = 1      0
+geometry.is_periodic = 1         0
 geometry.coord_sys   = 0                  # 0 => cart, 1 => RZ  2=>spherical
-geometry.prob_lo     = 0      0
-geometry.prob_hi     = 8.192e7 4.096e7
-amr.n_cell           = 256    128
+geometry.prob_lo     = 0         0
+geometry.prob_hi     = 12.192e7  6.096e7
+amr.n_cell           = 256       128
 
 # >>>>>>>>>>>>>  BC FLAGS <<<<<<<<<<<<<<<<
 # 0 = Interior           3 = Symmetry
 # 1 = Inflow             4 = SlipWall
 # 2 = Outflow            5 = NoSlipWall
 # >>>>>>>>>>>>>  BC FLAGS <<<<<<<<<<<<<<<<
-castro.lo_bc       =  0   1
+castro.lo_bc       =  0   3
 castro.hi_bc       =  0   2
-
 castro.yl_ext_bc_type = 1
+
 castro.hse_interp_temp = 1
 castro.hse_reflect_vels = 1
 
@@ -27,25 +27,28 @@ castro.ambient_outflow_vel = 1
 # WHICH PHYSICS
 castro.do_hydro = 1           #hydrodynamics
 castro.do_react = 1           #microphysics
-castro.add_ext_src = 0        #external sources
 castro.do_grav = 1            #gravity
 castro.do_sponge = 0          #using sponge
+
 castro.ppm_type = 1
-castro.riemann_solver = 0
+castro.grav_source_type = 2
+castro.use_pslope = 1
+castro.pslope_cutoff_density = 1.0e-5
+
 castro.use_flattening = 1
 
-castro.small_temp = 1.0e5
+castro.small_temp = 1.0e6
 castro.small_dens = 1.0e-6
-castro.retry_small_density_cutoff = 1.0
 
 gravity.gravity_type = ConstantGrav
 gravity.const_grav   = -7.33e8
+
 castro.react_rho_min = 1.0e2
-castro.react_T_min = 1.0e6
+castro.react_T_min = 1.0e7
 
 # TIME STEP CONTROL
-castro.cfl            = 0.5     # cfl number for hyperbolic system
-#castro.init_shrink    = 0.1   # scale back initial timestep
+castro.cfl            = 0.8     # cfl number for hyperbolic system
+castro.init_shrink    = 0.1     # scale back initial timestep
 castro.change_max     = 1.1     # max time step growth
 
 # DIAGNOSTICS & VERBOSITY
@@ -55,12 +58,12 @@ amr.v                 = 1       # verbosity in Amr.cpp
 
 
 # REFINEMENT / REGRIDDING
-amr.max_level       = 1       # maximum level number allowed
-amr.ref_ratio       = 2 2 2 2 # refinement ratio
-amr.regrid_int      = 2 2 2 2 # how often to regrid
-amr.blocking_factor = 64     # block factor in grid generation
+amr.max_level       = 5       # maximum level number allowed
+amr.ref_ratio       = 2 2 2 2 2 2 # refinement ratio
+amr.regrid_int      = 2 2 2 2 2 2 # how often to regrid
+amr.blocking_factor = 128     # block factor in grid generation
 amr.max_grid_size   = 256
-amr.n_error_buf     = 2 2 2 2 # number of buffer cells in error est
+amr.n_error_buf     = 2 2 2 2 2 2 # number of buffer cells in error est
 
 # CHECKPOINT FILES
 amr.check_file      = chk        # root name of checkpoint file
@@ -76,7 +79,7 @@ amr.data_log = log
 
 # problem initialization
 
-problem.model_name =  "glasner_T91600.hse"
+problem.model_name =  "glasner_T9_1e+04cm.hse"
 
 problem.apply_vel_field = 0
 #problem.velpert_scale = 5.0e5
@@ -101,13 +104,17 @@ integrator.atol_spec = 1.e-6
 integrator.rtol_enuc = 1.e-6
 
 # refinement
+amr.refinement_indicators = dens temp edot
 
-amr.refinement_indicators = dens temp
-
-amr.refine.dens.max_level = 1
-amr.refine.dens.value_greater = 1.0e3
+amr.refine.dens.max_level = 5
+amr.refine.dens.value_greater = 1.0e2
 amr.refine.dens.field_name = density
 
-amr.refine.temp.max_level = 1
-amr.refine.temp.value_greater = 2.0e7
+amr.refine.temp.max_level = 5
+amr.refine.temp.value_greater = 3.0e7
 amr.refine.temp.field_name = Temp
+
+amr.refine.edot.max_level = 5
+amr.refine.edot.value_greater = 1.0e10
+amr.refine.edot.field_name = enuc
+