From 694227094198b8c1fe101af22dc506e9aeff9ebe Mon Sep 17 00:00:00 2001 From: Dusan Jovic <48258889+DusanJovic-NOAA@users.noreply.github.com> Date: Tue, 19 Mar 2024 17:18:20 -0400 Subject: [PATCH] Fix type mismatch compiler error when gfortran 10 is used without '-fallow-argument-mismatch' flag (#770) * Resolve argument mismatch errors when using gfortran * Switch from 'use mpi' to 'use mpi_f08' * More argument mismatch fixes * Merge Dom's ccpp-framework and ccpp-physics feature/depend_on_mpi braches * Check output_grid type and inline post compatibility * If output grid is 'cubed_sphere_grid' AND inline post is turned on print error and terminate the model. * Use type(MPI_Comm) in io/module_wrt_grid_comp.F90 --- CMakeLists.txt | 1 + atmos_model.F90 | 2 +- ccpp/config/ccpp_prebuild_config.py | 1 + ccpp/data/GFS_typedefs.F90 | 28 +++++++--------------------- ccpp/data/GFS_typedefs.meta | 2 +- ccpp/framework | 2 +- ccpp/physics | 2 +- io/module_write_netcdf.F90 | 10 ++++++---- io/module_write_restart_netcdf.F90 | 8 ++++---- io/module_wrt_grid_comp.F90 | 20 +++++++++++++------- io/post_fv3.F90 | 6 +++--- module_fcst_grid_comp.F90 | 6 +++--- module_fv3_config.F90 | 5 +++-- 13 files changed, 45 insertions(+), 48 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5d0dcf6f6..eede1ef1e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,7 @@ endif() ### CCPP ############################################################################### +set(MPI ON) add_subdirectory(ccpp) ############################################################################### diff --git a/atmos_model.F90 b/atmos_model.F90 index 3f5de6b2c..6a0834c65 100644 --- a/atmos_model.F90 +++ b/atmos_model.F90 @@ -450,7 +450,7 @@ end subroutine update_atmos_radiation_physics ! variable type are allocated for the global grid (without halo regions). ! subroutine atmos_timestep_diagnostics(Atmos) - use mpi + use mpi_f08 implicit none type (atmos_data_type), intent(in) :: Atmos !--- local variables--- diff --git a/ccpp/config/ccpp_prebuild_config.py b/ccpp/config/ccpp_prebuild_config.py index b27e5a2f6..976b1d4f6 100755 --- a/ccpp/config/ccpp_prebuild_config.py +++ b/ccpp/config/ccpp_prebuild_config.py @@ -29,6 +29,7 @@ TYPEDEFS_NEW_METADATA = { 'ccpp_types' : { 'ccpp_t' : 'cdata', + 'MPI_Comm' : '', 'ccpp_types' : '', }, 'machine' : { diff --git a/ccpp/data/GFS_typedefs.F90 b/ccpp/data/GFS_typedefs.F90 index 7cd1d4a35..5fae71258 100644 --- a/ccpp/data/GFS_typedefs.F90 +++ b/ccpp/data/GFS_typedefs.F90 @@ -1,5 +1,6 @@ module GFS_typedefs + use mpi_f08 use machine, only: kind_phys, kind_dbl_prec, kind_sngl_prec use physcons, only: con_cp, con_fvirt, con_g, rholakeice, & con_hvap, con_hfus, con_pi, con_rd, con_rv, & @@ -94,7 +95,7 @@ module GFS_typedefs type GFS_init_type integer :: me !< my MPI-rank integer :: master !< master MPI-rank - integer :: fcst_mpi_comm !< forecast tasks mpi communicator + type(MPI_Comm) :: fcst_mpi_comm !< forecast tasks mpi communicator integer :: fcst_ntasks !< total number of forecast tasks integer :: tile_num !< tile number for this MPI rank integer :: isc !< starting i-index for this MPI-domain @@ -700,7 +701,7 @@ module GFS_typedefs integer :: me !< MPI rank designator integer :: master !< MPI rank of master atmosphere processor - integer :: communicator !< MPI communicator + type(MPI_Comm) :: communicator !< MPI communicator integer :: ntasks !< MPI size in communicator integer :: nthreads !< OpenMP threads available for physics integer :: nlunit !< unit for namelist @@ -3306,7 +3307,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & real(kind=kind_phys), dimension(:), intent(in) :: bk logical, intent(in) :: restart logical, intent(in) :: hydrostatic - integer, intent(in) :: communicator + type(MPI_Comm), intent(in) :: communicator integer, intent(in) :: ntasks integer, intent(in) :: nthreads @@ -3316,9 +3317,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & integer :: seed0 logical :: exists real(kind=kind_phys) :: tem - real(kind=kind_phys) :: rinc(5) - real(kind=kind_sngl_prec) :: rinc4(5) - real(kind=kind_dbl_prec) :: rinc8(5) + real(kind=kind_dbl_prec) :: rinc(5) real(kind=kind_phys) :: wrk(1) real(kind=kind_phys), parameter :: con_hr = 3600. @@ -3974,7 +3973,6 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & real(kind=kind_phys) :: radar_tten_limits(2) = (/ limit_unspecified, limit_unspecified /) integer :: itime - integer :: w3kindreal,w3kindint !--- END NAMELIST VARIABLES @@ -5608,19 +5606,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & Model%cdec = -9999. Model%clstp = -9999 rinc(1:5) = 0 - call w3kind(w3kindreal,w3kindint) - if (w3kindreal == 8) then - rinc8(1:5) = 0 - call w3difdat(jdat,idat,4,rinc8) - rinc = rinc8 - else if (w3kindreal == 4) then - rinc4(1:5) = 0 - call w3difdat(jdat,idat,4,rinc4) - rinc = rinc4 - else - write(0,*)' FATAL ERROR: Invalid w3kindreal' - call abort - endif + call w3difdat(jdat,idat,4,rinc) Model%phour = rinc(4)/con_hr Model%fhour = (rinc(4) + Model%dtp)/con_hr Model%zhour = mod(Model%phour,Model%fhzero) @@ -6418,7 +6404,7 @@ subroutine control_print(Model) print *, 'basic control parameters' print *, ' me : ', Model%me print *, ' master : ', Model%master - print *, ' communicator : ', Model%communicator + print *, ' communicator : ', Model%communicator%mpi_val print *, ' nlunit : ', Model%nlunit print *, ' fn_nml : ', trim(Model%fn_nml) print *, ' fhzero : ', Model%fhzero diff --git a/ccpp/data/GFS_typedefs.meta b/ccpp/data/GFS_typedefs.meta index f7b9239da..20a3b4ce4 100644 --- a/ccpp/data/GFS_typedefs.meta +++ b/ccpp/data/GFS_typedefs.meta @@ -3332,7 +3332,7 @@ long_name = MPI communicator units = index dimensions = () - type = integer + type = MPI_Comm [ntasks] standard_name = number_of_mpi_tasks long_name = number of MPI tasks in communicator diff --git a/ccpp/framework b/ccpp/framework index f1db41531..011db4f80 160000 --- a/ccpp/framework +++ b/ccpp/framework @@ -1 +1 @@ -Subproject commit f1db41531c772cc60b71296334e0a9616e8b4a91 +Subproject commit 011db4f80a02cba6d65958ace56e8efb197be62b diff --git a/ccpp/physics b/ccpp/physics index 9f4a96bcf..983968088 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 9f4a96bcfb18c19336e308b41d8c8cb11a347c48 +Subproject commit 9839680885141338fb14ff09bab6fe87a051b820 diff --git a/io/module_write_netcdf.F90 b/io/module_write_netcdf.F90 index d9d8ff970..7d4289a89 100644 --- a/io/module_write_netcdf.F90 +++ b/io/module_write_netcdf.F90 @@ -16,7 +16,7 @@ !> @author Dusan Jovic @date Nov 1, 2017 module module_write_netcdf - use mpi + use mpi_f08 use esmf use netcdf use module_fv3_io_def,only : ideflate, quantize_mode, quantize_nsd, zstandard_level, & @@ -44,13 +44,15 @@ module module_write_netcdf !> !> @author Dusan Jovic @date Nov 1, 2017 subroutine write_netcdf(wrtfb, filename, & - use_parallel_netcdf, mpi_comm, mype, & + use_parallel_netcdf, comm, mype, & grid_id, rc) ! + use mpi_f08 + type(ESMF_FieldBundle), intent(in) :: wrtfb character(*), intent(in) :: filename logical, intent(in) :: use_parallel_netcdf - integer, intent(in) :: mpi_comm + type(MPI_Comm), intent(in) :: comm integer, intent(in) :: mype integer, intent(in) :: grid_id integer, optional,intent(out) :: rc @@ -233,7 +235,7 @@ subroutine write_netcdf(wrtfb, filename, & if (par) then ncerr = nf90_create(trim(filename),& cmode=IOR(NF90_CLOBBER,NF90_NETCDF4),& - comm=mpi_comm, info = MPI_INFO_NULL, ncid=ncid); NC_ERR_STOP(ncerr) + comm=comm%mpi_val, info = MPI_INFO_NULL%mpi_val, ncid=ncid); NC_ERR_STOP(ncerr) else ncerr = nf90_create(trim(filename),& cmode=IOR(NF90_CLOBBER,NF90_NETCDF4),& diff --git a/io/module_write_restart_netcdf.F90 b/io/module_write_restart_netcdf.F90 index 2fd4c7732..4d1e73a9d 100644 --- a/io/module_write_restart_netcdf.F90 +++ b/io/module_write_restart_netcdf.F90 @@ -7,7 +7,7 @@ module module_write_restart_netcdf - use mpi + use mpi_f08 use esmf use fms use mpp_mod, only : mpp_chksum ! needed for fms 2023.02 @@ -24,13 +24,13 @@ module module_write_restart_netcdf !---------------------------------------------------------------------------------------- subroutine write_restart_netcdf(wrtfb, filename, & - use_parallel_netcdf, mpi_comm, mype, & + use_parallel_netcdf, comm, mype, & rc) ! type(ESMF_FieldBundle), intent(in) :: wrtfb character(*), intent(in) :: filename logical, intent(in) :: use_parallel_netcdf - integer, intent(in) :: mpi_comm + type(MPI_Comm), intent(in) :: comm integer, intent(in) :: mype integer, optional,intent(out) :: rc ! @@ -223,7 +223,7 @@ subroutine write_restart_netcdf(wrtfb, filename, & if (par) then ncerr = nf90_create(trim(filename),& cmode=IOR(NF90_CLOBBER,NF90_NETCDF4),& - comm=mpi_comm, info = MPI_INFO_NULL, ncid=ncid); NC_ERR_STOP(ncerr) + comm=comm%mpi_val, info = MPI_INFO_NULL%mpi_val, ncid=ncid); NC_ERR_STOP(ncerr) else ncerr = nf90_create(trim(filename),& ! cmode=IOR(NF90_CLOBBER,NF90_64BIT_OFFSET),& diff --git a/io/module_wrt_grid_comp.F90 b/io/module_wrt_grid_comp.F90 index de0cedb6f..eb3ccd8dc 100644 --- a/io/module_wrt_grid_comp.F90 +++ b/io/module_wrt_grid_comp.F90 @@ -26,7 +26,7 @@ module module_wrt_grid_comp ! !--------------------------------------------------------------------------------- ! - use mpi + use mpi_f08 use esmf use fms_mod, only : uppercase use fms @@ -67,7 +67,7 @@ module module_wrt_grid_comp integer,save :: itasks, jtasks !<-- # of write tasks in i/j direction in the current group integer,save :: ngrids - integer,save :: wrt_mpi_comm !<-- the mpi communicator in the write comp + type(MPI_Comm),save :: wrt_mpi_comm !<-- the mpi communicator in the write comp integer,save :: idate(7), start_time(7) logical,save :: write_nsflip logical,save :: change_wrtidate=.false. @@ -159,7 +159,7 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, integer,dimension(2,6) :: decomptile integer,dimension(2) :: regDecomp !define delayout for the nest grid integer :: fieldCount - integer :: vm_mpi_comm + type(MPI_Comm) :: vm_mpi_comm character(40) :: fieldName type(ESMF_Config) :: cf, cf_output_grid type(ESMF_Info) :: info @@ -242,7 +242,7 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, ! call ESMF_VMGetCurrent(vm=VM,rc=RC) call ESMF_VMGet(vm=VM, localPet=wrt_int_state%mype, & - petCount=wrt_int_state%petcount,mpiCommunicator=vm_mpi_comm,rc=rc) + petCount=wrt_int_state%petcount,mpiCommunicator=vm_mpi_comm%mpi_val,rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return call mpi_comm_dup(vm_mpi_comm, wrt_mpi_comm, rc) @@ -253,7 +253,7 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, last_write_task = ntasks -1 lprnt = lead_write_task == wrt_int_state%mype - call fms_init(wrt_mpi_comm) + call fms_init(wrt_mpi_comm%mpi_val) ! print *,'in wrt, lead_write_task=', & ! lead_write_task,'last_write_task=',last_write_task, & @@ -386,6 +386,12 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, print *,'grid_id= ', n, ' output_grid= ', trim(output_grid(n)) end if + if (trim(output_grid(n)) == 'cubed_sphere_grid' .and. wrt_int_state%write_dopost) then + write(0,*) 'wrt_initialize_p1: Inline post is not supported with cubed_sphere_grid outputs' + call ESMF_LogWrite("wrt_initialize_p1: Inline post is not supported with cubed_sphere_grid output",ESMF_LOGMSG_ERROR,rc=RC) + call ESMF_Finalize(endflag=ESMF_END_ABORT) + end if + call ESMF_ConfigGetAttribute(config=CF, value=itasks,default=1,label ='itasks:',rc=rc) jtasks = ntasks if(itasks > 0 ) jtasks = ntasks/itasks @@ -3386,7 +3392,7 @@ subroutine ioCompRun(comp, importState, exportState, clock, rc) logical :: thereAreVerticals integer :: ch_dimid, timeiso_varid character(len=ESMF_MAXSTR) :: time_iso - integer :: wrt_mpi_comm + type(MPI_Comm) :: wrt_mpi_comm type(ESMF_VM) :: vm rc = ESMF_SUCCESS @@ -3439,7 +3445,7 @@ subroutine ioCompRun(comp, importState, exportState, clock, rc) call ESMF_GridCompGet(comp, localPet=localPet, petCount=petCount, vm=vm, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_VMGet(vm=vm, mpiCommunicator=wrt_mpi_comm, rc=rc) + call ESMF_VMGet(vm=vm, mpiCommunicator=wrt_mpi_comm%mpi_val, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return if (petCount > 1) then diff --git a/io/post_fv3.F90 b/io/post_fv3.F90 index 30eae6ba4..e00accfa0 100644 --- a/io/post_fv3.F90 +++ b/io/post_fv3.F90 @@ -1,6 +1,6 @@ module post_fv3 - use mpi + use mpi_f08 use module_fv3_io_def, only : wrttasks_per_group, filename_base, & lon1, lat1, lon2, lat2, dlon, dlat, & @@ -56,7 +56,7 @@ subroutine post_run_fv3(wrt_int_state,grid_id,mype,mpicomp,lead_write, & type(wrt_internal_state),intent(inout) :: wrt_int_state integer,intent(in) :: grid_id integer,intent(in) :: mype - integer,intent(in) :: mpicomp + type(MPI_Comm),intent(in) :: mpicomp integer,intent(in) :: lead_write integer,intent(in) :: itasks, jtasks integer,intent(in) :: mynfhr @@ -586,7 +586,7 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) type(wrt_internal_state),intent(in) :: wrt_int_state integer,intent(in) :: grid_id integer,intent(in) :: mype - integer,intent(in) :: mpicomp + type(MPI_Comm),intent(in) :: mpicomp ! !----------------------------------------------------------------------- ! diff --git a/module_fcst_grid_comp.F90 b/module_fcst_grid_comp.F90 index ea622369c..df28a246b 100644 --- a/module_fcst_grid_comp.F90 +++ b/module_fcst_grid_comp.F90 @@ -15,7 +15,7 @@ module module_fcst_grid_comp ! !--------------------------------------------------------------------------------- ! - use mpi + use mpi_f08 use esmf use nuopc @@ -593,7 +593,7 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) call ESMF_VMGetCurrent(vm=vm,rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_VMGet(vm=vm, localPet=mype, mpiCommunicator=fcst_mpi_comm, & + call ESMF_VMGet(vm=vm, localPet=mype, mpiCommunicator=fcst_mpi_comm%mpi_val, & petCount=fcst_ntasks, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return if (mype == 0) write(*,*)'in fcst comp init, fcst_ntasks=',fcst_ntasks @@ -615,7 +615,7 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return if (mype == 0) print *,'af ufs config,restart_interval=',restart_interval ! - call fms_init(fcst_mpi_comm) + call fms_init(fcst_mpi_comm%mpi_val) call mpp_init() initClock = mpp_clock_id( 'Initialization' ) call mpp_clock_begin (initClock) !nesting problem diff --git a/module_fv3_config.F90 b/module_fv3_config.F90 index 9733fa8fc..6874c5cc5 100644 --- a/module_fv3_config.F90 +++ b/module_fv3_config.F90 @@ -6,6 +6,8 @@ !> !> @author Jun Wang @date 01/2017 module module_fv3_config + + use mpi_f08 use esmf implicit none @@ -18,12 +20,11 @@ module module_fv3_config integer :: first_kdt !> MPI communicator for the forecast grid component - integer :: fcst_mpi_comm + type(MPI_Comm) :: fcst_mpi_comm !> Total number of mpi tasks for the forecast grid components integer :: fcst_ntasks - !> ID number for the coupled grids integer :: cpl_grid_id