From 363a427b9b865e3082d9b144a0b559679fa25ce5 Mon Sep 17 00:00:00 2001 From: Ben Ryan Date: Wed, 11 Dec 2024 10:25:32 -0700 Subject: [PATCH 1/8] Latest Kokkos develop --- external/Kokkos | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/Kokkos b/external/Kokkos index 15dc143e5f39..b9d10a411fff 160000 --- a/external/Kokkos +++ b/external/Kokkos @@ -1 +1 @@ -Subproject commit 15dc143e5f39949eece972a798e175c4b463d4b8 +Subproject commit b9d10a411fff8c8b6ec3fbf658a3df3c5798a251 From 260a79e0a3e654cee1c338c87232275f107ae9df Mon Sep 17 00:00:00 2001 From: Ben Ryan Date: Wed, 11 Dec 2024 11:07:57 -0700 Subject: [PATCH 2/8] Revert to 4.5 release --- external/Kokkos | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/Kokkos b/external/Kokkos index b9d10a411fff..09e775bfc585 160000 --- a/external/Kokkos +++ b/external/Kokkos @@ -1 +1 @@ -Subproject commit b9d10a411fff8c8b6ec3fbf658a3df3c5798a251 +Subproject commit 09e775bfc585840bb9ab1156cbd8d7d1c9e0cc6d From 52b3cd96d606d999e7b1a60878fd6892f98d85ee Mon Sep 17 00:00:00 2001 From: Ben Ryan Date: Wed, 11 Dec 2024 11:08:54 -0700 Subject: [PATCH 3/8] CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed674a70fd2c..10111311d92a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - [[PR 1161]](https://github.com/parthenon-hpc-lab/parthenon/pull/1161) Make flux field Metadata accessible, add Metadata::CellMemAligned flag, small perfomance upgrades ### Changed (changing behavior/API/variables/...) +- [[PR 1216]](https://github.com/parthenon-hpc-lab/parthenon/pull/1216) Move to Kokkos 4.5 - [[PR 1191]](https://github.com/parthenon-hpc-lab/parthenon/pull/1191) Update Kokkos version to 4.4.1 - [[PR 1209]](https://github.com/parthenon-hpc-lab/parthenon/pull/1209) Ordered history output - [[PR 1206]](https://github.com/parthenon-hpc-lab/parthenon/pull/1206) Leapfrog fix From 4788d35645d6f12116e89745e2330fd934c4d20f Mon Sep 17 00:00:00 2001 From: Ben Ryan Date: Wed, 18 Dec 2024 14:01:27 -0700 Subject: [PATCH 4/8] Add check against not being in a buffer --- .../parthinput.particle_tracers | 10 ++-- example/particle_tracers/particle_tracers.cpp | 28 ++++++++++ src/interface/swarm.cpp | 30 +++++++++++ src/interface/swarm_comms.cpp | 52 ++++++++++++++----- 4 files changed, 103 insertions(+), 17 deletions(-) diff --git a/example/particle_tracers/parthinput.particle_tracers b/example/particle_tracers/parthinput.particle_tracers index cc56117a3eac..f40d231a4f47 100644 --- a/example/particle_tracers/parthinput.particle_tracers +++ b/example/particle_tracers/parthinput.particle_tracers @@ -35,10 +35,10 @@ x3max = 0.5 ix3_bc = periodic ox3_bc = periodic - -nx1 = 16 -nx2 = 16 -nx3 = 1 +# +#nx1 = 16 +#nx2 = 16 +#nx3 = 1 tlim = 1.0 @@ -55,7 +55,7 @@ dt = 0.05 variables = advected, tracer_deposition -num_tracers = 10000 +num_tracers = 10 file_type = hdf5 diff --git a/example/particle_tracers/particle_tracers.cpp b/example/particle_tracers/particle_tracers.cpp index d4c10bbee1e6..5f982b0e984a 100644 --- a/example/particle_tracers/particle_tracers.cpp +++ b/example/particle_tracers/particle_tracers.cpp @@ -60,6 +60,8 @@ namespace advection_package { // *************************************************// Real EstimateTimestepBlock(MeshBlockData *mbd) { + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); auto pmb = mbd->GetBlockPointer(); auto pkg = pmb->packages.Get("advection_package"); const auto &cfl = pkg->Param("cfl"); @@ -76,6 +78,8 @@ Real EstimateTimestepBlock(MeshBlockData *mbd) { Real min_dt = dx_i / std::abs(vx + TINY_NUMBER); min_dt = std::min(min_dt, dx_j / std::abs(vy + TINY_NUMBER)); min_dt = std::min(min_dt, dx_k / std::abs(vz + TINY_NUMBER)); + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); return cfl * min_dt; } @@ -118,6 +122,8 @@ namespace particles_package { // *************************************************// Real EstimateTimestepBlock(MeshBlockData *mbd) { + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); auto pmb = mbd->GetBlockPointer(); auto pkg = pmb->packages.Get("advection_package"); @@ -133,12 +139,16 @@ Real EstimateTimestepBlock(MeshBlockData *mbd) { Real min_dt = dx_i / std::abs(vx + TINY_NUMBER); min_dt = std::min(min_dt, dx_j / std::abs(vy + TINY_NUMBER)); min_dt = std::min(min_dt, dx_k / std::abs(vz + TINY_NUMBER)); + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); // No CFL number for particles return min_dt; } std::shared_ptr Initialize(ParameterInput *pin) { + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); auto pkg = std::make_shared("particles_package"); int num_tracers = pin->GetOrAddReal("Tracers", "num_tracers", 100); @@ -152,6 +162,8 @@ std::shared_ptr Initialize(ParameterInput *pin) { pkg->AddSwarmValue("id", swarm_name, Metadata({Metadata::Integer})); pkg->EstimateTimestepBlock = EstimateTimestepBlock; + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); return pkg; } @@ -159,6 +171,8 @@ std::shared_ptr Initialize(ParameterInput *pin) { } // namespace particles_package TaskStatus AdvectTracers(MeshBlock *pmb, const StagedIntegrator *integrator) { + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); auto swarm = pmb->meshblock_data.Get()->GetSwarmData()->Get("tracers"); auto adv_pkg = pmb->packages.Get("advection_package"); @@ -183,11 +197,15 @@ TaskStatus AdvectTracers(MeshBlock *pmb, const StagedIntegrator *integrator) { z(n) += vz * dt; } }); + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); return TaskStatus::complete; } TaskStatus DepositTracers(MeshBlock *pmb) { + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); auto swarm = pmb->meshblock_data.Get()->GetSwarmData()->Get("tracers"); // Meshblock geometry @@ -238,11 +256,15 @@ TaskStatus DepositTracers(MeshBlock *pmb) { } } }); + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); return TaskStatus::complete; } TaskStatus CalculateFluxes(MeshBlockData *mbd) { + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); auto pmb = mbd->GetBlockPointer(); auto adv_pkg = pmb->packages.Get("advection_package"); const auto &vx = adv_pkg->Param("vx"); @@ -297,6 +319,8 @@ TaskStatus CalculateFluxes(MeshBlockData *mbd) { } }); } + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); return TaskStatus::complete; } @@ -308,6 +332,8 @@ TaskStatus CalculateFluxes(MeshBlockData *mbd) { // *************************************************// void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); auto &adv_pkg = pmb->packages.Get("advection_package"); auto &tr_pkg = pmb->packages.Get("particles_package"); auto &mbd = pmb->meshblock_data.Get(); @@ -394,6 +420,8 @@ void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { rng_pool.free_state(rng_gen); }); + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); } TaskCollection ParticleDriver::MakeTaskCollection(BlockList_t &blocks, int stage) { diff --git a/src/interface/swarm.cpp b/src/interface/swarm.cpp index 39f1e8a28123..15f661bb1151 100644 --- a/src/interface/swarm.cpp +++ b/src/interface/swarm.cpp @@ -192,6 +192,8 @@ void Swarm::Remove(const std::string &label) { } void Swarm::SetPoolMax(const std::int64_t nmax_pool) { + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); PARTHENON_REQUIRE(nmax_pool > nmax_pool_, "Must request larger pool size!"); std::int64_t n_new = nmax_pool - nmax_pool_; @@ -243,10 +245,14 @@ void Swarm::SetPoolMax(const std::int64_t nmax_pool) { for (auto &partition : pm->GetDefaultBlockPartitions()) { pm->mesh_data.Add("base", partition)->ClearSwarmCaches(); } + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); } NewParticlesContext Swarm::AddEmptyParticles(const int num_to_add) { PARTHENON_DEBUG_REQUIRE(num_to_add >= 0, "Cannot add negative numbers of particles!"); + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); auto pmb = GetBlockPointer(); @@ -281,6 +287,8 @@ NewParticlesContext Swarm::AddEmptyParticles(const int num_to_add) { } else { new_indices_max_idx_ = -1; } + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); // Create and return NewParticlesContext return NewParticlesContext(new_indices_max_idx_, new_indices_); @@ -289,6 +297,8 @@ NewParticlesContext Swarm::AddEmptyParticles(const int num_to_add) { // Updates the empty_indices_ array so the first N elements contain an ascending list of // indices into empty elements of the swarm pool, where N is the number of empty indices void Swarm::UpdateEmptyIndices() { + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); auto &mask = mask_; auto &empty_indices = empty_indices_; @@ -316,12 +326,16 @@ void Swarm::UpdateEmptyIndices() { empty_indices(empty_indices_scan(n) - 1) = n; } }); + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); } // No active particles: nmax_active_index = inactive_max_active_index (= -1) // No particles removed: nmax_active_index unchanged // Particles removed: nmax_active_index is new max active index void Swarm::RemoveMarkedParticles() { + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); int &max_active_index = max_active_index_; auto &mask = mask_; @@ -345,10 +359,16 @@ void Swarm::RemoveMarkedParticles() { num_active_ -= num_removed; UpdateEmptyIndices(); + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); } void Swarm::Defrag() { + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); if (GetNumActive() == 0) { + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); return; } @@ -429,6 +449,8 @@ void Swarm::Defrag() { // Update max_active_index_ max_active_index_ = num_active_ - 1; + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); } /// @@ -438,6 +460,8 @@ void Swarm::Defrag() { /// cell_sorted_ cell_sorted_number_: Per-cell array of number of particles in each cell /// void Swarm::SortParticlesByCell() { + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); auto pmb = GetBlockPointer(); auto &x = Get(swarm_position::x::name()).Get(); @@ -544,9 +568,13 @@ void Swarm::SortParticlesByCell() { } } }); + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); } void Swarm::Validate(bool test_comms) const { + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); auto mask = mask_; auto neighbor_indices = neighbor_indices_; auto empty_indices = empty_indices_; @@ -592,6 +620,8 @@ void Swarm::Validate(bool test_comms) const { Kokkos::Sum(num_err)); PARTHENON_REQUIRE(num_err == 0, "empty_indices_ array pointing to unmasked particle indices!"); + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); } } // namespace parthenon diff --git a/src/interface/swarm_comms.cpp b/src/interface/swarm_comms.cpp index 054ded34fabb..6a5717ca4546 100644 --- a/src/interface/swarm_comms.cpp +++ b/src/interface/swarm_comms.cpp @@ -31,6 +31,8 @@ namespace parthenon { /// GetNeighborBlockIndex() /// void Swarm::SetNeighborIndices_() { + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); auto pmb = GetBlockPointer(); const int ndim = pmb->pmy_mesh->ndim; auto neighbor_indices_h = neighbor_indices_.GetHostMirror(); @@ -131,9 +133,13 @@ void Swarm::SetNeighborIndices_() { } neighbor_indices_.DeepCopy(neighbor_indices_h); + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); } void Swarm::SetupPersistentMPI() { + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); auto pmb = GetBlockPointer(); vbswarm->SetupPersistentMPI(); @@ -154,9 +160,13 @@ void Swarm::SetupPersistentMPI() { neighbor_buffer_index.DeepCopy(neighbor_buffer_index_h); neighbor_buffer_index_ = neighbor_buffer_index; } + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); } void Swarm::LoadBuffers_() { + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); auto swarm_d = GetDeviceContext(); auto pmb = GetBlockPointer(); const int particle_size = GetParticleDataSize(); @@ -192,6 +202,8 @@ void Swarm::LoadBuffers_() { } else { buffer_sorted(n) = SwarmKey(this_block_, n); } + printf("sort idx (n): %i swarm_idx: %i\n", buffer_sorted(n).sort_idx_, + buffer_sorted(n).swarm_idx_); }); // sort by buffer index @@ -245,23 +257,29 @@ void Swarm::LoadBuffers_() { auto neighbor_buffer_index = neighbor_buffer_index_; // Loop over active particles buffer_sorted, use index n and buffer_start to /// get index in buffer m, pack that particle in buffer - pmb->par_for( + printf("MAX %i\n", max_active_index_); + parthenon::par_for( + // pmb->par_for( PARTHENON_AUTO_LABEL, 0, max_active_index_, KOKKOS_LAMBDA(const int n) { + printf("n: %i\n", n); auto p_index = buffer_sorted(n).swarm_idx_; if (swarm_d.IsActive(p_index)) { const int m = buffer_sorted(n).sort_idx_; - const int bufid = neighbor_buffer_index(m); if (m >= 0) { - const int bid = n - buffer_start[m]; - int buffer_index = bid * particle_size; - swarm_d.MarkParticleForRemoval(p_index); - for (int i = 0; i < realPackDim; i++) { - bdvar.send[bufid](buffer_index) = vreal(i, p_index); - buffer_index++; - } - for (int i = 0; i < intPackDim; i++) { - bdvar.send[bufid](buffer_index) = static_cast(vint(i, p_index)); - buffer_index++; + printf("m: %i\n", m); + const int bufid = neighbor_buffer_index(m); + if (m >= 0) { + const int bid = n - buffer_start[m]; + int buffer_index = bid * particle_size; + swarm_d.MarkParticleForRemoval(p_index); + for (int i = 0; i < realPackDim; i++) { + bdvar.send[bufid](buffer_index) = vreal(i, p_index); + buffer_index++; + } + for (int i = 0; i < intPackDim; i++) { + bdvar.send[bufid](buffer_index) = static_cast(vint(i, p_index)); + buffer_index++; + } } } } @@ -275,9 +293,13 @@ void Swarm::LoadBuffers_() { vbswarm->send_size[bufid] = 0; } } + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); } void Swarm::Send(BoundaryCommSubset phase) { + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); auto pmb = GetBlockPointer(); const int nneighbor = pmb->neighbors.size(); auto swarm_d = GetDeviceContext(); @@ -288,9 +310,13 @@ void Swarm::Send(BoundaryCommSubset phase) { // Send buffer data vbswarm->Send(phase); + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); } void Swarm::UnloadBuffers_() { + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); auto pmb = GetBlockPointer(); // Count received particles @@ -369,6 +395,8 @@ void Swarm::UnloadBuffers_() { } }); } + printf("%s:%i\n", __FILE__, __LINE__); + fflush(stdout); } bool Swarm::Receive(BoundaryCommSubset phase) { From 04a97bab846b085abfe2d3dfa6a91c451c754bf6 Mon Sep 17 00:00:00 2001 From: Ben Ryan Date: Wed, 18 Dec 2024 14:05:25 -0700 Subject: [PATCH 5/8] Remove printfs --- example/particle_tracers/particle_tracers.cpp | 28 ----------------- src/interface/swarm.cpp | 30 ------------------- src/interface/swarm_comms.cpp | 24 --------------- 3 files changed, 82 deletions(-) diff --git a/example/particle_tracers/particle_tracers.cpp b/example/particle_tracers/particle_tracers.cpp index 5f982b0e984a..d4c10bbee1e6 100644 --- a/example/particle_tracers/particle_tracers.cpp +++ b/example/particle_tracers/particle_tracers.cpp @@ -60,8 +60,6 @@ namespace advection_package { // *************************************************// Real EstimateTimestepBlock(MeshBlockData *mbd) { - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); auto pmb = mbd->GetBlockPointer(); auto pkg = pmb->packages.Get("advection_package"); const auto &cfl = pkg->Param("cfl"); @@ -78,8 +76,6 @@ Real EstimateTimestepBlock(MeshBlockData *mbd) { Real min_dt = dx_i / std::abs(vx + TINY_NUMBER); min_dt = std::min(min_dt, dx_j / std::abs(vy + TINY_NUMBER)); min_dt = std::min(min_dt, dx_k / std::abs(vz + TINY_NUMBER)); - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); return cfl * min_dt; } @@ -122,8 +118,6 @@ namespace particles_package { // *************************************************// Real EstimateTimestepBlock(MeshBlockData *mbd) { - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); auto pmb = mbd->GetBlockPointer(); auto pkg = pmb->packages.Get("advection_package"); @@ -139,16 +133,12 @@ Real EstimateTimestepBlock(MeshBlockData *mbd) { Real min_dt = dx_i / std::abs(vx + TINY_NUMBER); min_dt = std::min(min_dt, dx_j / std::abs(vy + TINY_NUMBER)); min_dt = std::min(min_dt, dx_k / std::abs(vz + TINY_NUMBER)); - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); // No CFL number for particles return min_dt; } std::shared_ptr Initialize(ParameterInput *pin) { - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); auto pkg = std::make_shared("particles_package"); int num_tracers = pin->GetOrAddReal("Tracers", "num_tracers", 100); @@ -162,8 +152,6 @@ std::shared_ptr Initialize(ParameterInput *pin) { pkg->AddSwarmValue("id", swarm_name, Metadata({Metadata::Integer})); pkg->EstimateTimestepBlock = EstimateTimestepBlock; - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); return pkg; } @@ -171,8 +159,6 @@ std::shared_ptr Initialize(ParameterInput *pin) { } // namespace particles_package TaskStatus AdvectTracers(MeshBlock *pmb, const StagedIntegrator *integrator) { - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); auto swarm = pmb->meshblock_data.Get()->GetSwarmData()->Get("tracers"); auto adv_pkg = pmb->packages.Get("advection_package"); @@ -197,15 +183,11 @@ TaskStatus AdvectTracers(MeshBlock *pmb, const StagedIntegrator *integrator) { z(n) += vz * dt; } }); - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); return TaskStatus::complete; } TaskStatus DepositTracers(MeshBlock *pmb) { - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); auto swarm = pmb->meshblock_data.Get()->GetSwarmData()->Get("tracers"); // Meshblock geometry @@ -256,15 +238,11 @@ TaskStatus DepositTracers(MeshBlock *pmb) { } } }); - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); return TaskStatus::complete; } TaskStatus CalculateFluxes(MeshBlockData *mbd) { - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); auto pmb = mbd->GetBlockPointer(); auto adv_pkg = pmb->packages.Get("advection_package"); const auto &vx = adv_pkg->Param("vx"); @@ -319,8 +297,6 @@ TaskStatus CalculateFluxes(MeshBlockData *mbd) { } }); } - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); return TaskStatus::complete; } @@ -332,8 +308,6 @@ TaskStatus CalculateFluxes(MeshBlockData *mbd) { // *************************************************// void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); auto &adv_pkg = pmb->packages.Get("advection_package"); auto &tr_pkg = pmb->packages.Get("particles_package"); auto &mbd = pmb->meshblock_data.Get(); @@ -420,8 +394,6 @@ void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { rng_pool.free_state(rng_gen); }); - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); } TaskCollection ParticleDriver::MakeTaskCollection(BlockList_t &blocks, int stage) { diff --git a/src/interface/swarm.cpp b/src/interface/swarm.cpp index 15f661bb1151..39f1e8a28123 100644 --- a/src/interface/swarm.cpp +++ b/src/interface/swarm.cpp @@ -192,8 +192,6 @@ void Swarm::Remove(const std::string &label) { } void Swarm::SetPoolMax(const std::int64_t nmax_pool) { - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); PARTHENON_REQUIRE(nmax_pool > nmax_pool_, "Must request larger pool size!"); std::int64_t n_new = nmax_pool - nmax_pool_; @@ -245,14 +243,10 @@ void Swarm::SetPoolMax(const std::int64_t nmax_pool) { for (auto &partition : pm->GetDefaultBlockPartitions()) { pm->mesh_data.Add("base", partition)->ClearSwarmCaches(); } - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); } NewParticlesContext Swarm::AddEmptyParticles(const int num_to_add) { PARTHENON_DEBUG_REQUIRE(num_to_add >= 0, "Cannot add negative numbers of particles!"); - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); auto pmb = GetBlockPointer(); @@ -287,8 +281,6 @@ NewParticlesContext Swarm::AddEmptyParticles(const int num_to_add) { } else { new_indices_max_idx_ = -1; } - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); // Create and return NewParticlesContext return NewParticlesContext(new_indices_max_idx_, new_indices_); @@ -297,8 +289,6 @@ NewParticlesContext Swarm::AddEmptyParticles(const int num_to_add) { // Updates the empty_indices_ array so the first N elements contain an ascending list of // indices into empty elements of the swarm pool, where N is the number of empty indices void Swarm::UpdateEmptyIndices() { - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); auto &mask = mask_; auto &empty_indices = empty_indices_; @@ -326,16 +316,12 @@ void Swarm::UpdateEmptyIndices() { empty_indices(empty_indices_scan(n) - 1) = n; } }); - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); } // No active particles: nmax_active_index = inactive_max_active_index (= -1) // No particles removed: nmax_active_index unchanged // Particles removed: nmax_active_index is new max active index void Swarm::RemoveMarkedParticles() { - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); int &max_active_index = max_active_index_; auto &mask = mask_; @@ -359,16 +345,10 @@ void Swarm::RemoveMarkedParticles() { num_active_ -= num_removed; UpdateEmptyIndices(); - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); } void Swarm::Defrag() { - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); if (GetNumActive() == 0) { - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); return; } @@ -449,8 +429,6 @@ void Swarm::Defrag() { // Update max_active_index_ max_active_index_ = num_active_ - 1; - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); } /// @@ -460,8 +438,6 @@ void Swarm::Defrag() { /// cell_sorted_ cell_sorted_number_: Per-cell array of number of particles in each cell /// void Swarm::SortParticlesByCell() { - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); auto pmb = GetBlockPointer(); auto &x = Get(swarm_position::x::name()).Get(); @@ -568,13 +544,9 @@ void Swarm::SortParticlesByCell() { } } }); - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); } void Swarm::Validate(bool test_comms) const { - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); auto mask = mask_; auto neighbor_indices = neighbor_indices_; auto empty_indices = empty_indices_; @@ -620,8 +592,6 @@ void Swarm::Validate(bool test_comms) const { Kokkos::Sum(num_err)); PARTHENON_REQUIRE(num_err == 0, "empty_indices_ array pointing to unmasked particle indices!"); - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); } } // namespace parthenon diff --git a/src/interface/swarm_comms.cpp b/src/interface/swarm_comms.cpp index 6a5717ca4546..46c8b0470269 100644 --- a/src/interface/swarm_comms.cpp +++ b/src/interface/swarm_comms.cpp @@ -31,8 +31,6 @@ namespace parthenon { /// GetNeighborBlockIndex() /// void Swarm::SetNeighborIndices_() { - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); auto pmb = GetBlockPointer(); const int ndim = pmb->pmy_mesh->ndim; auto neighbor_indices_h = neighbor_indices_.GetHostMirror(); @@ -133,13 +131,9 @@ void Swarm::SetNeighborIndices_() { } neighbor_indices_.DeepCopy(neighbor_indices_h); - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); } void Swarm::SetupPersistentMPI() { - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); auto pmb = GetBlockPointer(); vbswarm->SetupPersistentMPI(); @@ -160,13 +154,9 @@ void Swarm::SetupPersistentMPI() { neighbor_buffer_index.DeepCopy(neighbor_buffer_index_h); neighbor_buffer_index_ = neighbor_buffer_index; } - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); } void Swarm::LoadBuffers_() { - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); auto swarm_d = GetDeviceContext(); auto pmb = GetBlockPointer(); const int particle_size = GetParticleDataSize(); @@ -202,8 +192,6 @@ void Swarm::LoadBuffers_() { } else { buffer_sorted(n) = SwarmKey(this_block_, n); } - printf("sort idx (n): %i swarm_idx: %i\n", buffer_sorted(n).sort_idx_, - buffer_sorted(n).swarm_idx_); }); // sort by buffer index @@ -257,11 +245,9 @@ void Swarm::LoadBuffers_() { auto neighbor_buffer_index = neighbor_buffer_index_; // Loop over active particles buffer_sorted, use index n and buffer_start to /// get index in buffer m, pack that particle in buffer - printf("MAX %i\n", max_active_index_); parthenon::par_for( // pmb->par_for( PARTHENON_AUTO_LABEL, 0, max_active_index_, KOKKOS_LAMBDA(const int n) { - printf("n: %i\n", n); auto p_index = buffer_sorted(n).swarm_idx_; if (swarm_d.IsActive(p_index)) { const int m = buffer_sorted(n).sort_idx_; @@ -293,13 +279,9 @@ void Swarm::LoadBuffers_() { vbswarm->send_size[bufid] = 0; } } - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); } void Swarm::Send(BoundaryCommSubset phase) { - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); auto pmb = GetBlockPointer(); const int nneighbor = pmb->neighbors.size(); auto swarm_d = GetDeviceContext(); @@ -310,13 +292,9 @@ void Swarm::Send(BoundaryCommSubset phase) { // Send buffer data vbswarm->Send(phase); - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); } void Swarm::UnloadBuffers_() { - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); auto pmb = GetBlockPointer(); // Count received particles @@ -395,8 +373,6 @@ void Swarm::UnloadBuffers_() { } }); } - printf("%s:%i\n", __FILE__, __LINE__); - fflush(stdout); } bool Swarm::Receive(BoundaryCommSubset phase) { From d79d399b4dab484006fde01eaaac769cadf5860b Mon Sep 17 00:00:00 2001 From: Ben Ryan Date: Wed, 18 Dec 2024 14:08:32 -0700 Subject: [PATCH 6/8] CHANGELOG.md --- CHANGELOG.md | 1 + example/particle_tracers/parthinput.particle_tracers | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10111311d92a..62b35447f459 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - [[PR 1172]](https://github.com/parthenon-hpc-lab/parthenon/pull/1172) Make parthenon manager robust against external MPI init and finalize calls ### Fixed (not changing behavior/API/variables/...) +- [[PR 1217]](https://github.com/parthenon-hpc-lab/parthenon/pull/1217) Swarm comm debug fix - [[PR 1215]](https://github.com/parthenon-hpc-lab/parthenon/pull/1215) Fix issue with uninitialized input parameter block data - [[PR 1211]](https://github.com/parthenon-hpc-lab/parthenon/pull/1211) remove inline from WriteTaskGraph - [[PR 1188]](https://github.com/parthenon-hpc-lab/parthenon/pull/1188) Fix hdf5 output issue for metadata none variables, update test. diff --git a/example/particle_tracers/parthinput.particle_tracers b/example/particle_tracers/parthinput.particle_tracers index f40d231a4f47..cc56117a3eac 100644 --- a/example/particle_tracers/parthinput.particle_tracers +++ b/example/particle_tracers/parthinput.particle_tracers @@ -35,10 +35,10 @@ x3max = 0.5 ix3_bc = periodic ox3_bc = periodic -# -#nx1 = 16 -#nx2 = 16 -#nx3 = 1 + +nx1 = 16 +nx2 = 16 +nx3 = 1 tlim = 1.0 @@ -55,7 +55,7 @@ dt = 0.05 variables = advected, tracer_deposition -num_tracers = 10 +num_tracers = 10000 file_type = hdf5 From f12d25104a46e8a6dbcf618cf3d86ebbdcfdb443 Mon Sep 17 00:00:00 2001 From: Ben Ryan Date: Wed, 18 Dec 2024 14:09:12 -0700 Subject: [PATCH 7/8] Further cleanup --- src/interface/swarm_comms.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/interface/swarm_comms.cpp b/src/interface/swarm_comms.cpp index 46c8b0470269..b1e49d16a911 100644 --- a/src/interface/swarm_comms.cpp +++ b/src/interface/swarm_comms.cpp @@ -245,14 +245,12 @@ void Swarm::LoadBuffers_() { auto neighbor_buffer_index = neighbor_buffer_index_; // Loop over active particles buffer_sorted, use index n and buffer_start to /// get index in buffer m, pack that particle in buffer - parthenon::par_for( - // pmb->par_for( + pmb->par_for( PARTHENON_AUTO_LABEL, 0, max_active_index_, KOKKOS_LAMBDA(const int n) { auto p_index = buffer_sorted(n).swarm_idx_; if (swarm_d.IsActive(p_index)) { const int m = buffer_sorted(n).sort_idx_; if (m >= 0) { - printf("m: %i\n", m); const int bufid = neighbor_buffer_index(m); if (m >= 0) { const int bid = n - buffer_start[m]; From 2087ee9e2af2ccbf6df1e287ca098b6f63892d52 Mon Sep 17 00:00:00 2001 From: Ben Ryan Date: Wed, 18 Dec 2024 14:26:22 -0700 Subject: [PATCH 8/8] oops --- src/interface/swarm_comms.cpp | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/interface/swarm_comms.cpp b/src/interface/swarm_comms.cpp index b1e49d16a911..4df9606b01e2 100644 --- a/src/interface/swarm_comms.cpp +++ b/src/interface/swarm_comms.cpp @@ -252,18 +252,16 @@ void Swarm::LoadBuffers_() { const int m = buffer_sorted(n).sort_idx_; if (m >= 0) { const int bufid = neighbor_buffer_index(m); - if (m >= 0) { - const int bid = n - buffer_start[m]; - int buffer_index = bid * particle_size; - swarm_d.MarkParticleForRemoval(p_index); - for (int i = 0; i < realPackDim; i++) { - bdvar.send[bufid](buffer_index) = vreal(i, p_index); - buffer_index++; - } - for (int i = 0; i < intPackDim; i++) { - bdvar.send[bufid](buffer_index) = static_cast(vint(i, p_index)); - buffer_index++; - } + const int bid = n - buffer_start[m]; + int buffer_index = bid * particle_size; + swarm_d.MarkParticleForRemoval(p_index); + for (int i = 0; i < realPackDim; i++) { + bdvar.send[bufid](buffer_index) = vreal(i, p_index); + buffer_index++; + } + for (int i = 0; i < intPackDim; i++) { + bdvar.send[bufid](buffer_index) = static_cast(vint(i, p_index)); + buffer_index++; } } }