diff --git a/src/particle/src/algorithm/4C_particle_algorithm.cpp b/src/particle/src/algorithm/4C_particle_algorithm.cpp index 90f5c0aac5..629187fe44 100644 --- a/src/particle/src/algorithm/4C_particle_algorithm.cpp +++ b/src/particle/src/algorithm/4C_particle_algorithm.cpp @@ -134,16 +134,31 @@ void Particle::ParticleAlgorithm::setup() // setup initial rigid bodies if (particlerigidbody_) setup_initial_rigid_bodies(); - // distribute load among processors + // distribute load among processors (first pass: by particle count) distribute_load_among_procs(); // ghost particles on other processors particleengine_->ghost_particles(); - // build global id to local index map + if (particleinteraction_) + { + // build particle-to-particle neighbors only, to populate bin_interaction_costs_ for + // cost-aware rebalancing below; the global id map and wall neighbor relations are skipped + // here since both would be invalidated by the redistribution pass right after anyway + build_potential_neighbor_relation(/*include_wall_neighbors=*/false); + + // second redistribution pass: redistribute load using interaction-pair counts as bin weights + // now that bin_interaction_costs_ is available from the neighbor build above + distribute_load_among_procs(); + + // re-ghost after second redistribution + particleengine_->ghost_particles(); + } + + // build global id to local index map (once, for the final distribution) particleengine_->build_global_id_to_local_index_map(); - // build potential neighbor relation + // build potential neighbor relation, including wall neighbors, for the final distribution if (particleinteraction_) build_potential_neighbor_relation(); // setup initial states @@ -970,14 +985,14 @@ void Particle::ParticleAlgorithm::distribute_load_among_procs() Core::IO::cout(Core::IO::verbose) << "distribute load in step " << step() << Core::IO::endl; } -void Particle::ParticleAlgorithm::build_potential_neighbor_relation() +void Particle::ParticleAlgorithm::build_potential_neighbor_relation(bool include_wall_neighbors) { TEUCHOS_FUNC_TIME_MONITOR("Particle::ParticleAlgorithm::build_potential_neighbor_relation"); // build particle to particle neighbors particleengine_->build_particle_to_particle_neighbors(); - if (particlewall_) + if (particlewall_ && include_wall_neighbors) { // relate bins to column wall elements particlewall_->relate_bins_to_col_wall_eles(); diff --git a/src/particle/src/algorithm/4C_particle_algorithm.hpp b/src/particle/src/algorithm/4C_particle_algorithm.hpp index 25078b49a7..dc19f0d7d7 100644 --- a/src/particle/src/algorithm/4C_particle_algorithm.hpp +++ b/src/particle/src/algorithm/4C_particle_algorithm.hpp @@ -340,8 +340,10 @@ namespace Particle /*! * \brief build potential neighbor relation * + * \param[in] include_wall_neighbors also build particle to wall neighbors + * */ - void build_potential_neighbor_relation(); + void build_potential_neighbor_relation(bool include_wall_neighbors = true); /*! * \brief set initial conditions diff --git a/src/particle/src/engine/4C_particle_engine.cpp b/src/particle/src/engine/4C_particle_engine.cpp index 6ffb6ca48c..1cb901a14d 100644 --- a/src/particle/src/engine/4C_particle_engine.cpp +++ b/src/particle/src/engine/4C_particle_engine.cpp @@ -474,8 +474,9 @@ void Particle::ParticleEngine::build_particle_to_particle_neighbors() // relate half neighboring bins to owned bins if (not binning_->validhalfneighboringbins_) relate_half_neighboring_bins_to_owned_bins(); - // clear potential particle neighbors + // clear potential particle neighbors and accumulated interaction costs potentialparticleneighbors_.clear(); + bin_interaction_costs_.clear(); // invalidate flag denoting validity of particle neighbors map validparticleneighbors_ = false; @@ -562,6 +563,12 @@ void Particle::ParticleEngine::build_particle_to_particle_neighbors() potentialparticleneighbors_.push_back( std::make_pair(std::make_tuple(type, Status::Owned, ownedindex), std::make_tuple(neighbortype, neighborstatus, neighborindex))); + + // accumulate interaction cost for load balancing weight estimation, weighted by the + // average of the dynamic load balance factors of the two interacting particle types + bin_interaction_costs_[gidofbin] += + 0.5 * + (typeweights_[static_cast(type)] + typeweights_[static_cast(neighbortype)]); } } } @@ -1978,7 +1985,7 @@ void Particle::ParticleEngine::remove_particles_from_containers( // iterate in reversed order over particles to be removed std::set::reverse_iterator rit; for (rit = particlestoremove[static_cast(type)].rbegin(); - rit != particlestoremove[static_cast(type)].rend(); ++rit) + rit != particlestoremove[static_cast(type)].rend(); ++rit) container->remove_particle(*rit); } @@ -2098,12 +2105,19 @@ void Particle::ParticleEngine::determine_bin_weights() // get global id of bin const int gidofbin = binning_->binrowmap_->gid(rowlidofbin); - // iterate over owned particles in current bin - for (const auto& particleIt : particlestobins_[binning_->bincolmap_->lid(gidofbin)]) + // use neighbor pair count from the previous interaction evaluation as a proxy for the + // compute cost of this bin; fall back to particle count * type weight when no interaction + // data is available yet (e.g. on the very first load redistribution) + auto costIt = bin_interaction_costs_.find(gidofbin); + if (costIt != bin_interaction_costs_.end()) + { + binning_->binweights_->get_vector(0).get_values()[rowlidofbin] += costIt->second; + } + else { - // add weight of particle of specific type - binning_->binweights_->get_vector(0).get_values()[rowlidofbin] += - typeweights_[static_cast(particleIt.first)]; + for (const auto& particleIt : particlestobins_[binning_->bincolmap_->lid(gidofbin)]) + binning_->binweights_->get_vector(0).get_values()[rowlidofbin] += + typeweights_[static_cast(particleIt.first)]; } } } diff --git a/src/particle/src/engine/4C_particle_engine.hpp b/src/particle/src/engine/4C_particle_engine.hpp index bcd9734813..92ea036246 100644 --- a/src/particle/src/engine/4C_particle_engine.hpp +++ b/src/particle/src/engine/4C_particle_engine.hpp @@ -739,6 +739,9 @@ namespace Particle //! relate potential particle neighbors of all types and statuses PotentialParticleNeighbors potentialparticleneighbors_; + //! accumulated interaction pair count per bin global id for load balancing weight estimation + std::unordered_map bin_interaction_costs_; + //! owned particles being communicated (transfered/distributed) to target processors std::vector> communicatedparticletargets_;