diff --git a/engine/report/json/report_json.cpp b/engine/report/json/report_json.cpp index 5a25c85d0d6..47787909a4f 100644 --- a/engine/report/json/report_json.cpp +++ b/engine/report/json/report_json.cpp @@ -1022,6 +1022,19 @@ void profileset_json2( const profileset::profilesets_t& profileset, const sim_t& obj[ "iterations" ] = as( result.iterations() ); + if ( sim.profileset_cull.enabled ) + { + if ( profileset->culled() ) + { + obj[ "culled" ] = true; + obj[ "culled_reason" ] = profileset->culled_reason(); + obj[ "culled_iterations" ] = profileset->culled_iterations(); + obj[ "culled_mean" ] = profileset->culled_mean(); + obj[ "culled_error" ] = profileset->culled_error(); + obj[ "culled_error_type" ] = profileset->culled_error_type_cstr(); + } + } + if ( profileset->results() > 1 ) { auto results2 = obj[ "additional_metrics" ].make_array(); @@ -1068,6 +1081,20 @@ void profileset_json3( const profileset::profilesets_t& profilesets, const sim_t obj[ "name" ] = profileset->name(); auto results_obj = obj[ "metrics" ].make_array(); + // Profileset culling metadata at profileset level + if ( sim.profileset_cull.enabled ) + { + if ( profileset->culled() ) + { + obj[ "culled" ] = true; + obj[ "culled_reason" ] = profileset->culled_reason(); + obj[ "culled_iterations" ] = profileset->culled_iterations(); + obj[ "culled_mean" ] = profileset->culled_mean(); + obj[ "culled_error" ] = profileset->culled_error(); + obj[ "culled_error_type" ] = profileset->culled_error_type_cstr(); + } + } + for ( size_t midx = 0; midx < sim.profileset_metric.size(); ++midx ) { const auto& result = profileset->result( sim.profileset_metric[ midx ] ); @@ -1231,6 +1258,19 @@ void to_json( const ::report::json::report_configuration_t& report_configuration options_root[ "profileset_metric" ] = util::scale_metric_type_abbrev( sim.profileset_metric.front() ); options_root[ "profileset_multiactor_base_name" ] = sim.profileset_multiactor_base_name; + if ( sim.profileset_cull.enabled ) + { + auto cull = options_root[ "profileset_cull" ]; + cull[ "enabled" ] = true; + cull[ "method" ] = sim.profileset_cull.method_name(); + cull[ "min_iterations" ] = sim.profileset_cull.min_iterations; + if ( sim.profileset_cull.uses_alpha() ) + cull[ "alpha" ] = sim.profileset_cull.alpha; + else + cull[ "margin" ] = sim.profileset_cull.margin; + cull[ "metric" ] = util::scale_metric_type_abbrev( sim.profileset_cull.metric ); + } + to_json( options_root[ "dbc" ], *sim.dbc ); if ( sim.scaling->calculate_scale_factors ) @@ -1308,6 +1348,26 @@ void to_json( const ::report::json::report_configuration_t& report_configuration add_non_zero( stats_root, "total_heal", sim.total_heal ); add_non_zero( stats_root, "total_absorb", sim.total_absorb ); + if ( sim.profileset_cull.enabled ) + { + const std::string best_name = sim.profileset_cull.best_name.empty() + ? sim.profileset_multiactor_base_name + : sim.profileset_cull.best_name; + + auto cull_stats = stats_root[ "profileset_cull" ]; + cull_stats[ "method" ] = sim.profileset_cull.method_name(); + cull_stats[ "metric" ] = util::scale_metric_type_abbrev( sim.profileset_cull.metric ); + + add_non_zero( cull_stats, "best_name", best_name ); + + if ( sim.profileset_cull.baseline_seeded ) + { + cull_stats[ "best_error" ] = sim.profileset_cull.best_error; + cull_stats[ "best_iterations" ] = sim.profileset_cull.best_iterations; + cull_stats[ "best_mean" ] = sim.profileset_cull.best_mean; + } + } + if ( sim.report_details != 0 ) { // Targets diff --git a/engine/report/report_html_sim.cpp b/engine/report/report_html_sim.cpp index b1c0bd7520d..de7079b7685 100644 --- a/engine/report/report_html_sim.cpp +++ b/engine/report/report_html_sim.cpp @@ -1160,6 +1160,48 @@ void print_profilesets( std::ostream& out, const profileset::profilesets_t& prof print_profilesets_chart( out, sim ); + // Profileset culling indicator and culled list (moved below charts) + if ( sim.profileset_cull.enabled ) + { + out << "

Profileset culling details

\n"; + out << "
\n"; + + out << "
"; + out << "Profileset culling enabled: method=" + << sim.profileset_cull.method_name() + << ", min_iters=" << sim.profileset_cull.min_iterations; + if ( sim.profileset_cull.uses_alpha() ) + out << ", alpha=" << sim.profileset_cull.alpha; + else + out << ", margin=" << sim.profileset_cull.margin; + out << "
\n"; + + // List culled profiles if any + bool any_culled = false; + for ( const auto& pset : profilesets.profilesets() ) + { + if ( pset->culled() ) { any_culled = true; break; } + } + if ( any_culled ) + { + out << "
Culled profiles:
    "; + for ( const auto& pset : profilesets.profilesets() ) + { + if ( !pset->culled() ) continue; + out << "
  • " << util::encode_html( pset->name() ) + << ": " << util::encode_html( pset->culled_reason() ) + << " (iters=" << pset->culled_iterations() + << ", mean=" << util::round( pset->culled_mean(), 2 ) + << ", error=" << util::round( pset->culled_error(), 4 ) + << ", type=" << pset->culled_error_type_cstr() << ")"; + out << "
    • "; + } + out << "
\n"; + } + + out << "
\n"; // end toggle-content + } + out << ""; out << ""; } diff --git a/engine/sc_main.cpp b/engine/sc_main.cpp index 2e746602a84..dce3337d599 100644 --- a/engine/sc_main.cpp +++ b/engine/sc_main.cpp @@ -362,6 +362,10 @@ int sim_t::main( const std::vector& args ) plot->analyze(); reforge_plot->analyze(); + if ( profileset_cull.enabled ) { + seed_profileset_cull_from_baseline(); + } + if ( canceled == 0 && !profilesets->iterate( this ) ) canceled = true; else diff --git a/engine/sim/profileset.cpp b/engine/sim/profileset.cpp index fa20cab2db4..33a4576a27b 100644 --- a/engine/sim/profileset.cpp +++ b/engine/sim/profileset.cpp @@ -110,6 +110,7 @@ void simulate_profileset( sim_t* parent, profileset::profile_set_t& set, sim_t*& // Reset random seed for the profileset sims profile_sim -> seed = 0; profile_sim -> profileset_enabled = true; + profile_sim -> profileset_current_name = set.name(); profile_sim -> report_details = 0; if ( parent -> profileset_work_threads > 0 ) { @@ -156,6 +157,22 @@ void simulate_profileset( sim_t* parent, profileset::profile_set_t& set, sim_t*& .stddev( data.std_dev ) .mean_stddev( data.mean_std_dev ) .iterations( progress.current_iterations ); + + // If culled, persist snapshot information for JSON/HTML reporting on primary metric only + if ( profile_sim->culled && metric == parent->profileset_metric.front() ) + { + // error to record depends on method: CI mode wants half-width, t-test wants SE + auto etype = ( parent->profileset_cull.prefers_standard_error() ) ? + profileset::profile_set_t::cull_error_type_e::STANDARD_ERROR : + profileset::profile_set_t::cull_error_type_e::CI_HALF_WIDTH; + double err_val = parent->profileset_cull.select_error(data.mean_std_dev * parent->confidence_estimator, data.mean_std_dev / sqrt(parent->iterations) ); + set.set_culled( true, + profile_sim->culled_reason, + progress.current_iterations, + data.mean, + err_val, + etype ); + } } ); if ( ! parent -> profileset_output_data.empty() ) @@ -174,6 +191,11 @@ void simulate_profileset( sim_t* parent, profileset::profile_set_t& set, sim_t*& parent -> event_mgr.total_events_processed += profile_sim -> event_mgr.total_events_processed; set.cleanup_options(); + + if ( profile_sim->culled ) + { + fmt::print( stderr, "\nProfileset '{}' culled: {}\n", set.name(), profile_sim->culled_reason ); + } } // Figure out if the option defines new actor(s) with their own scope diff --git a/engine/sim/profileset.hpp b/engine/sim/profileset.hpp index 70230f0124d..5b33081dc44 100644 --- a/engine/sim/profileset.hpp +++ b/engine/sim/profileset.hpp @@ -384,6 +384,19 @@ class profile_set_t std::vector m_results; std::unique_ptr m_output_data; + // Culled metadata (set when profileset culling terminates a run early) + bool m_culled = false; + std::string m_culled_reason; + uint64_t m_culled_iterations = 0; + double m_culled_mean = 0.0; + double m_culled_error = 0.0; + // CI half-width or standard error depending on cull method + +public: + enum class cull_error_type_e { NONE = 0, CI_HALF_WIDTH, STANDARD_ERROR }; +private: + cull_error_type_e m_culled_error_type = cull_error_type_e::NONE; + public: profile_set_t( std::string name, sim_control_t* opts, bool has_output ); @@ -415,6 +428,35 @@ class profile_set_t return *m_output_data; } + + // Culled metadata accessors + bool culled() const { return m_culled; } + const std::string& culled_reason() const { return m_culled_reason; } + uint64_t culled_iterations() const { return m_culled_iterations; } + double culled_mean() const { return m_culled_mean; } + double culled_error() const { return m_culled_error; } + profile_set_t::cull_error_type_e culled_error_type() const { return m_culled_error_type; } + const char* culled_error_type_cstr() const { + switch ( m_culled_error_type ) { + case profile_set_t::cull_error_type_e::CI_HALF_WIDTH: return "ci_half_width"; + case profile_set_t::cull_error_type_e::STANDARD_ERROR: return "standard_error"; + default: return "none"; + } + } + void set_culled( bool culled, + std::string reason, + uint64_t iterations, + double mean, + double error, + profile_set_t::cull_error_type_e etype ) + { + m_culled = culled; + m_culled_reason = std::move( reason ); + m_culled_iterations = iterations; + m_culled_mean = mean; + m_culled_error = error; + m_culled_error_type = etype; + } }; class worker_t diff --git a/engine/sim/sim.cpp b/engine/sim/sim.cpp index 095c93a8e17..676f30ddb30 100644 --- a/engine/sim/sim.cpp +++ b/engine/sim/sim.cpp @@ -2112,9 +2112,13 @@ void sim_t::datacollection_end() void sim_t::analyze_error() { if ( thread_index != 0 ) return; - if ( target_error <= 0 ) return; if ( current_iteration < 1 ) return; + // We want analyze_error to run either for normal target_error handling OR for profileset culling elimination logic + bool need_precision_handling = target_error > 0; + bool need_culling = ( parent && parent->profileset_cull.enabled && profileset_enabled ); + if ( !need_precision_handling && !need_culling ) return; + work_queue -> lock(); // First iterations of each thread are considered statistically insignificant and not @@ -2137,6 +2141,7 @@ void sim_t::analyze_error() double mean_total=0; int mean_count=0; + double current_standard_error = 0.0; current_error = 0; @@ -2154,7 +2159,9 @@ void sim_t::analyze_error() current_mean = cd.target_metric.mean(); if ( current_mean != 0 ) { - current_error = sim_t::distribution_mean_error( *this, cd.target_metric ) / current_mean; + double mean_error = sim_t::distribution_mean_error( *this, cd.target_metric ); + current_error = mean_error / current_mean; + current_standard_error = cd.target_metric.std_dev / sqrt(cd.target_metric.count()); } } } @@ -2210,7 +2217,7 @@ void sim_t::analyze_error() current_error *= 100; - if ( current_error > 0 ) + if ( need_precision_handling && current_error > 0 ) { if ( current_error < target_error ) { @@ -2236,6 +2243,82 @@ void sim_t::analyze_error() } } + if ( need_culling && current_mean > 0 ) + { + auto &s = parent->profileset_cull; + // Convert relative percent error to absolute half-width + double abs_error = ( current_error / 100.0 ) * current_mean; + + // ensure enough iterations + if ( n_iterations >= s.min_iterations ) + { + AUTO_LOCK( s.mtx ); + // If no best yet, only promote if baseline hasn't been seeded + // (i.e., fallback to old behavior only if baseline seeding fails) + if ( s.best_name.empty() && !s.baseline_seeded ) + { + s.best_name = profileset_current_name; + s.best_mean = current_mean; + s.best_error = s.select_error( abs_error, current_standard_error ); + s.best_iterations = n_iterations; + + fmt::print( stderr, "\nprofileset_cull: initial best '{}' mean={:.2f} err={:.4f} ({:.3f}% rel) iters={}\n", s.best_name, s.best_mean, s.best_error, current_error, s.best_iterations ); + } + else + { + if ( profileset_current_name == s.best_name ) + { + // Update best uncertainty window if shrunk + if ( s.select_error( abs_error, current_standard_error ) < s.best_error ) + { + s.best_error = s.select_error( abs_error, current_standard_error ); + s.best_mean = current_mean; + s.best_iterations = n_iterations; + } + } + else + { + // Candidate: test elimination vs current best using current method + double error_for_method = s.select_error( abs_error, current_standard_error ); + if ( s.should_cull( current_mean, error_for_method, n_iterations, s.best_mean, s.best_error ) ) + { + culled = true; + // Use a friendly label for baseline if best_name is empty + const std::string& best_label = s.best_name.empty() ? parent->profileset_multiactor_base_name : s.best_name; + culled_reason = fmt::format( "profileset_cull: eliminated vs '{}' using {}", best_label, s.method_name() ); + if ( s.verbose >= 1 ) + { + fmt::print( stderr, "\n{}\n", culled_reason ); + } + interrupt(); + work_queue -> unlock(); + return; + } + // Promotion check if candidate clearly better + bool promote = s.should_promote( current_mean, + s.select_error( abs_error, current_standard_error ), + n_iterations, + s.best_mean, + s.best_error ); + + if ( promote ) + { + s.best_name = profileset_current_name; + s.best_mean = current_mean; + s.best_error = s.select_error( abs_error, current_standard_error ); + s.best_iterations = n_iterations; + if ( s.verbose >= 1 ) + { + fmt::print( stderr, + "\nprofileset_cull: new best '{}' mean={:.2f} error={:.4f} iters={}\n", + s.best_name, s.best_mean, s.best_error, s.best_iterations ); + } + } + } + } + } + } + work_queue -> unlock(); } @@ -3040,6 +3123,172 @@ void sim_t::analyze_iteration_data() range::sort( high_iteration_data, iteration_data_cmp ); } +// --- profileset_cull_state_t helpers (concrete methods, no strategy classes) --- + +double sim_t::profileset_cull_state_t::z_critical_one_sided() const +{ + if ( alpha <= 0.001 ) return 3.09; // z_0.001 + if ( alpha <= 0.005 ) return 2.58; // z_0.005 + if ( alpha <= 0.01 ) return 2.33; // z_0.01 + if ( alpha <= 0.05 ) return 1.64; // z_0.05 + return 1.28; // z_0.10 +} + +bool sim_t::profileset_cull_state_t::ttest_is_significant( double candidate_mean, double candidate_se, + int candidate_iterations, double best_mean_val, + double best_se, ttest_direction dir ) const +{ + // Require enough iterations for normal approximation + if ( candidate_iterations < 30 || best_iterations < 30 ) + { + if ( verbose >= 2 ) + { + fmt::print( stderr, "profileset_cull: TTEST {}=NO | insufficient iterations (cand={}, best={}, need >= 30)\n", + ( dir == ttest_direction::BETTER ? "better" : "worse" ), candidate_iterations, best_iterations ); + } + return false; + } + + const double pooled_se = std::sqrt( candidate_se * candidate_se + best_se * best_se ); + if ( pooled_se <= 0 ) + { + if ( verbose >= 2 ) + { + fmt::print( stderr, "profileset_cull: TTEST {}=NO | pooled_se <= 0 ({:.6f})\n", + ( dir == ttest_direction::BETTER ? "better" : "worse" ), pooled_se ); + } + return false; + } + const double t_stat = ( candidate_mean - best_mean_val ) / pooled_se; + const double tcrit = z_critical_one_sided(); + if ( dir == ttest_direction::WORSE ) + return t_stat < -tcrit; // one-sided lower tail + else + return t_stat > tcrit; // one-sided upper tail +} + +bool sim_t::profileset_cull_state_t::should_cull( double candidate_mean, double candidate_error_ci_or_se, + int candidate_iterations, double best_mean_val, + double best_error_val ) const +{ + if ( candidate_iterations < min_iterations ) + { + if ( verbose >= 2 ) + { + fmt::print( stderr, + "profileset_cull: should_cull=NO (candidate_iterations={}, min_iterations={})\n", + candidate_iterations, min_iterations ); + } + return false; + } + + if ( method == CI_OVERLAP ) + { + double safety = margin > 0 ? margin * best_mean_val : 0.0; + double candidate_upper = candidate_mean + candidate_error_ci_or_se; + double best_lower = best_mean_val - best_error_val; + bool result = candidate_upper + safety < best_lower; + + if ( verbose >= 2 ) + { + fmt::print( stderr, + "profileset_cull: should_cull={} | method=CI_OVERLAP | cand_mean={:.2f} cand_err={:.4f} cand_upper={:.2f} | best_mean={:.2f} best_err={:.4f} best_lower={:.2f} | safety={:.4f} | test: {:.2f} < {:.2f}\n", + result ? "YES" : "NO", candidate_mean, candidate_error_ci_or_se, candidate_upper, best_mean_val, best_error_val, + best_lower, safety, candidate_upper + safety, best_lower ); + if ( result ) + { + double candidate_lower = candidate_mean - candidate_error_ci_or_se; + double best_upper = best_mean_val + best_error_val; + fmt::print( stderr, + "\nprofileset_cull: compare CI cand='{}' [{:.2f},{:.2f}] vs best='{}' [{:.2f},{:.2f}] | culling=YES\n", + best_name.empty() ? "candidate" : "candidate", + candidate_lower, candidate_upper, + best_name.empty() ? "best" : best_name, + best_lower, best_upper ); + } + } + + return result; + } + else // T_TEST + { + // candidate_error_ci_or_se is standard error in T_TEST mode + bool result = ttest_is_significant( candidate_mean, candidate_error_ci_or_se, candidate_iterations, best_mean_val, best_error_val, + ttest_direction::WORSE ); + if ( verbose >= 2 && result ) + { + fmt::print( stderr, + "\nprofileset_cull: compare TTEST cand='{}' mean={:.2f} se={:.4f} vs best='{}' mean={:.2f} se={:.4f} | culling=YES\n", + best_name.empty() ? "candidate" : "candidate", + candidate_mean, candidate_error_ci_or_se, + best_name.empty() ? "best" : best_name, + best_mean_val, best_error_val ); + } + return result; + } +} + +bool sim_t::profileset_cull_state_t::should_promote( double candidate_mean, double candidate_error_ci_or_se, + int candidate_iterations, double best_mean_val, + double best_error_val ) const +{ + if ( method == CI_OVERLAP ) + { + return ( candidate_mean - candidate_error_ci_or_se ) > ( best_mean_val + best_error_val ); + } + else // T_TEST + { + return ttest_is_significant( candidate_mean, candidate_error_ci_or_se, candidate_iterations, best_mean_val, best_error_val, + ttest_direction::BETTER ); + } +} + +// sim_t::seed_profileset_cull_from_baseline ============================ + +void sim_t::seed_profileset_cull_from_baseline() +{ + // Only seed if culling is enabled and we haven't seeded already + if ( !profileset_cull.enabled || profileset_cull.baseline_seeded ) + return; + + // Only the parent (baseline) sim should seed + if ( parent || profileset_enabled ) + return; + + // Get the baseline player for the chosen metric + if ( player_no_pet_list.empty() || profileset_report_player_index >= player_no_pet_list.size() ) + return; + + const auto baseline_player = player_no_pet_list[ profileset_report_player_index ]; + + // Get baseline statistics for the culling metric + auto baseline_data = profileset::metric_data( baseline_player, profileset_cull.metric ); + + // Calculate absolute error (half-width) from relative error + double relative_error = baseline_data.mean_std_dev / baseline_data.mean; + double absolute_error = relative_error * baseline_data.mean; + + // Get iterations from current simulation progress + auto current_progress = progress(); + + // Seed the culling state with baseline values + { + std::lock_guard lock( profileset_cull.mtx ); + + profileset_cull.best_name = ""; + profileset_cull.best_mean = baseline_data.mean; + profileset_cull.best_error = profileset_cull.select_error( absolute_error, baseline_data.mean_std_dev ); + profileset_cull.best_iterations = current_progress.current_iterations; + profileset_cull.baseline_seeded = true; + + if ( profileset_cull.verbose >= 1 ) + { + fmt::print( stderr, "\nprofileset_cull: baseline seeded '{}' mean={:.2f} err={:.4f} ({:.3f}% rel) iters={}\n", + profileset_multiactor_base_name, profileset_cull.best_mean, profileset_cull.best_error, relative_error * 100.0, + profileset_cull.best_iterations); + } + } +} // sim_t::iterate =========================================================== @@ -3774,6 +4023,29 @@ void sim_t::create_options() add_option( opt_int( "min_report_iteration_data", min_report_iteration_data ) ); add_option( opt_bool( "average_range", average_range ) ); add_option( opt_bool( "average_gauss", average_gauss ) ); + // Profileset culling (early elimination) options + add_option( opt_bool( "profileset_cull", profileset_cull.enabled ) ); + add_option( opt_func( "profileset_cull_method", []( sim_t* sim, util::string_view, util::string_view value ) { + auto m = sim_t::profileset_cull_state_t::parse_method( value ); + if ( m >= sim_t::profileset_cull_state_t::METHOD_MAX ) + { + sim->error( "Invalid profileset_cull_method '{}' , valid options: ci, t_test", value ); + return false; + } + sim->profileset_cull.method = m; + return true; + } ) ); + add_option( opt_string( "profileset_cull_metric", profileset_cull.cull_metric_str ) ); + add_option( opt_func( "profileset_cull_min_iterations", []( sim_t* sim, util::string_view, util::string_view value ) { + unsigned val = std::stoul( std::string( value ) ); + if ( val < 30 && sim->profileset_cull.method == sim_t::profileset_cull_state_t::T_TEST ) { + sim->error( "profileset_cull_min_iterations={} is too low for reliable t-test (need >= 30 for normal approximation)", val ); + } + sim->profileset_cull.min_iterations = val; + return true; + } ) ); + add_option( opt_float( "profileset_cull_alpha", profileset_cull.alpha ) ); + add_option( opt_int( "profileset_cull_verbose", profileset_cull.verbose ) ); // Misc add_option( opt_list( "party", party_encoding ) ); add_option( opt_func( "active", parse_active ) ); @@ -4238,6 +4510,38 @@ void sim_t::setup( sim_control_t* c ) if ( player_list.empty() && spell_query == nullptr && !display_bonus_ids && display_build <= 1 ) throw sc_runtime_error( "Nothing to sim!" ); + // Finalize profileset_cull configuration on parent sim only + if ( !parent && profileset_cull.enabled ) + { + // Determine metric + if ( !profileset_cull.cull_metric_str.empty() ) + { + auto m = util::parse_scale_metric( profileset_cull.cull_metric_str ); + if ( m == SCALE_METRIC_NONE ) + { + error( "profileset_cull: unknown metric '{}' disabling feature", profileset_cull.cull_metric_str ); + profileset_cull.enabled = false; + } + else + { + profileset_cull.metric = m; + } + } + else + { + // If only one profileset metric specified, use it; otherwise require explicit option + if ( profileset_metric.size() == 1 ) + { + profileset_cull.metric = profileset_metric.front(); + } + else + { + error( "profileset_cull: multiple profileset metrics active, specify profileset_cull_metric=... disabling feature" ); + profileset_cull.enabled = false; + } + } + } + range::for_each( player_list, []( player_t* p ) { p->validate_sim_options(); } ); if ( parent || profileset_enabled ) diff --git a/engine/sim/sim.hpp b/engine/sim/sim.hpp index d449957600d..608d6191e83 100644 --- a/engine/sim/sim.hpp +++ b/engine/sim/sim.hpp @@ -17,6 +17,7 @@ #include "util/util.hpp" #include "util/vector_with_callback.hpp" +#include #include #include @@ -96,6 +97,65 @@ struct sim_t : private sc_thread_t double current_mean; int analyze_error_interval, analyze_number; + // Profileset culling (early elimination) state (shared on parent sim) + struct profileset_cull_state_t { + mutex_t mtx; + bool enabled = false; + // Metric currently only supports primary profileset metric (DPS-family) + scale_metric_e metric = SCALE_METRIC_DPS; + std::string best_name; // profileset name + double best_mean = 0.0; // mean of current best + double best_error = 0.0; // absolute half-width (same units as mean) + int best_iterations = 0; // iterations when last updated + bool baseline_seeded = false; // whether baseline has seeded the initial best + enum method_e { CI_OVERLAP, T_TEST, METHOD_MAX } method = T_TEST; + int min_iterations = 100; // minimum iterations before evaluating elimination + double margin = 0.001; // fractional safety margin for CI mode (fraction of best mean) + double alpha = 0.01; // alpha level for t-test mode (one-sided) + int verbose = 0; // 0 silent, 1 events, 2 verbose + std::string cull_metric_str; // raw option string for metric + + // Encapsulated decision helpers (no virtual dispatch yet) + double z_critical_one_sided() const; + enum class ttest_direction { BETTER, WORSE }; + bool ttest_is_significant(double candidate_mean, double candidate_se, int candidate_iterations, + double best_mean_val, double best_se, ttest_direction dir) const; + bool should_cull(double candidate_mean, double candidate_error_ci_or_se, int candidate_iterations, + double best_mean_val, double best_error_val) const; + bool should_promote(double candidate_mean, double candidate_error_ci_or_se, int candidate_iterations, + double best_mean_val, double best_error_val) const; + + static const char* method_to_string( method_e m ) { + switch ( m ) { + case T_TEST: + return "t_test"; + case CI_OVERLAP: + return "ci"; + default: + return "unknown"; + } + } + static method_e parse_method( util::string_view v ) { + for ( int i = static_cast( CI_OVERLAP ); i < static_cast( METHOD_MAX ); ++i ) { + auto e = static_cast( i ); + if ( util::str_compare_ci( v, method_to_string( e ) ) ) + return e; + } + return METHOD_MAX; // invalid + } + const char* method_name() const { return method_to_string( method ); } + bool prefers_standard_error() const { return method == T_TEST; } + bool uses_alpha() const { return method == T_TEST; } + double select_error(double candidate_ci_half_width, double candidate_standard_error) const { + return prefers_standard_error() ? candidate_standard_error : candidate_ci_half_width; + } + } profileset_cull; + + // Per-sim (child) flags used for reporting elimination + bool culled = false; // set true if this profileset was culled + std::string culled_reason; // human readable reason + std::string profileset_current_name; // name of the profileset for this sim (child only) + sim_control_t* control; sim_t* parent; player_t* target; @@ -716,6 +776,7 @@ struct sim_t : private sc_thread_t bool execute(); void analyze_error(); void analyze_iteration_data(); + void seed_profileset_cull_from_baseline(); void print_options(); void add_option( std::unique_ptr opt ); void create_options();