Add group sizes to PerformanceResults + report group latency results

Author: pgmpablo157321

loadgen/bindings/python_api.cc

@@ -340,7 +340,7 @@ void StartTestWithGroupedQSL(
   GroupedQuerySampleLibraryTrampoline* qsl_cast =
       reinterpret_cast<GroupedQuerySampleLibraryTrampoline*>(qsl);
   LogSettings default_log_settings;
-  assert(test_settings.use_grouped_qsl);
+  assert(TestSettings.use_grouped_qsl);
   mlperf::StartTest(sut_cast, qsl_cast, test_settings, default_log_settings,
                     audit_config_filename);
 }

loadgen/issue_query_controller.cc

@@ -107,6 +107,14 @@ void QueryMetadata::CoalesceQueries(QueryMetadata* queries, size_t first,
 
 void QueryMetadata::Decoalesce() { query_to_send.resize(1); }
 
+std::vector<QuerySampleIndex> QueryMetadata::GetSampleIndices(){
+  std::vector<QuerySampleIndex> sample_indices;
+  for (auto s: this->samples_){
+    sample_indices.push_back(s.sample_index);
+  }
+  return sample_indices;
+}
+
 /// \brief A base template that should never be used since each scenario has
 /// its own specialization.
 template <TestScenario scenario>

loadgen/issue_query_controller.h

@@ -110,6 +110,8 @@ class QueryMetadata {
   /// \brief Set a coalesced query back to its original state.
   void Decoalesce();
 
+  std::vector<QuerySampleIndex> GetSampleIndices();
+
  public:
   std::vector<QuerySample> query_to_send;
   const std::chrono::nanoseconds scheduled_delta;

loadgen/loadgen.cc

@@ -417,7 +417,7 @@ std::vector<QueryMetadata> GenerateQueries(
       }
     } else if (settings.use_grouped_qsl) {
       g = grouped_sample_distribution(sample_rng);
-      group_size = qsl->GroupSize(qsl->GroupOf(groups_first[g]));
+      group_size = qsl->GroupSize(qsl->GroupOf(loaded_samples[groups_first[g]]));
     } else {
       for (auto& s : samples) {
         s = loaded_samples[settings.performance_issue_unique
@@ -639,6 +639,19 @@ PerformanceResult IssueQueries(SystemUnderTest* sut,
           queries[i].all_samples_done_time);
     }
   }
+  std::vector<size_t> group_sizes;
+  std::vector<QuerySampleIndex> sample_index;
+  if (settings.use_grouped_qsl){
+    for (size_t i = 0; i < queries.size(); i++){
+      for (auto s: queries[i].GetSampleIndices()){
+        sample_index.push_back(s);
+      }
+    }
+  }
+
+  for (size_t i = 0; i < qsl->NumberOfGroups(); i++) {
+    group_sizes.push_back(qsl->GroupSize(i));
+  }
 
   return PerformanceResult{
       std::move(sample_latencies),
@@ -649,7 +662,10 @@ PerformanceResult IssueQueries(SystemUnderTest* sut,
       final_query_issued_time,
       final_query_all_samples_done_time,
       TokenPerformanceResults{first_token_latencies, time_per_output_token_arr,
-                              tokens_per_sample}};
+                              tokens_per_sample},
+      std::move(group_sizes),
+      std::move(sample_index)
+      };
 }
 
 void LoadSamplesToRam(QuerySampleLibrary* qsl,

loadgen/results.cc

@@ -146,6 +146,50 @@ void PerformanceSummary::ProcessTokenLatencies() {
   }
 }
 
+void PerformanceSummary::ProcessGroupLatencies(){
+  if (pr.sample_latencies.empty() || pr.group_sizes.empty() || (!settings.use_grouped_qsl) || (group_latencies_processed)) {
+    return;
+  }
+  sample_count = pr.sample_latencies.size();
+  std::vector<size_t> group_initial_idx;
+  std::vector<QuerySampleLatency> group_latencies;
+  size_t acum_group_idx = 0;
+
+  for(size_t i = 0; i < pr.group_sizes.size(); i++){
+    group_initial_idx.push_back(acum_group_idx);
+    acum_group_idx += pr.group_sizes[i];
+  }
+  size_t i = 0;
+  QuerySampleLatency accumulated_sample_latency = 0;
+
+  while (i < pr.sample_index.size()) {
+    auto sample_index = pr.sample_index[i];
+    auto low = std::lower_bound (group_initial_idx.begin(), group_initial_idx.end(), sample_index);
+    size_t idx = low - group_initial_idx.begin();
+    if (group_initial_idx[idx] == sample_index){
+      group_count++;
+      QuerySampleLatency q = 0;
+      for (size_t j = 0; j < pr.group_sizes[idx]; j++){
+        q += pr.sample_latencies[i + j];
+      }
+      group_latencies.push_back(q);
+      accumulated_sample_latency += q;
+      i += pr.group_sizes[idx];
+    } else {
+      i = pr.sample_index.size();
+    }
+  }
+  std::sort(group_latencies.begin(), group_latencies.end());
+  group_latency_min = group_latencies.front();
+  group_latency_max = group_latencies.back();
+  group_latency_mean = accumulated_sample_latency / group_count;
+
+  for (auto& lp : group_latency_percentiles) {
+    lp.query_latency = group_latencies[group_count * lp.percentile];
+  }
+  group_latencies_processed = true;
+};
+
 bool PerformanceSummary::EarlyStopping(
     std::string* recommendation, int64_t queries_issued,
     std::vector<QuerySampleLatency>* sample_latencies,
@@ -380,6 +424,9 @@ bool PerformanceSummary::PerfConstraintsMet(std::string* recommendation) {
 }
 
 void PerformanceSummary::LogSummary(AsyncSummary& summary) {
+  if (settings.use_grouped_qsl) {
+    ProcessGroupLatencies();
+  }
   ProcessLatencies();
 
   summary(
@@ -480,6 +527,15 @@ void PerformanceSummary::LogSummary(AsyncSummary& summary) {
     }
   }
 
+  if (settings.use_grouped_qsl) {
+    double gps_as_completed =
+            group_count / pr.final_query_all_samples_done_time;
+    summary("Groups per second: ", group_count / pr.max_latency);
+    summary("Completed tokens per second: ",
+            DoubleToString(gps_as_completed));
+        
+  }
+
   std::string min_duration_recommendation;
   std::string perf_constraints_recommendation;
   std::string early_stopping_recommendation;
@@ -630,6 +686,17 @@ void PerformanceSummary::LogSummary(AsyncSummary& summary) {
     }
   }
 
+  if (settings.use_grouped_qsl) {
+    summary("Min group latency (ns)          : ", group_latency_min);
+    summary("Max group latency (ns)          : ", group_latency_max);
+    summary("Mean group latency (ns)         : ", group_latency_mean);
+    for (auto& lp : group_latency_percentiles) {
+      summary(
+          DoubleToString(lp.percentile * 100) + " group percentile latency (ns)   : ",
+          lp.query_latency);
+    }
+  }
+
   summary(
       "\n"
       "================================================\n"
@@ -640,6 +707,9 @@ void PerformanceSummary::LogSummary(AsyncSummary& summary) {
 
 void PerformanceSummary::LogDetail(AsyncDetail& detail) {
 #if USE_NEW_LOGGING_FORMAT
+  if (settings.use_grouped_qsl) {
+    ProcessGroupLatencies();
+  }
   ProcessLatencies();
 
   // General validity checking
@@ -848,8 +918,23 @@ void PerformanceSummary::LogDetail(AsyncDetail& detail) {
         break;
       }
     }
-#endif
   }
+
+  if(settings.use_grouped_qsl) {
+    MLPERF_LOG(detail, "result_group_min_latency_ns",
+                 group_latency_min);
+    MLPERF_LOG(detail, "result_group_max_latency_ns",
+                group_latency_max);
+    MLPERF_LOG(detail, "result_group_mean_latency_ns",
+                group_latency_mean);
+    for (auto& lp : group_latency_percentiles) {
+      MLPERF_LOG(detail,
+                  "result_group_" + DoubleToString(lp.percentile * 100) +
+                      "_percentile_latency_ns",
+                  lp.query_latency);
+    }
+  }
+#endif
 }
 }  // namespace loadgen
 }  // namespace mlperf

loadgen/results.h

@@ -44,6 +44,8 @@ struct PerformanceResult {
   double final_query_issued_time;            // seconds from start.
   double final_query_all_samples_done_time;  // seconds from start.
   TokenPerformanceResults token_results;
+  std::vector<size_t> group_sizes;
+  std::vector<QuerySampleIndex> sample_index;
 };
 
 /// \brief Wraps PerformanceResult with relevant context to change how
@@ -99,6 +101,15 @@ struct PerformanceSummary {
   PercentileEntry tpot_percentiles[6] = {{.50}, {.90}, {.95},
                                          {.97}, {.99}, {.999}};
 
+  // Set by ProcessGroupLatencies
+  size_t group_count = 0;
+  bool group_latencies_processed = false;
+  QuerySampleLatency group_latency_min = 0;
+  QuerySampleLatency group_latency_max = 0;
+  QuerySampleLatency group_latency_mean = 0;
+
+  PercentileEntry group_latency_percentiles[6] = {{.50}, {.90}, {.95},
+                                                  {.97}, {.99}, {.999}};
 #if defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined(WIN64)
   // MSVC complains if there is no explicit constructor.
   // (target_latency_percentile above depends on construction with settings)
@@ -109,6 +120,7 @@ struct PerformanceSummary {
 #endif
   void ProcessLatencies();
   void ProcessTokenLatencies();
+  void ProcessGroupLatencies();
 
   bool MinDurationMet(std::string* recommendation);
   bool EarlyStopping(std::string* recommendation, int64_t queries_issued,

loadgen/test_settings_internal.cc

@@ -342,6 +342,9 @@ void LogRequestedTestSettings(const TestSettings &s) {
                s.performance_sample_count_override);
     MLPERF_LOG(detail, "requested_sample_concatenate_permutation",
                s.sample_concatenate_permutation);
+    MLPERF_LOG(detail, "requested_server_constant_gen",
+               s.server_constant_gen);
+    MLPERF_LOG(detail, "requested_use_grouped_qsl", s.use_grouped_qsl);
     // Token latencies specific values
     if (s.use_token_latencies) {
       MLPERF_LOG(detail, "requested_use_token_latencies",
@@ -458,6 +461,7 @@ void TestSettingsInternal::LogEffectiveSettings() const {
                s.sample_concatenate_permutation);
     MLPERF_LOG(detail, "effective_server_constant_gen",
                s.server_constant_gen);
+    MLPERF_LOG(detail, "effective_use_grouped_qsl", s.use_grouped_qsl);
 #else
     detail("");
     detail("Effective Settings:");
@@ -531,7 +535,6 @@ void TestSettingsInternal::LogSummary(AsyncSummary &summary) const {
         "samples_per_query value");
   }
 }
-
 }  // namespace loadgen
 
 int TestSettings::FromConfig(const std::string &path, const std::string &model,

loadgen/test_settings_internal.h

@@ -22,6 +22,7 @@ limitations under the License.
 
 #include "logging.h"
 #include "test_settings.h"
+#include "query_sample_library.h"
 
 namespace mlperf {