From b0dd54387cb036da399e2896b6a8ad160369b102 Mon Sep 17 00:00:00 2001
From: kevand900 <kev9_125@hotmail.com>
Date: Thu, 31 Aug 2017 14:29:25 -0700
Subject: [PATCH] Adding a command line parameter --run-multiple which will
 automatically run the specified target 7 times, discarding the min and max
 runs (with respect to the Requests/s), and reporting the average of the
 remaining 5. This reduces errors caused by manual collection as well as
 reducing the effect of run-to-run variance. There is a 180s sleep in-between
 the runs for the system to become stable again. Each run is clearly labeled
 in the console output as well as the results for each run.

---
 base/NginxDaemon.php  |  4 ++++
 base/PerfOptions.php  |  5 ++++-
 base/PerfRunner.php   | 50 +++++++++++++++++++++++++++++++++++++++++--
 base/PerfSettings.php |  8 +++++++
 4 files changed, 64 insertions(+), 3 deletions(-)

diff --git a/base/NginxDaemon.php b/base/NginxDaemon.php
index 5980276..7591cf7 100644
--- a/base/NginxDaemon.php
+++ b/base/NginxDaemon.php
@@ -25,6 +25,10 @@ public function start(): void {
     );
   }
 
+  public function stop(): void {
+    parent::stop();
+  }
+
   public function clearAccessLog(): void {
     $log = $this->options->tempDir.'/access.log';
     invariant(
diff --git a/base/PerfOptions.php b/base/PerfOptions.php
index b8c2cb3..6880caf 100644
--- a/base/PerfOptions.php
+++ b/base/PerfOptions.php
@@ -129,7 +129,7 @@ final class PerfOptions {
 
   public ?string $remoteSiege;
   public ?string $siegeTmpDir;
-
+  public bool $runMultiple;
   public function __construct(Vector<string> $argv) {
     $def = Vector {
       'help',
@@ -192,6 +192,7 @@ public function __construct(Vector<string> $argv) {
       'server-threads:',
       'client-threads:',
       'remote-siege:',
+      'run-multiple',
     };
     $targets = $this->getTargetDefinitions()->keys();
     $def->addAll($targets);
@@ -357,6 +358,8 @@ public function __construct(Vector<string> $argv) {
     $this->srcDir = $this->getNullableString('src-dir');
 
     $this->remoteSiege = $this->getNullableString('remote-siege');
+
+    $this->runMultiple = $this->getBool('run-multiple');
   }
 
   public function validate() {
diff --git a/base/PerfRunner.php b/base/PerfRunner.php
index c037212..5f592f2 100644
--- a/base/PerfRunner.php
+++ b/base/PerfRunner.php
@@ -14,7 +14,53 @@
 final class PerfRunner {
   public static function RunWithArgv(Vector<string> $argv): PerfResult {
     $options = new PerfOptions($argv);
-    return self::RunWithOptions($options);
+    $data = self::RunWithOptions($options);
+
+    if ($options->runMultiple) {
+      print json_encode($data, JSON_PRETTY_PRINT)."\n";
+      $numRuns = PerfSettings::NumRuns();
+      $dataArr = Vector{$data};
+      for ($i = 0; $i<$numRuns-1; $i++) {
+        self::PrintProgress('Sleeping before run: ' . ($i+2) . '/' . $numRuns);
+        sleep(PerfSettings::SleepTime());
+        self::PrintProgress('Starting run: ' . ($i+2) . '/' . $numRuns);
+        $data = self::RunWithOptions($options);
+        $dataArr->add($data);
+        self::PrintProgress('Results for run: ' . ($i+2) . '/' . $numRuns);
+        print json_encode($data, JSON_PRETTY_PRINT)."\n";
+      }
+      $data = self::PostProcess($dataArr, $data);
+      self::PrintProgress('Average of ' . ($numRuns-2) . ':');
+    }
+
+    return $data;
+  }
+
+  public static function PerfSort($a, $b) {
+    if ($a['Combined']['Siege RPS'] == $b['Combined']['Siege RPS']) return 0;
+    return ($a['Combined']['Siege RPS'] < $b['Combined']['Siege RPS']) ? -1 : 1;
+  }
+
+  public static function PostProcess(Vector<PerfResult> $dataArr, PerfResult $data): PerfResult {
+    usort($dataArr, "self::PerfSort");
+    $dataArr->removeKey(0);
+    $dataArr->pop();
+    $keys = $dataArr->firstValue();
+
+    if ($keys) {
+      $keys = $keys->at('Combined')->toKeysArray();
+
+      for ($i=0; $i<count($keys)-1; $i++) {
+        $avg = 0;
+
+        for ($j=0; $j<$dataArr->count(); $j++) {
+          $avg += $dataArr->at($j)['Combined']->at($keys[$i]);
+        }
+        $avg = ($avg / $dataArr->count());
+        $data['Combined'][$keys[$i]] = $avg;
+      }
+    }
+    return $data;
   }
 
   public static function RunWithOptions(PerfOptions $options): PerfResult {
@@ -236,7 +282,7 @@ private static function RunWithOptionsAndEngine(
     } else {
       self::PrintProgress('There is no tearDownTest');
     }
-
+    $nginx->stop();
     return $combined_stats;
   }
 
diff --git a/base/PerfSettings.php b/base/PerfSettings.php
index b222c36..cb7829d 100644
--- a/base/PerfSettings.php
+++ b/base/PerfSettings.php
@@ -45,4 +45,12 @@ public static function BackendPort(): int {
   public static function BackendAdminPort(): int {
     return 8093;
   }
+
+  public static function SleepTime(): int {
+    return 180;
+  }
+
+  public static function NumRuns(): int {
+    return 7;
+  }
 }