Zero-loss throughput and expdesign selection

--exp-design parameters enable to set the way to explore the
experimental design.

By default it is full-factorial (--exp-design full)

--exp-design zlt(RATE,PPS) enables a zero-loss-throughput search

Instead of going through all possible rates, it will look at the result
of PPS and do some heuristics + binary search to find the maximal
zero-loss throughput.

integration/zlt.npf

@@ -0,0 +1,21 @@
+%config
+graph_filter_by={THROUGHPUT:DROPPEDPC>10}
+var_format={THROUGHPUT:%d}
+var_names={RATE:Input rate (Gbps),THROUGHPUT:Throughput (Gbps)}
+graph_legend=0
+var_lim={result:0-100}
+
+%variables
+RATE=[10-100#5]
+THRESH={50,90}
+
+%script
+if [ $RATE -lt $THRESH ] ; then
+    d=0
+else
+    d=$(echo "($RATE - $THRESH) / 2" | bc)
+fi
+
+t=$(echo "$RATE*(100-$d)/100" | bc)
+echo "RESULT-DROPPEDPC $d"
+echo "RESULT-THROUGHPUT $t"

npf/expdesign/fullexp.py

@@ -0,0 +1,35 @@
+from npf.variable import OrderedDict
+
+
+from collections import OrderedDict
+
+
+class FullVariableExpander:
+    """Expand all variables building the full
+    matrix first."""
+
+    def __init__(self, vlist, overriden):
+        self.expanded = [OrderedDict()]
+        for k, v in vlist.items():
+            if k in overriden:
+                continue
+            newList = []
+            l = v.makeValues()
+
+            for nvalue in l:
+                for ovalue in self.expanded:
+                    z = ovalue.copy()
+                    z.update(nvalue if type(nvalue) is OrderedDict else {k: nvalue})
+                    newList.append(z)
+
+            self.expanded = newList
+        self.it = self.expanded.__iter__()
+
+    def __iter__(self):
+        return self.expanded.__iter__()
+
+    def __next__(self):
+        return self.it.__next__()
+
+    def __len__(self):
+        return len(self.expanded)

npf/expdesign/optimexp.py

@@ -0,0 +1,39 @@
+from npf.variable import OrderedDict
+
+
+from collections import OrderedDict
+
+from skopt import Optimizer
+from skopt.space import Real
+from joblib import Parallel, delayed
+# example objective taken from skopt
+from skopt.benchmarks import branin
+
+class OptimizeVariableExpander:
+    """Use scipy optimize function"""
+
+    def __init__(self, vlist, overriden):
+        dimensions = []
+        for k, v in vlist.items():
+            if k in overriden:
+                continue
+
+            l = v.makeValues()
+            dimensions.append(l)
+
+        self.optimizer = Optimizer(
+            dimensions=dimensions,
+            random_state=1,
+            base_estimator='gp'
+        ) 
+
+
+    def __iter__(self):
+        x = self.optimizer.ask(n_points=1)  # x is a list of n_points points
+        return x
+
+    def tell(x, y):
+        self.optimizer.tell(x, y)
+
+    def __next__(self):
+        return self.it.__next__()

npf/expdesign/randomexp.py

@@ -0,0 +1,11 @@
+from random import shuffle
+
+from npf.expdesign.fullexp import FullVariableExpander
+
+
+class RandomVariableExpander(FullVariableExpander):
+    """Same as BruteVariableExpander but shuffle the series to test"""
+
+    def __init__(self, vlist, overriden):
+        super().__init__(vlist, overriden)
+        shuffle(self.expanded)

npf/expdesign/zltexp.py

@@ -0,0 +1,101 @@
+from collections import OrderedDict
+from typing import Dict
+
+import numpy as np
+from npf.expdesign.fullexp import FullVariableExpander
+from npf.types.dataset import Run
+from npf.variable import Variable
+
+
+class ZLTVariableExpander(FullVariableExpander):
+
+    def __init__(self, vlist:Dict[str,Variable], results, overriden, input, output):
+
+
+        if not input in vlist:
+            raise Exception(f"{input} is not in the variables, please define a variable in the %variable section.")
+        self.results = results
+        self.input = input
+        self.input_values = vlist[input].makeValues()
+        del vlist[input]
+        self.current = None
+        self.output = output
+        self.passed = 0
+        super().__init__(vlist, overriden)
+
+    def __iter__(self):
+        self.it = self.expanded.__iter__()
+        self.passed = 0
+        return self
+
+    def __len__(self):
+        return len(self.expanded) * len(self.input_values) - self.passed
+
+    def __next__(self):
+        margin=1.01
+        if self.current == None:
+            self.current = self.it.__next__()
+
+        # get all outputs for all inputs
+        vals_for_current = {}
+        acceptable_rates = []
+        max_r = max(self.input_values)
+        for r, vals in self.results.items():
+            if Run(self.current).inside(r):
+                try:
+                    if self.output:
+                        r_out = np.mean(vals[self.output])
+                        r_in = r.variables[self.input]
+                        vals_for_current[r_in] = r_out
+                        if r_out >= r_in/margin:
+                            acceptable_rates.append(r_in)
+                        else:
+                            max_r = min(max_r, r_out)
+                except KeyError:
+                    raise Exception(f"{self.output} is not in the results. Sample of last result : {vals}")
+
+        #Step 1 : try the max output
+        if len(vals_for_current) == 0:
+            next_val = max_r
+        elif len(vals_for_current) == 1:
+            #If we're lucky, the max rate is doable
+
+            if len(acceptable_rates) == 1:
+                    self.current = None
+                    self.passed += len(self.input_values) - 1
+                    return self.__next__()
+
+            #Step 2 : go for the rate below the max output
+            maybe_achievable_inputs = list(filter(lambda x : x <= max_r, self.input_values))
+            next_val = max(maybe_achievable_inputs)
+        else:
+
+            maybe_achievable_inputs = list(filter(lambda x : x <= max_r*margin, self.input_values))
+            left_to_try = set(maybe_achievable_inputs).difference(vals_for_current.keys())
+
+            #Step 3...K : try to get an acceptable rate. This step might be skiped if we got an acceptable rate already
+            if len(acceptable_rates) == 0:
+                #Try the rate below the min already tried rate - its drop count. For instance if we tried 70 last run but got 67 of throughput, try the rate below 64
+                min_input = min(vals_for_current.keys())
+                min_output = vals_for_current[min_input]
+                target = min_output - (min_input - min_output)
+                next_val = max(filter(lambda x : x < target,left_to_try))
+            else:
+                #Step K... n : we do a binary search between the maximum acceptable rate and the minimal rate observed
+                max_acceptable = max(acceptable_rates)
+                #Consider we tried 100->95 (max_r=95), 90->90 (acceptable) we have to try values between 90..95
+                left_to_try_over_acceptable = list(filter(lambda x: x > max_acceptable, left_to_try))
+                if len(left_to_try_over_acceptable) == 0:
+                            #Found!
+                            self.current = None
+                            self.passed += len(self.input_values) - len(vals_for_current)
+                            return self.__next__()
+                #Binary search
+                next_val = left_to_try_over_acceptable[int(len(left_to_try_over_acceptable) / 2)]
+
+
+        copy = self.current.copy()
+        copy.update({self.input : next_val})
+        return copy
+
+

npf/npf.py

@@ -172,9 +172,9 @@ def add_testing_options(parser: ArgumentParser, regression: bool = False):
     t.add_argument('--no-mp', dest='allow_mp', action='store_false',
                    default=True, help='Run tests in the same thread. If there is multiple script, they will run '
                                       'one after the other, hence breaking most of the tests.')
-    t.add_argument('--expand', type=str, default=None, dest="expand")
-    t.add_argument('--rand-env', type=int, default=65536, dest="rand_env")
-    t.add_argument('--experimental-design', type=str, default="matrix.csv", help="The path towards the experimental design point selection file")
+    t.add_argument('--exp-design', type=str, default="full", dest="design", help="Experimental design method")
+    t.add_argument('--spacefill', type=str, default="matrix.csv", dest="spacefill", help="The path towards the space filling values matrix")
+    t.add_argument('--rand-env', type=int, default=65536, dest="rand_env", help="Add an environmental variable of a random size to prevent bias")
 
     c = parser.add_argument_group('Cluster options')
     c.add_argument('--cluster', metavar='role=user@address:path [...]', type=str, nargs='*', default=[],