add tests for pipeline element, permutation test, hyperparameters and…

… result handler

photonai/base/photon_elements.py

@@ -350,7 +350,9 @@ def fit(self, X, y=None, **kwargs):
 
     def __batch_predict(self, delegate, X, **kwargs):
         if not isinstance(X, list) and not isinstance(X, np.ndarray):
-            logger.warning("Cannot do batching on a single entity.")
+            warning = "Cannot do batching on a single entity."
+            logger.warning(warning)
+            raise Warning(warning)
             return delegate(X, **kwargs)
 
             # initialize return values
@@ -374,9 +376,6 @@ def __batch_predict(self, delegate, X, **kwargs):
     def __predict(self, X, **kwargs):
         if not self.disabled:
             if hasattr(self.base_element, 'predict'):
-                # Todo: check if element has kwargs, and give it to them
-                # todo: so this todo above was old, here are my changes:
-                #return self.base_element.predict(X)
                 return self.adjusted_predict_call(self.base_element.predict, X, **kwargs)
             else:
                 logger.error('BaseException. base Element should have function ' +
@@ -412,7 +411,6 @@ def __predict_proba(self, X, **kwargs):
                 #return self.base_element.predict_proba(X)
                 return self.adjusted_predict_call(self.base_element.predict_proba, X, **kwargs)
             else:
-
                 # todo: in case _final_estimator is a Branch, we do not know beforehand it the base elements will
                 #  have a predict_proba -> if not, just return None (@Ramona, does this make sense?)
                 # logger.error('BaseException. base Element should have "predict_proba" function.')
@@ -452,7 +450,9 @@ def inverse_transform(self, X, y=None, **kwargs):
 
     def __batch_transform(self, X, y=None, **kwargs):
         if not isinstance(X, list) and not isinstance(X, np.ndarray):
-            logger.warning("Cannot do batching on a single entity.")
+            warning = "Cannot do batching on a single entity."
+            logger.warning(warning)
+            raise Warning(warning)
             return self.__transform(X, y, **kwargs)
 
             # initialize return values

photonai/helper/helper.py

@@ -36,8 +36,6 @@ class PhotonPrintHelper:
 
     @staticmethod
     def config_to_human_readable_dict(pipe, specific_config):
-        """
-        """
         prettified_config = {}
         for el_key, el_value in specific_config.items():
             items = el_key.split('__')

photonai/optimization/hyperparameters.py

@@ -20,9 +20,9 @@ def get_random_value(self, definite_list: bool = True):
         if definite_list:
             return random.choice(self.values)
         else:
-            msg = "The PHOTON hyperparam has no own random function."
+            msg = "The PHOTONAI hyperparam has no own random function."
             logger.error(msg)
-            raise ValueError(msg)
+            raise NotImplementedError(msg)
 
     def __str__(self):
         return str(self.__class__) + str(self.values)
@@ -46,8 +46,8 @@ def __init__(self, values: list):
     def __getitem__(self, item):
         return self.values.__getitem__(item)
 
-    def index(self, obj):
-        return self.values.index(obj)
+    def __index__(self, obj):
+        return self.values.__index__(obj)
 
 
 class BooleanSwitch(PhotonHyperparam):
@@ -164,14 +164,13 @@ def transform(self):
                 values = np.geomspace(self.start, self.stop, dtype=self.num_type, **self.range_params)
 
         # convert to python datatype because mongodb needs it
-        if np.issubdtype(self.num_type, np.integer):
-            self.values = [int(i) for i in values]
-        elif np.issubdtype(self.num_type, np.floating):
-            self.values = [float(i) for i in values]
-        else:
+        try:
+            self.values = [values[i].item() for i in range(len(values))]
+        except:
             msg = "PHOTON can not guarantee full mongodb support since you chose a non [np.integer, np.floating] " \
                   "subtype in NumberType.dtype."
             logger.warn(msg)
+            raise Warning(msg)
             self.values = values
 
     @property

photonai/processing/permutation_test.py

@@ -88,14 +88,13 @@ def fit(self, X, y, **kwargs):
                     self.pipe.results.permutation_failed = str(e)
                     logger.error(e)
                     PermutationTest.clear_data_and_save(self.pipe)
+                raise e
 
         # check for sanity
         if not self.__validate_usability(existing_reference):
             raise RuntimeError("Permutation Test is not adviced because results are not better than dummy. Aborting.")
 
         # find how many permutations have been computed already:
-        # existing_permutations = MDBHyperpipe.objects.raw({'permutation_id': self.permutation_id,
-        #                                                   'computation_completed': True}).count()
         existing_permutations = list(MDBHyperpipe.objects.raw({'permutation_id': self.permutation_id,
                                                                'computation_completed': True}).only('name'))
         existing_permutations = [int(perm_run.name.split('_')[-1]) for perm_run in existing_permutations]

photonai/processing/results_handler.py

@@ -40,7 +40,9 @@ def load_from_mongodb(self, mongodb_connect_url: str, pipe_name: str):
             self.results = results[0]
         elif len(results) > 1:
             self.results = MDBHyperpipe.objects.order_by([("computation_start_time", DESCENDING)]).raw({'name': pipe_name}).first()
-            logger.warning('Found multiple hyperpipes with that name. Returning most recent one.')
+            warn_text = 'Found multiple hyperpipes with that name. Returning most recent one.'
+            logger.warning(warn_text)
+            raise Warning(warn_text)
         else:
             raise FileNotFoundError('Could not load hyperpipe from MongoDB.')
 

test/base_tests/test_hyperpipe.py

@@ -32,7 +32,8 @@ def setup_hyperpipe(self, output_settings=None):
         self.hyperpipe = Hyperpipe('god', inner_cv=self.inner_cv_object,
                                    metrics=self.metrics,
                                    best_config_metric=self.best_config_metric,
-                                   output_settings=output_settings)
+                                   output_settings=output_settings,
+                                   verbosity=2)
         self.hyperpipe += self.ss_pipe_element
         self.hyperpipe += self.pca_pipe_element
         self.hyperpipe.add(self.svc_pipe_element)

test/base_tests/test_photon_batch.py

@@ -16,22 +16,27 @@ def fit(self, X, y, **kwargs):
         pass
 
     def transform(self, X, y=None, **kwargs):
-        X_new = []
-        for i, x in enumerate(X):
-            X_new.append([str(sub_x) + str(y[i]) for sub_x in x])
+        if hasattr(X, '__iter__'):
+            X_new = []
+            for i, x in enumerate(X):
+                X_new.append([str(sub_x) + str(y[i]) for sub_x in x])
 
-        if not np.array_equal(y, kwargs["animals"]):
-            raise Exception("Batching y and kwargs delivery is strange")
+            if not np.array_equal(y, kwargs["animals"]):
+                raise Exception("Batching y and kwargs delivery is strange")
 
-        kwargs["animals"] = [i[::-1] for i in kwargs["animals"]]
+            kwargs["animals"] = [i[::-1] for i in kwargs["animals"]]
 
-        return np.asarray(X_new), np.asarray(y), kwargs
+            return np.asarray(X_new), np.asarray(y), kwargs
+        else:
+            return X, y, kwargs
 
     def predict(self, X, y=None, **kwargs):
-
-        self.predict_count += 1
-        predictions = np.ones((X.shape[0],)) * self.predict_count
-        return predictions
+        if hasattr(X, '__iter__'):
+            self.predict_count += 1
+            predictions = np.ones((X.shape[0],)) * self.predict_count
+            return predictions
+        else:
+            return y
 
 
 class BatchingTests(unittest.TestCase):
@@ -68,8 +73,13 @@ def test_transform(self):
         self.assertEqual(kwargs_new["animals"][0], "effa")
         self.assertEqual(kwargs_new["animals"][49], "ewöl")
 
+        with self.assertRaises(Warning):
+            self.neuro_batch.transform('str', [0])
+
     def test_predict(self):
         y_predicted = self.neuro_batch.predict(self.data, **self.kwargs)
         # assure that predict is batch wisely called
         self.assertEqual(y_predicted[0], 1)
         self.assertEqual(y_predicted[-1], (self.data.shape[0]/self.batch_size))
+        with self.assertRaises(Warning):
+            self.neuro_batch.predict('str')

test/base_tests/test_photon_elements.py

@@ -43,7 +43,15 @@ def test_pipeline_element_create(self):
         # correct name
         self.assertEqual(self.pca_pipe_element.name, 'PCA')
 
-    def test_fit(self):
+    def test_create_from_object(self):
+        pca = PipelineElement.create('obj_name', PCA(), hyperparameters={'n_components': [1, 4]})
+        self.assertIsInstance(pca.base_element, PCA)
+        self.assertEqual(pca.name, 'obj_name')
+
+        with self.assertRaises(ValueError):
+            pca_2 = PipelineElement.create('obj_name', PCA, hyperparameters={'n_components': [1, 4]})
+
+    def test_fit_and_score(self):
         tmp_pca = PCA().fit(self.X, self.y)
         self.pca_pipe_element.fit(self.X, self.y)
         self.assertEqual(self.pca_pipe_element.base_element.components_.shape, (30, 30))
@@ -53,6 +61,10 @@ def test_fit(self):
         self.svc_pipe_element.fit(self.X, self.y)
         self.assertAlmostEqual(self.svc_pipe_element.base_element._intercept_[0], tmp_svc._intercept_[0])
 
+        sk_score = tmp_svc.score(self.X, self.y)
+        p_score = self.svc_pipe_element.score(self.X, self.y)
+        self.assertTrue(np.array_equal(sk_score, p_score))
+
     def test_transform(self):
         self.pca_pipe_element.fit(self.X, self.y)
 
@@ -92,6 +104,10 @@ def test_one_hyperparameter_setup(self):
                                                                              'PCA__disabled': False},
                                                                             {'PCA__disabled': True}])
 
+    def test_hyperprameter_sanity_check(self):
+        with self.assertRaises(ValueError):
+            error_element = PipelineElement('PCA', hyperparameters={'kernel': ['rbf', 'linear']})
+
     def test_more_hyperparameters_setup(self):
         # sklearn attributes are generated
         self.assertDictEqual(self.svc_pipe_element.hyperparameters, {'SVC__C': [0.1, 1],
@@ -117,6 +133,13 @@ def test_set_params(self):
         with self.assertRaises(ValueError):
             self.pca_pipe_element.set_params(**{'any_weird_param': 1})
 
+    def test_get_params(self):
+        photon_params = PipelineElement('PCA').get_params()
+        del photon_params["name"]
+        sk_params = PCA().get_params()
+        self.assertDictEqual(photon_params, sk_params)
+        self.assertIsNone(PipelineElement.create('any_not_existing_object', object(), hyperparameters={}).get_params())
+
     def test_set_random_state(self):
         # we handle all elements in one method that is inherited so we capture them all in this test
         random_state = 53
@@ -200,6 +223,11 @@ def test_predict_when_no_transform(self):
         self.assertTrue(np.array_equal(kwargs['covariates'], self.kwargs['covariates']))
         self.assertEqual(y, None)
 
+    def test_predict_on_transformer(self):
+        est = PipelineElement.create('Estimator', base_element=DummyTransformer(), hyperparameters={})
+        with self.assertRaises(BaseException):
+            est.predict(self.X)
+
     def test_copy_me(self):
         svc = PipelineElement('SVC', {'C': [0.1, 1], 'kernel': ['rbf', 'sigmoid']})
         svc.set_params(**{'C': 0.1, 'kernel': 'sigmoid'})
@@ -868,6 +896,20 @@ def test_use_probabilities(self):
         self.assertEqual(probas.shape[1], 3)
 
 
+class PreprocessingTests(unittest.TestCase):
+
+    def test_hyperparameter_add(self):
+        pe = Preprocessing()
+        with self.assertRaises(ValueError):
+            pe.add(PipelineElement('PCA', hyperparameters={'n_components': [1, 5]}))
+
+    def test_predict_warning(self):
+        pe = Preprocessing()
+        pe.add(PipelineElement('SVC'))
+        with self.assertRaises(Warning):
+            pe.predict([0, 1, 2])
+
+
 class DataFilterTests(unittest.TestCase):
 
     def setUp(self):
@@ -944,4 +986,4 @@ def test_callback(self):
             pipeline.fit(self.X, self.y).predict(self.X)
 
         with self.assertRaises(Warning):
-            self.callback_branch_pipeline_error.fit(self.X, self.y).predict(self.X)
+            self.callback_branch_pipeline_error.fit(self.X, self.y).predict(self.X)

test/optimization_tests/test_hyperparameters.py

@@ -4,15 +4,16 @@
 from photonai.optimization import FloatRange, IntegerRange, Categorical, BooleanSwitch
 
 
-class BaseTest(unittest.TestCase):
+class HyperparameterBaseTest(unittest.TestCase):
 
     def setUp(self):
         """
         Set default start setting for all tests.
         """
         self.intger_range = IntegerRange(2,6)
         self.float_range = FloatRange(0.1, 5.7)
-        self.categorical = Categorical(["a","b","c","d","e","f","g","h"])
+        self.cateogrical_truth = ["a","b","c","d","e","f","g","h"]
+        self.categorical = Categorical(self.cateogrical_truth)
         self.bool = BooleanSwitch()
 
     def test_rand_success(self):
@@ -23,7 +24,7 @@ def test_rand_success(self):
             self.assertGreaterEqual(self.float_range.get_random_value(), 0.1)
             self.assertLess(self.float_range.get_random_value(), 5.7)
 
-            self.assertIn(self.categorical.get_random_value(), ["a","b","c","d","e","f","g","h"])
+            self.assertIn(self.categorical.get_random_value(), self.cateogrical_truth)
 
             self.assertIn(self.bool.get_random_value(), [True, False])
 
@@ -47,13 +48,18 @@ def test_domain(self):
         self.assertListEqual(self.categorical.values, ["a","b","c","d","e","f","g","h"])
         self.assertListEqual(self.bool.values, [True, False])
 
-
     def test_rand_error(self):
         with self.assertRaises(ValueError):
             self.intger_range.get_random_value(definite_list=True)
+        with self.assertRaises(ValueError):
             self.float_range.get_random_value(definite_list=True)
-            self.bool.get_random_value(definite_list=True)
-            self.categorical.get_random_value(definite_list=True)
+        with self.assertRaises(NotImplementedError):
+            self.categorical.get_random_value(definite_list=False)
+        with self.assertRaises(NotImplementedError):
+            self.categorical.get_random_value(definite_list=False)
+
+    def test_categorical(self):
+        self.assertEqual(self.categorical[2], self.cateogrical_truth[2])
 
 
 class NumberRangeTest(unittest.TestCase):