diff --git a/.gitignore b/.gitignore index ec537a3d1..df376e412 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ target_algo_runs.json doc/main_options.rst doc/scenario_options.rst doc/smac_options.rst +doc/apidoc diff --git a/.travis.yml b/.travis.yml index b8c14d4eb..62f17fba7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,7 +60,7 @@ before_install: install: - pip install pep8 codecov mypy flake8 - cat requirements.txt | xargs -n 1 -L 1 pip install - - python setup.py install + - pip install .[all] script: - ci_scripts/$TESTSUITE diff --git a/MANIFEST.in b/MANIFEST.in index 540b72040..cde3d6032 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ -include requirements.txt \ No newline at end of file +include requirements.txt +include extras_require.json diff --git a/README.md b/README.md index fd45d33fb..59b3b3295 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ used in SMAC3 requires SWIG (>= 3.0). ## Installation via pip -SMAC3 is available on pipy. +SMAC3 is available on PyPI. ```pip install smac``` @@ -63,7 +63,7 @@ SMAC3 is available on pipy. ``` git clone https://github.com/automl/SMAC3.git && cd SMAC3 cat requirements.txt | xargs -n 1 -L 1 pip install -python setup.py install +pip install . ``` ## Installation in Anaconda @@ -73,6 +73,29 @@ packages **before** you can install SMAC: ```conda install gxx_linux-64 gcc_linux-64 swig``` +## Optional dependencies + +SMAC3 comes with a set of optional dependencies that can be installed using +[setuptools extras](https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-extras-optional-features-with-their-own-dependencies): + +- `lhd`: Latin hypercube design +- `gp`: Gaussian process models + +These can be installed from PyPI or manually: + +``` +# from PyPI +pip install smac[gp] + +# manually +pip install .[gp,lhd] +``` + +For convenience there is also an `all` meta-dependency that installs all optional dependencies: +``` +pip install smac[all] +``` + # License This program is free software: you can redistribute it and/or modify diff --git a/changelog.md b/changelog.md index 589886071..476c8f1b7 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,30 @@ +# 0.11.0 + +## Major changes + +* Local search now starts from observed configurations with high acquisition function values, low cost and the from + unobserved configurations with high acquisition function values found by random search (#509) +* Reduces the number of mandatory requirements (#516) +* Make Gaussian processes more resilient to linalg error by more aggressively adding noise to the diagonal (#511) +* Inactive hyperparameters are now imputed with a value outside of the modeled range (-1) (#508) +* Replace the GP library George by scikit-learn (#505) +* Renames facades to better reflect their use cases (#492), and adds a table to help deciding which facade to use (#495) +* SMAC facades now accept class arguments instead of object arguments (#486) + +## Minor changes + +* Vectorize local search for improved speed (#500) +* Extend the Sobol and LHD initial design to work for non-continuous hyperparameters as well applying an idea similar + to inverse transform sampling (#494) + + +## Bug fixes + +* Fixes a regression in the validation scripts (#519) +* Fixes a unit test regression with numpy 1.17 (#523) +* Fixes an error message (#510) +* Fixes an error making random search behave identical for all seeds + # 0.10.0 ## Major changes diff --git a/ci_scripts/circle_install.sh b/ci_scripts/circle_install.sh index a54ddd922..66e6974d0 100644 --- a/ci_scripts/circle_install.sh +++ b/ci_scripts/circle_install.sh @@ -8,12 +8,9 @@ source activate testenv # install documentation building dependencies pip install --upgrade numpy -pip install --upgrade matplotlib setuptools nose coverage sphinx pillow sphinx-gallery sphinx_bootstrap_theme cython numpydoc -# And finally, all other dependencies -cat requirements.txt | xargs -n 1 -L 1 pip install +pip install --upgrade matplotlib setuptools nose coverage sphinx pillow sphinx-gallery sphinx_bootstrap_theme numpydoc -python setup.py clean -python setup.py develop +pip install -e .[all] # pipefail is necessary to propagate exit codes set -o pipefail && cd doc && make html 2>&1 | tee ~/log.txt \ No newline at end of file diff --git a/ci_scripts/create_doc.sh b/ci_scripts/create_doc.sh index 2b295ec49..cef3a96e8 100644 --- a/ci_scripts/create_doc.sh +++ b/ci_scripts/create_doc.sh @@ -9,7 +9,7 @@ if ! [[ -z ${DOCPUSH+x} ]]; then if [[ "$DOCPUSH" == "true" ]]; then # install documentation building dependencies - pip install --upgrade matplotlib seaborn setuptools nose coverage sphinx pillow sphinx-gallery sphinx_bootstrap_theme cython numpydoc nbformat nbconvert mock + pip install --upgrade sphinx_rtd_theme # $1 is the branch name # $2 is the global variable where we set the script status @@ -20,7 +20,7 @@ if ! [[ -z ${DOCPUSH+x} ]]; then fi # create the documentation - cd doc && make html 2>&1 + cd doc && make buildapi && make html 2>&1 # create directory with branch name # the documentation for dev/stable from git will be stored here diff --git a/ci_scripts/run_examples.sh b/ci_scripts/run_examples.sh index e17299f9f..8ec7ca879 100755 --- a/ci_scripts/run_examples.sh +++ b/ci_scripts/run_examples.sh @@ -1,6 +1,6 @@ cd examples -for script in rf.py rosenbrock.py svm.py +for script in fmin_rosenbrock.py SMAC4BO_rosenbrock.py SMAC4HPO_acq_rosenbrock.py SMAC4HPO_rf.py SMAC4HPO_rosenbrock.py SMAC4HPO_svm.py do python $script rval=$? @@ -17,3 +17,19 @@ if [ "$rval" != 0 ]; then echo "Error running example QCP" exit $rval fi +cd .. + +cd branin +python branin_fmin.py +rval=$? +if [ "$rval" != 0 ]; then + echo "Error running example QCP" + exit $rval +fi + +python ../../scripts/smac --scenario scenario.txt +rval=$? +if [ "$rval" != 0 ]; then + echo "Error running example QCP" + exit $rval +fi diff --git a/ci_scripts/run_unittests.sh b/ci_scripts/run_unittests.sh index 651c32231..d2644903c 100755 --- a/ci_scripts/run_unittests.sh +++ b/ci_scripts/run_unittests.sh @@ -1,2 +1,4 @@ echo Testing revision $(git rev-parse HEAD) ... -make test && bash scripts/test_no_files_left.sh \ No newline at end of file +echo Testing from directory `pwd` +conda list +make test && bash scripts/test_no_files_left.sh diff --git a/doc/apidoc/smac.configspace.rst b/doc/apidoc/smac.configspace.rst deleted file mode 100644 index 6047c0f7f..000000000 --- a/doc/apidoc/smac.configspace.rst +++ /dev/null @@ -1,14 +0,0 @@ -smac\.configspace package -========================= - -.. automodule:: smac.configspace - :members: - :undoc-members: - :show-inheritance: - :inherited-members: - - -.. toctree:: - - smac.configspace.util - diff --git a/doc/apidoc/smac.configspace.util.rst b/doc/apidoc/smac.configspace.util.rst deleted file mode 100644 index 940fa4a91..000000000 --- a/doc/apidoc/smac.configspace.util.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.configspace\.util module -============================== - -.. automodule:: smac.configspace.util - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.epm.base_epm.rst b/doc/apidoc/smac.epm.base_epm.rst deleted file mode 100644 index 36d80c524..000000000 --- a/doc/apidoc/smac.epm.base_epm.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.epm\.base\_epm module -=========================== - -.. automodule:: smac.epm.base_epm - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.epm.base_imputor.rst b/doc/apidoc/smac.epm.base_imputor.rst deleted file mode 100644 index 0f644929b..000000000 --- a/doc/apidoc/smac.epm.base_imputor.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.epm\.base\_imputor module -=============================== - -.. automodule:: smac.epm.base_imputor - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.epm.random_epm.rst b/doc/apidoc/smac.epm.random_epm.rst deleted file mode 100644 index 223365d6c..000000000 --- a/doc/apidoc/smac.epm.random_epm.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.epm\.random\_epm module -============================= - -.. automodule:: smac.epm.random_epm - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.epm.rf_with_instances.rst b/doc/apidoc/smac.epm.rf_with_instances.rst deleted file mode 100644 index 3f52e3841..000000000 --- a/doc/apidoc/smac.epm.rf_with_instances.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.epm\.rf\_with\_instances module -===================================== - -.. automodule:: smac.epm.rf_with_instances - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.epm.rfr_imputator.rst b/doc/apidoc/smac.epm.rfr_imputator.rst deleted file mode 100644 index cd93f76b9..000000000 --- a/doc/apidoc/smac.epm.rfr_imputator.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.epm\.rfr\_imputator module -================================ - -.. automodule:: smac.epm.rfr_imputator - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.epm.rst b/doc/apidoc/smac.epm.rst deleted file mode 100644 index f9b284755..000000000 --- a/doc/apidoc/smac.epm.rst +++ /dev/null @@ -1,19 +0,0 @@ -smac\.epm package -================= - -.. automodule:: smac.epm - :members: - :undoc-members: - :show-inheritance: - :inherited-members: - - -.. toctree:: - - smac.epm.base_epm - smac.epm.base_imputor - smac.epm.random_epm - smac.epm.rf_with_instances - smac.epm.rfr_imputator - smac.epm.uncorrelated_mo_rf_with_instances - diff --git a/doc/apidoc/smac.epm.uncorrelated_mo_rf_with_instances.rst b/doc/apidoc/smac.epm.uncorrelated_mo_rf_with_instances.rst deleted file mode 100644 index 95c62bead..000000000 --- a/doc/apidoc/smac.epm.uncorrelated_mo_rf_with_instances.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.epm\.uncorrelated\_mo\_rf\_with\_instances module -======================================================= - -.. automodule:: smac.epm.uncorrelated_mo_rf_with_instances - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.facade.epils_facade.rst b/doc/apidoc/smac.facade.epils_facade.rst deleted file mode 100644 index 6757e278f..000000000 --- a/doc/apidoc/smac.facade.epils_facade.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.facade\.epils\_facade module -================================== - -.. automodule:: smac.facade.epils_facade - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.facade.func_facade.rst b/doc/apidoc/smac.facade.func_facade.rst deleted file mode 100644 index 76ec25583..000000000 --- a/doc/apidoc/smac.facade.func_facade.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.facade\.func\_facade module -================================= - -.. automodule:: smac.facade.func_facade - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.facade.roar_facade.rst b/doc/apidoc/smac.facade.roar_facade.rst deleted file mode 100644 index 31825c42a..000000000 --- a/doc/apidoc/smac.facade.roar_facade.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.facade\.roar\_facade module -================================= - -.. automodule:: smac.facade.roar_facade - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.facade.rst b/doc/apidoc/smac.facade.rst deleted file mode 100644 index e854694b6..000000000 --- a/doc/apidoc/smac.facade.rst +++ /dev/null @@ -1,17 +0,0 @@ -smac\.facade package -==================== - -.. automodule:: smac.facade - :members: - :undoc-members: - :show-inheritance: - :inherited-members: - - -.. toctree:: - - smac.facade.epils_facade - smac.facade.func_facade - smac.facade.roar_facade - smac.facade.smac_facade - diff --git a/doc/apidoc/smac.facade.smac_facade.rst b/doc/apidoc/smac.facade.smac_facade.rst deleted file mode 100644 index b5cb54076..000000000 --- a/doc/apidoc/smac.facade.smac_facade.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.facade\.smac\_facade module -================================= - -.. automodule:: smac.facade.smac_facade - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.initial_design.default_configuration_design.rst b/doc/apidoc/smac.initial_design.default_configuration_design.rst deleted file mode 100644 index c59e783c5..000000000 --- a/doc/apidoc/smac.initial_design.default_configuration_design.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.initial\_design\.default\_configuration\_design module -============================================================ - -.. automodule:: smac.initial_design.default_configuration_design - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.initial_design.initial_design.rst b/doc/apidoc/smac.initial_design.initial_design.rst deleted file mode 100644 index cb609d8a6..000000000 --- a/doc/apidoc/smac.initial_design.initial_design.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.initial\_design\.initial\_design module -============================================= - -.. automodule:: smac.initial_design.initial_design - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.initial_design.multi_config_initial_design.rst b/doc/apidoc/smac.initial_design.multi_config_initial_design.rst deleted file mode 100644 index 1fa5d6de4..000000000 --- a/doc/apidoc/smac.initial_design.multi_config_initial_design.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.initial\_design\.multi\_config\_initial\_design module -============================================================ - -.. automodule:: smac.initial_design.multi_config_initial_design - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.initial_design.random_configuration_design.rst b/doc/apidoc/smac.initial_design.random_configuration_design.rst deleted file mode 100644 index bb76f9250..000000000 --- a/doc/apidoc/smac.initial_design.random_configuration_design.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.initial\_design\.random\_configuration\_design module -=========================================================== - -.. automodule:: smac.initial_design.random_configuration_design - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.initial_design.rst b/doc/apidoc/smac.initial_design.rst deleted file mode 100644 index 8cc916509..000000000 --- a/doc/apidoc/smac.initial_design.rst +++ /dev/null @@ -1,18 +0,0 @@ -smac\.initial\_design package -============================= - -.. automodule:: smac.initial_design - :members: - :undoc-members: - :show-inheritance: - :inherited-members: - - -.. toctree:: - - smac.initial_design.default_configuration_design - smac.initial_design.initial_design - smac.initial_design.multi_config_initial_design - smac.initial_design.random_configuration_design - smac.initial_design.single_config_initial_design - diff --git a/doc/apidoc/smac.initial_design.single_config_initial_design.rst b/doc/apidoc/smac.initial_design.single_config_initial_design.rst deleted file mode 100644 index 2ffd5a5ad..000000000 --- a/doc/apidoc/smac.initial_design.single_config_initial_design.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.initial\_design\.single\_config\_initial\_design module -============================================================= - -.. automodule:: smac.initial_design.single_config_initial_design - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.intensification.intensification.rst b/doc/apidoc/smac.intensification.intensification.rst deleted file mode 100644 index b153b72d2..000000000 --- a/doc/apidoc/smac.intensification.intensification.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.intensification\.intensification module -============================================= - -.. automodule:: smac.intensification.intensification - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.intensification.rst b/doc/apidoc/smac.intensification.rst deleted file mode 100644 index 26efddcf2..000000000 --- a/doc/apidoc/smac.intensification.rst +++ /dev/null @@ -1,14 +0,0 @@ -smac\.intensification package -============================= - -.. automodule:: smac.intensification - :members: - :undoc-members: - :show-inheritance: - :inherited-members: - - -.. toctree:: - - smac.intensification.intensification - diff --git a/doc/apidoc/smac.optimizer.acquisition.rst b/doc/apidoc/smac.optimizer.acquisition.rst deleted file mode 100644 index 8283acb3b..000000000 --- a/doc/apidoc/smac.optimizer.acquisition.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.optimizer\.acquisition module -=================================== - -.. automodule:: smac.optimizer.acquisition - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.optimizer.ei_optimization.rst b/doc/apidoc/smac.optimizer.ei_optimization.rst deleted file mode 100644 index 2e75a5d17..000000000 --- a/doc/apidoc/smac.optimizer.ei_optimization.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.optimizer\.ei\_optimization module -======================================== - -.. automodule:: smac.optimizer.ei_optimization - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.optimizer.epils.rst b/doc/apidoc/smac.optimizer.epils.rst deleted file mode 100644 index 3887e86aa..000000000 --- a/doc/apidoc/smac.optimizer.epils.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.optimizer\.epils module -============================= - -.. automodule:: smac.optimizer.epils - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.optimizer.objective.rst b/doc/apidoc/smac.optimizer.objective.rst deleted file mode 100644 index e1307a3ac..000000000 --- a/doc/apidoc/smac.optimizer.objective.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.optimizer\.objective module -================================= - -.. automodule:: smac.optimizer.objective - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.optimizer.pSMAC.rst b/doc/apidoc/smac.optimizer.pSMAC.rst deleted file mode 100644 index 10e5c322c..000000000 --- a/doc/apidoc/smac.optimizer.pSMAC.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.optimizer\.pSMAC module -============================= - -.. automodule:: smac.optimizer.pSMAC - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.optimizer.random_configuration_chooser.rst b/doc/apidoc/smac.optimizer.random_configuration_chooser.rst deleted file mode 100644 index cd3e10ed5..000000000 --- a/doc/apidoc/smac.optimizer.random_configuration_chooser.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.optimizer\.random\_configuration\_chooser module -====================================================== - -.. automodule:: smac.optimizer.random_configuration_chooser - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.optimizer.rst b/doc/apidoc/smac.optimizer.rst deleted file mode 100644 index ffac635bc..000000000 --- a/doc/apidoc/smac.optimizer.rst +++ /dev/null @@ -1,20 +0,0 @@ -smac\.optimizer package -======================= - -.. automodule:: smac.optimizer - :members: - :undoc-members: - :show-inheritance: - :inherited-members: - - -.. toctree:: - - smac.optimizer.acquisition - smac.optimizer.ei_optimization - smac.optimizer.epils - smac.optimizer.objective - smac.optimizer.pSMAC - smac.optimizer.random_configuration_chooser - smac.optimizer.smbo - diff --git a/doc/apidoc/smac.optimizer.smbo.rst b/doc/apidoc/smac.optimizer.smbo.rst deleted file mode 100644 index dfa5d0341..000000000 --- a/doc/apidoc/smac.optimizer.smbo.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.optimizer\.smbo module -============================ - -.. automodule:: smac.optimizer.smbo - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.runhistory.rst b/doc/apidoc/smac.runhistory.rst deleted file mode 100644 index bd0ffbec6..000000000 --- a/doc/apidoc/smac.runhistory.rst +++ /dev/null @@ -1,15 +0,0 @@ -smac\.runhistory package -======================== - -.. automodule:: smac.runhistory - :members: - :undoc-members: - :show-inheritance: - :inherited-members: - - -.. toctree:: - - smac.runhistory.runhistory - smac.runhistory.runhistory2epm - diff --git a/doc/apidoc/smac.runhistory.runhistory.rst b/doc/apidoc/smac.runhistory.runhistory.rst deleted file mode 100644 index 4aecedc75..000000000 --- a/doc/apidoc/smac.runhistory.runhistory.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.runhistory\.runhistory module -=================================== - -.. automodule:: smac.runhistory.runhistory - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.runhistory.runhistory2epm.rst b/doc/apidoc/smac.runhistory.runhistory2epm.rst deleted file mode 100644 index dc4462c3f..000000000 --- a/doc/apidoc/smac.runhistory.runhistory2epm.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.runhistory\.runhistory2epm module -======================================= - -.. automodule:: smac.runhistory.runhistory2epm - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.scenario.rst b/doc/apidoc/smac.scenario.rst deleted file mode 100644 index f28792fde..000000000 --- a/doc/apidoc/smac.scenario.rst +++ /dev/null @@ -1,14 +0,0 @@ -smac\.scenario package -====================== - -.. automodule:: smac.scenario - :members: - :undoc-members: - :show-inheritance: - :inherited-members: - - -.. toctree:: - - smac.scenario.scenario - diff --git a/doc/apidoc/smac.scenario.scenario.rst b/doc/apidoc/smac.scenario.scenario.rst deleted file mode 100644 index 2813addeb..000000000 --- a/doc/apidoc/smac.scenario.scenario.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.scenario\.scenario module -=============================== - -.. automodule:: smac.scenario.scenario - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.smac_cli.rst b/doc/apidoc/smac.smac_cli.rst deleted file mode 100644 index a21a3815c..000000000 --- a/doc/apidoc/smac.smac_cli.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.smac\_cli module -====================== - -.. automodule:: smac.smac_cli - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.stats.rst b/doc/apidoc/smac.stats.rst deleted file mode 100644 index 779b748f2..000000000 --- a/doc/apidoc/smac.stats.rst +++ /dev/null @@ -1,14 +0,0 @@ -smac\.stats package -=================== - -.. automodule:: smac.stats - :members: - :undoc-members: - :show-inheritance: - :inherited-members: - - -.. toctree:: - - smac.stats.stats - diff --git a/doc/apidoc/smac.stats.stats.rst b/doc/apidoc/smac.stats.stats.rst deleted file mode 100644 index 12a7996b6..000000000 --- a/doc/apidoc/smac.stats.stats.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.stats\.stats module -========================= - -.. automodule:: smac.stats.stats - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.tae.execute_func.rst b/doc/apidoc/smac.tae.execute_func.rst deleted file mode 100644 index 9f064a44f..000000000 --- a/doc/apidoc/smac.tae.execute_func.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.tae\.execute\_func module -=============================== - -.. automodule:: smac.tae.execute_func - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.tae.execute_ta_run.rst b/doc/apidoc/smac.tae.execute_ta_run.rst deleted file mode 100644 index 4d1ce59c1..000000000 --- a/doc/apidoc/smac.tae.execute_ta_run.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.tae\.execute\_ta\_run module -================================== - -.. automodule:: smac.tae.execute_ta_run - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.tae.execute_ta_run_aclib.rst b/doc/apidoc/smac.tae.execute_ta_run_aclib.rst deleted file mode 100644 index 6eeb327af..000000000 --- a/doc/apidoc/smac.tae.execute_ta_run_aclib.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.tae\.execute\_ta\_run\_aclib module -========================================= - -.. automodule:: smac.tae.execute_ta_run_aclib - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.tae.execute_ta_run_old.rst b/doc/apidoc/smac.tae.execute_ta_run_old.rst deleted file mode 100644 index 600c046e1..000000000 --- a/doc/apidoc/smac.tae.execute_ta_run_old.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.tae\.execute\_ta\_run\_old module -======================================= - -.. automodule:: smac.tae.execute_ta_run_old - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.tae.rst b/doc/apidoc/smac.tae.rst deleted file mode 100644 index 59f79cd0d..000000000 --- a/doc/apidoc/smac.tae.rst +++ /dev/null @@ -1,17 +0,0 @@ -smac\.tae package -================= - -.. automodule:: smac.tae - :members: - :undoc-members: - :show-inheritance: - :inherited-members: - - -.. toctree:: - - smac.tae.execute_func - smac.tae.execute_ta_run - smac.tae.execute_ta_run_aclib - smac.tae.execute_ta_run_old - diff --git a/doc/apidoc/smac.utils.constants.rst b/doc/apidoc/smac.utils.constants.rst deleted file mode 100644 index 0359bd289..000000000 --- a/doc/apidoc/smac.utils.constants.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.utils\.constants module -============================= - -.. automodule:: smac.utils.constants - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.utils.duplicate_filter_logging.rst b/doc/apidoc/smac.utils.duplicate_filter_logging.rst deleted file mode 100644 index 993a6bc65..000000000 --- a/doc/apidoc/smac.utils.duplicate_filter_logging.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.utils\.duplicate\_filter\_logging module -============================================== - -.. automodule:: smac.utils.duplicate_filter_logging - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.utils.io.cmd_reader.rst b/doc/apidoc/smac.utils.io.cmd_reader.rst deleted file mode 100644 index abf905849..000000000 --- a/doc/apidoc/smac.utils.io.cmd_reader.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.utils\.io\.cmd\_reader module -=================================== - -.. automodule:: smac.utils.io.cmd_reader - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.utils.io.input_reader.rst b/doc/apidoc/smac.utils.io.input_reader.rst deleted file mode 100644 index 80bebb875..000000000 --- a/doc/apidoc/smac.utils.io.input_reader.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.utils\.io\.input\_reader module -===================================== - -.. automodule:: smac.utils.io.input_reader - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.utils.io.output_directory.rst b/doc/apidoc/smac.utils.io.output_directory.rst deleted file mode 100644 index cdcc2a39e..000000000 --- a/doc/apidoc/smac.utils.io.output_directory.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.utils\.io\.output\_directory module -========================================= - -.. automodule:: smac.utils.io.output_directory - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.utils.io.output_writer.rst b/doc/apidoc/smac.utils.io.output_writer.rst deleted file mode 100644 index f3abd847c..000000000 --- a/doc/apidoc/smac.utils.io.output_writer.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.utils\.io\.output\_writer module -====================================== - -.. automodule:: smac.utils.io.output_writer - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.utils.io.rst b/doc/apidoc/smac.utils.io.rst deleted file mode 100644 index 4f4c45501..000000000 --- a/doc/apidoc/smac.utils.io.rst +++ /dev/null @@ -1,18 +0,0 @@ -smac\.utils\.io package -======================= - -.. automodule:: smac.utils.io - :members: - :undoc-members: - :show-inheritance: - :inherited-members: - - -.. toctree:: - - smac.utils.io.cmd_reader - smac.utils.io.input_reader - smac.utils.io.output_directory - smac.utils.io.output_writer - smac.utils.io.traj_logging - diff --git a/doc/apidoc/smac.utils.io.traj_logging.rst b/doc/apidoc/smac.utils.io.traj_logging.rst deleted file mode 100644 index fc2508f0f..000000000 --- a/doc/apidoc/smac.utils.io.traj_logging.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.utils\.io\.traj\_logging module -===================================== - -.. automodule:: smac.utils.io.traj_logging - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.utils.logging.rst b/doc/apidoc/smac.utils.logging.rst deleted file mode 100644 index 181671821..000000000 --- a/doc/apidoc/smac.utils.logging.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.utils\.logging module -=========================== - -.. automodule:: smac.utils.logging - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.utils.merge_foreign_data.rst b/doc/apidoc/smac.utils.merge_foreign_data.rst deleted file mode 100644 index d55fd79ad..000000000 --- a/doc/apidoc/smac.utils.merge_foreign_data.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.utils\.merge\_foreign\_data module -======================================== - -.. automodule:: smac.utils.merge_foreign_data - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.utils.rst b/doc/apidoc/smac.utils.rst deleted file mode 100644 index ad6380c74..000000000 --- a/doc/apidoc/smac.utils.rst +++ /dev/null @@ -1,26 +0,0 @@ -smac\.utils package -=================== - -.. automodule:: smac.utils - :members: - :undoc-members: - :show-inheritance: - :inherited-members: - -Subpackages - -.. toctree:: - - smac.utils.io - - -.. toctree:: - - smac.utils.constants - smac.utils.duplicate_filter_logging - smac.utils.logging - smac.utils.merge_foreign_data - smac.utils.test_helpers - smac.utils.util_funcs - smac.utils.validate - diff --git a/doc/apidoc/smac.utils.test_helpers.rst b/doc/apidoc/smac.utils.test_helpers.rst deleted file mode 100644 index fd7627edb..000000000 --- a/doc/apidoc/smac.utils.test_helpers.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.utils\.test\_helpers module -================================= - -.. automodule:: smac.utils.test_helpers - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.utils.util_funcs.rst b/doc/apidoc/smac.utils.util_funcs.rst deleted file mode 100644 index 99ed66c26..000000000 --- a/doc/apidoc/smac.utils.util_funcs.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.utils\.util\_funcs module -=============================== - -.. automodule:: smac.utils.util_funcs - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/apidoc/smac.utils.validate.rst b/doc/apidoc/smac.utils.validate.rst deleted file mode 100644 index 2e55cb34f..000000000 --- a/doc/apidoc/smac.utils.validate.rst +++ /dev/null @@ -1,8 +0,0 @@ -smac\.utils\.validate module -============================ - -.. automodule:: smac.utils.validate - :members: - :undoc-members: - :show-inheritance: - :inherited-members: diff --git a/doc/index.rst b/doc/index.rst index 642309064..762878753 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -21,6 +21,7 @@ Contents: installation quickstart + usage_recomendation manual api faq diff --git a/doc/installation.rst b/doc/installation.rst index 199cb7eee..6799d1613 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -54,4 +54,4 @@ command line: git clone https://github.com/automl/SMAC3 cd SMAC3 cat requirements.txt | xargs -n 1 -L 1 pip install - python setup.py install + pip install . diff --git a/doc/quickstart.rst b/doc/quickstart.rst index 95187add9..cd802517d 100644 --- a/doc/quickstart.rst +++ b/doc/quickstart.rst @@ -125,7 +125,7 @@ This example is located in :code:`examples/svm.py`. To use *SMAC* directly with Python, we first import the necessary modules -.. literalinclude:: ../examples/svm.py +.. literalinclude:: ../examples/SMAC4HPO_svm.py :lines: 9-22 :lineno-match: @@ -142,14 +142,14 @@ the configuration. Let's start by creating a ConfigSpace-object and adding the first hyperparameter: the choice of the kernel. -.. literalinclude:: ../examples/svm.py +.. literalinclude:: ../examples/SMAC4HPO_svm.py :lines: 60-65 :lineno-match: We can add Integers, Floats or Categoricals to the ConfigSpace-object all at once, by passing them in a list. -.. literalinclude:: ../examples/svm.py +.. literalinclude:: ../examples/SMAC4HPO_svm.py :lines: 67-70 :lineno-match: @@ -158,7 +158,7 @@ We can reflect this in optimization using **conditions** to deactivate hyperpara Deactivated hyperparameters are not considered during optimization, limiting the search-space to reasonable configurations. This way human knowledge about the problem is introduced. -.. literalinclude:: ../examples/svm.py +.. literalinclude:: ../examples/SMAC4HPO_svm.py :lines: 72-78 :lineno-match: @@ -166,7 +166,7 @@ Conditions can be used for various reasons. The `gamma`-hyperparameter for example can be set to "auto" or to a fixed float-value. We introduce a hyperparameters that is only activated if `gamma` is not set to "auto". -.. literalinclude:: ../examples/svm.py +.. literalinclude:: ../examples/SMAC4HPO_svm.py :lines: 80-88 :lineno-match: @@ -174,7 +174,7 @@ Of course we also define a function to evaluate the configured SVM on the IRIS-d Some options, such as the *kernel* or *C*, can be passed directly. Others, such as *gamma*, need to be translated before the call to the SVM. -.. literalinclude:: ../examples/svm.py +.. literalinclude:: ../examples/SMAC4HPO_svm.py :pyobject: svm_from_cfg :lineno-match: @@ -186,7 +186,7 @@ __ scenario_ The initialization of a scenario in the code uses the same keywords as a scenario-file, which we used in the Branin example. -.. literalinclude:: ../examples/svm.py +.. literalinclude:: ../examples/SMAC4HPO_svm.py :lines: 91-96 :lineno-match: @@ -196,7 +196,7 @@ To automatically handle the exploration of the search space and evaluation of the function, SMAC needs as inputs the scenario object as well as the function. -.. literalinclude:: ../examples/svm.py +.. literalinclude:: ../examples/SMAC4HPO_svm.py :lines: 103-112 :lineno-match: @@ -234,7 +234,7 @@ so that as final output we can see the error value of the incumbent. As a bonus, we can validate our results. This is more useful when optimizing on instances, but we include the code so it is easily applicable for any usecase. -.. literalinclude:: ../examples/svm.py +.. literalinclude:: ../examples/SMAC4HPO_svm.py :lines: 115- :lineno-match: diff --git a/doc/usage_recomendation.rst b/doc/usage_recomendation.rst new file mode 100644 index 000000000..5375b42e8 --- /dev/null +++ b/doc/usage_recomendation.rst @@ -0,0 +1,47 @@ +.. _scenario: options.html#scenario +.. _PCS: options.html#paramcs +.. _TAE: tae.html + +Usage Recommendation +==================== +*SMAC* of course itself offers a lot of design choices, some of which are crucial to achieve peak performance. Luckily, often it is sufficient to distinguish between a few problem classes. +To make the usage of *SMAC* as easy as possible, we provide several facades designed for these different use cases. Here we give some general recommendations on +when to use which facade. These recommendations are based on our experience and technical limitations and is by far not intended to be complete: + ++-----------------------+----------------------+-----------------------+-------------------------+ +| | SMAC4BO | SMAC4HPO | SMAC4AC | ++=======================+======================+=======================+=========================+ +| # parameter | low | low/medium/high | low/medium/high | ++-----------------------+----------------------+-----------------------+-------------------------+ +| categorical parameter | yes | supported | supported | ++-----------------------+----------------------+-----------------------+-------------------------+ +| conditional parameter | yes | supported | supported | ++-----------------------+----------------------+-----------------------+-------------------------+ +| instances | no | None or CV-folds | yes | ++-----------------------+----------------------+-----------------------+-------------------------+ +| stochasticity | no | supported | supported | ++-----------------------+----------------------+-----------------------+-------------------------+ +| objective | any (except runtime) | e.g. validation loss | e.g. runtime or quality | ++-----------------------+----------------------+-----------------------+-------------------------+ + +Some examples of typical use cases: + +*SMAC4BO*: Bayesian Optimization using a *Gaussian Process* and *Expected Improvement* + - Optimizing the objective value of Branin and other low dimensional artificial test functions + - Finding the best learning rate for training a neural network wrt. RMSE on a validation dataset + - Optimizing the choice of kernel and penalty of a SVM wrt. RMSE on a validation dataset + +*SMAC4HPO*: Bayesian optimization using a *Random Forest* + - Finding the optimal choice of machine learning algorithm and its hyperparameters wrt. validation error + - Tuning the architecture and training parameters of a neural network wrt. classification error on a validation dataset + - Optimize hyperparameters of a SVM wrt. the CV-fold error + - Minimize objective values of problems that are noisy and/or yield crashed runs (e.g. due to mem-outs). + - Finding the best setting of an RL-agent to minimize regret (or a set of RL problems) + +*SMAC4AC*: Algorithm configuration using a *Random Forest* + - Minimizing the average time it takes for a SAT-solver to solve a set of SAT instances + - Configuring a MIP solver to solve a set of mixed-integer-problems as fast as possible + - Optimizing the average quality of solutions returned by a configurable TSP solver + +**Important:** If your problem is not covered in this table, this doesn't mean you can't benefit from using our tool. In case of doubt, please create an issue on Github. + diff --git a/examples/SMAC4BO_rosenbrock.py b/examples/SMAC4BO_rosenbrock.py new file mode 100644 index 000000000..381fc48bd --- /dev/null +++ b/examples/SMAC4BO_rosenbrock.py @@ -0,0 +1,56 @@ +import logging + +import numpy as np + +# Import ConfigSpace and different types of parameters +from smac.configspace import ConfigurationSpace +from ConfigSpace.hyperparameters import UniformFloatHyperparameter + +# Import SMAC-utilities +from smac.scenario.scenario import Scenario +from smac.facade.smac_bo_facade import SMAC4BO + + +def rosenbrock_2d(x): + """ The 2 dimensional Rosenbrock function as a toy model + The Rosenbrock function is well know in the optimization community and + often serves as a toy problem. It can be defined for arbitrary + dimensions. The minimium is always at x_i = 1 with a function value of + zero. All input parameters are continuous. The search domain for + all x's is the interval [-5, 10]. + """ + x1 = x["x0"] + x2 = x["x1"] + + val = 100. * (x2 - x1 ** 2.) ** 2. + (1 - x1) ** 2. + return val + + +logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output + +# Build Configuration Space which defines all parameters and their ranges +cs = ConfigurationSpace() +x0 = UniformFloatHyperparameter("x0", -5, 10, default_value=-3) +x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=-4) +cs.add_hyperparameters([x0, x1]) + +# Scenario object +scenario = Scenario({"run_obj": "quality", # we optimize quality (alternatively runtime) + "runcount-limit": 10, # max. number of function evaluations; for this example set to a low number + "cs": cs, # configuration space + "deterministic": "true" + }) + +# Example call of the function +# It returns: Status, Cost, Runtime, Additional Infos +def_value = rosenbrock_2d(cs.get_default_configuration()) +print("Default Value: %.2f" % def_value) + +# Optimize, using a SMAC-object +print("Optimizing! Depending on your machine, this might take a few minutes.") +smac = SMAC4BO(scenario=scenario, + rng=np.random.RandomState(42), + tae_runner=rosenbrock_2d, + ) + +smac.optimize() diff --git a/examples/SMAC4HPO_acq_rosenbrock.py b/examples/SMAC4HPO_acq_rosenbrock.py new file mode 100644 index 000000000..541d86092 --- /dev/null +++ b/examples/SMAC4HPO_acq_rosenbrock.py @@ -0,0 +1,66 @@ +import logging + +import numpy as np + +# Import ConfigSpace and different types of parameters +from smac.configspace import ConfigurationSpace +from ConfigSpace.hyperparameters import UniformFloatHyperparameter + +# Import SMAC-utilities +from smac.scenario.scenario import Scenario +from smac.facade.smac_hpo_facade import SMAC4HPO +from smac.optimizer.acquisition import LCB, EI, PI +from smac.initial_design.latin_hypercube_design import LHDesign +from smac.runhistory.runhistory2epm import RunHistory2EPM4InvScaledCost + + +def rosenbrock_2d(x): + """ The 2 dimensional Rosenbrock function as a toy model + The Rosenbrock function is well know in the optimization community and + often serves as a toy problem. It can be defined for arbitrary + dimensions. The minimium is always at x_i = 1 with a function value of + zero. All input parameters are continuous. The search domain for + all x's is the interval [-5, 10]. + """ + x1 = x["x0"] + x2 = x["x1"] + + val = 100. * (x2 - x1 ** 2.) ** 2. + (1 - x1) ** 2. + return val + + +logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output + +# Build Configuration Space which defines all parameters and their ranges +cs = ConfigurationSpace() +x0 = UniformFloatHyperparameter("x0", -5, 10, default_value=-3) +x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=-4) +cs.add_hyperparameters([x0, x1]) + +# Scenario object +scenario = Scenario({"run_obj": "quality", # we optimize quality (alternatively runtime) + "runcount-limit": 10, # max. number of function evaluations; for this example set to a low number + "cs": cs, # configuration space + "deterministic": "true" + }) + +# Example call of the function +# It returns: Status, Cost, Runtime, Additional Infos +def_value = rosenbrock_2d(cs.get_default_configuration()) +print("Default Value: %.2f" % def_value) + +# Optimize, using a SMAC-object +for acquisition_func in (LCB, EI, PI): + print("Optimizing with %s! Depending on your machine, this might take a few minutes." % acquisition_func) + smac = SMAC4HPO(scenario=scenario, rng=np.random.RandomState(42), + tae_runner=rosenbrock_2d, + initial_design=LHDesign, + initial_design_kwargs={'n_configs_x_params': 4, + 'max_config_fracs': 1.0}, + runhistory2epm=RunHistory2EPM4InvScaledCost, + acquisition_function_optimizer_kwargs={'max_steps': 100}, + acquisition_function=acquisition_func, + acquisition_function_kwargs={'par': 0.01} + ) + + smac.optimize() diff --git a/examples/rf.py b/examples/SMAC4HPO_rf.py similarity index 91% rename from examples/rf.py rename to examples/SMAC4HPO_rf.py index 5b4e1aaad..26407e729 100644 --- a/examples/rf.py +++ b/examples/SMAC4HPO_rf.py @@ -1,6 +1,4 @@ import logging -import os -import inspect import numpy as np from sklearn.metrics import make_scorer @@ -12,12 +10,12 @@ from ConfigSpace.hyperparameters import CategoricalHyperparameter, \ UniformFloatHyperparameter, UniformIntegerHyperparameter -from smac.tae.execute_func import ExecuteTAFuncDict from smac.scenario.scenario import Scenario -from smac.facade.smac_facade import SMAC +from smac.facade.smac_hpo_facade import SMAC4HPO boston = load_boston() + def rf_from_cfg(cfg, seed): """ Creates a random forest regressor from sklearn and fits the given data on it. @@ -86,20 +84,20 @@ def rmse(y, y_pred): # SMAC scenario oject scenario = Scenario({"run_obj": "quality", # we optimize quality (alternative runtime) - "runcount-limit": 50, # maximum number of function evaluations + "runcount-limit": 10, # max. number of function evaluations; for this example set to a low number "cs": cs, # configuration space "deterministic": "true", "memory_limit": 3072, # adapt this to reasonable value for your hardware }) # To optimize, we pass the function to the SMAC-object -smac = SMAC(scenario=scenario, rng=np.random.RandomState(42), - tae_runner=rf_from_cfg) +smac = SMAC4HPO(scenario=scenario, rng=np.random.RandomState(42), + tae_runner=rf_from_cfg) # Example call of the function with default values # It returns: Status, Cost, Runtime, Additional Infos def_value = smac.get_tae_runner().run(cs.get_default_configuration(), 1)[1] -print("Value for default configuration: %.2f" % (def_value)) +print("Value for default configuration: %.2f" % def_value) # Start optimization try: @@ -108,4 +106,4 @@ def rmse(y, y_pred): incumbent = smac.solver.incumbent inc_value = smac.get_tae_runner().run(incumbent, 1)[1] -print("Optimized Value: %.2f" % (inc_value)) +print("Optimized Value: %.2f" % inc_value) diff --git a/examples/SMAC4HPO_rosenbrock.py b/examples/SMAC4HPO_rosenbrock.py new file mode 100644 index 000000000..f51868aa7 --- /dev/null +++ b/examples/SMAC4HPO_rosenbrock.py @@ -0,0 +1,55 @@ +import logging + +import numpy as np + +# Import ConfigSpace and different types of parameters +from smac.configspace import ConfigurationSpace +from ConfigSpace.hyperparameters import UniformFloatHyperparameter + +# Import SMAC-utilities +from smac.scenario.scenario import Scenario +from smac.facade.smac_hpo_facade import SMAC4HPO + + +def rosenbrock_2d(x): + """ The 2 dimensional Rosenbrock function as a toy model + The Rosenbrock function is well know in the optimization community and + often serves as a toy problem. It can be defined for arbitrary + dimensions. The minimium is always at x_i = 1 with a function value of + zero. All input parameters are continuous. The search domain for + all x's is the interval [-5, 10]. + """ + x1 = x["x0"] + x2 = x["x1"] + + val = 100. * (x2 - x1 ** 2.) ** 2. + (1 - x1) ** 2. + return val + + +logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output + +# Build Configuration Space which defines all parameters and their ranges +cs = ConfigurationSpace() +x0 = UniformFloatHyperparameter("x0", -5, 10, default_value=-3) +x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=-4) +cs.add_hyperparameters([x0, x1]) + +# Scenario object +scenario = Scenario({"run_obj": "quality", # we optimize quality (alternatively runtime) + "runcount-limit": 10, # max. number of function evaluations; for this example set to a low number + "cs": cs, # configuration space + "deterministic": "true" + }) + +# Example call of the function +# It returns: Status, Cost, Runtime, Additional Infos +def_value = rosenbrock_2d(cs.get_default_configuration()) +print("Default Value: %.2f" % def_value) + +# Optimize, using a SMAC-object +print("Optimizing! Depending on your machine, this might take a few minutes.") +smac = SMAC4HPO(scenario=scenario, + rng=np.random.RandomState(42), + tae_runner=rosenbrock_2d) + +smac.optimize() diff --git a/examples/svm.py b/examples/SMAC4HPO_svm.py similarity index 95% rename from examples/svm.py rename to examples/SMAC4HPO_svm.py index f6905a2f0..41e5c5b54 100644 --- a/examples/svm.py +++ b/examples/SMAC4HPO_svm.py @@ -19,7 +19,7 @@ # Import SMAC-utilities from smac.tae.execute_func import ExecuteTAFuncDict from smac.scenario.scenario import Scenario -from smac.facade.smac_facade import SMAC +from smac.facade.smac_hpo_facade import SMAC4HPO # We load the iris-dataset (a widely used benchmark) @@ -90,7 +90,7 @@ def svm_from_cfg(cfg): # Scenario object scenario = Scenario({"run_obj": "quality", # we optimize quality (alternatively runtime) - "runcount-limit": 200, # maximum function evaluations + "runcount-limit": 50, # max. number of function evaluations; for this example set to a low number "cs": cs, # configuration space "deterministic": "true" }) @@ -102,7 +102,7 @@ def svm_from_cfg(cfg): # Optimize, using a SMAC-object print("Optimizing! Depending on your machine, this might take a few minutes.") -smac = SMAC(scenario=scenario, rng=np.random.RandomState(42), +smac = SMAC4HPO(scenario=scenario, rng=np.random.RandomState(42), tae_runner=svm_from_cfg) incumbent = smac.optimize() diff --git a/examples/branin/branin_fmin.py b/examples/branin/branin_fmin.py index 6963d7668..8db9bb319 100644 --- a/examples/branin/branin_fmin.py +++ b/examples/branin/branin_fmin.py @@ -16,6 +16,6 @@ x, cost, _ = fmin_smac(func=branin, # function x0=[0, 0], # default configuration bounds=[(-5, 10), (0, 15)], # limits - maxfun=500, # maximum number of evaluations + maxfun=10, # maximum number of evaluations rng=3) # random seed print("Optimum at {} with cost of {}".format(x, cost)) diff --git a/examples/branin/scenario.txt b/examples/branin/scenario.txt index 294a213a7..de0ced846 100644 --- a/examples/branin/scenario.txt +++ b/examples/branin/scenario.txt @@ -1,5 +1,5 @@ algo = python cmdline_wrapper.py paramfile = param_config_space.pcs run_obj = quality -runcount_limit = 50 +runcount_limit = 10 deterministic = 1 diff --git a/examples/rosenbrock.py b/examples/fmin_rosenbrock.py similarity index 74% rename from examples/rosenbrock.py rename to examples/fmin_rosenbrock.py index 2a41abd49..c5c91808c 100644 --- a/examples/rosenbrock.py +++ b/examples/fmin_rosenbrock.py @@ -11,7 +11,7 @@ def rosenbrock_2d(x): often serves as a toy problem. It can be defined for arbitrary dimensions. The minimium is always at x_i = 1 with a function value of zero. All input parameters are continuous. The search domain for - all x's is the interval [-5, 5]. + all x's is the interval [-5, 10]. """ x1 = x[0] x2 = x[1] @@ -21,12 +21,14 @@ def rosenbrock_2d(x): # debug output logging.basicConfig(level=20) -logger = logging.getLogger("Optimizer") # Enable to show Debug outputs +logger = logging.getLogger("Optimizer") # Enable to show Debug outputs +# fmin_smac assumes that the function is deterministic +# and uses under the hood the SMAC4HPO x, cost, _ = fmin_smac(func=rosenbrock_2d, x0=[-3, -4], - bounds=[(-5, 5), (-5, 5)], - maxfun=50, + bounds=[(-5, 10), (-5, 10)], + maxfun=10, rng=3) # Passing a seed makes fmin_smac determistic print("Best x: %s; with cost: %f"% (str(x), cost)) diff --git a/extras_require.json b/extras_require.json new file mode 100644 index 000000000..6f86c69ae --- /dev/null +++ b/extras_require.json @@ -0,0 +1,9 @@ +{ + "gp": [ + "emcee>=2.1.0", + "scikit-optimize" + ], + "lhd": [ + "pyDOE" + ] +} diff --git a/requirements.txt b/requirements.txt index 53d1e2d96..b960bbb6e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,10 @@ -setuptools -cython numpy>=1.7.1 scipy>=0.18.1 -six psutil pynisher>=0.4.1 ConfigSpace>=0.4.6,<0.5 scikit-learn>=0.18.0 -pyrfr>=0.5.0 -sphinx -sphinx_rtd_theme -joblib -nose>=1.3.0 -pyDOE +pyrfr>=0.8.0 sobol_seq -statsmodels -emcee>=2.1.0 -george \ No newline at end of file +joblib +lazy_import diff --git a/scripts/plot_traj_perf.py b/scripts/plot_traj_perf.py index 243086e19..1389594ab 100644 --- a/scripts/plot_traj_perf.py +++ b/scripts/plot_traj_perf.py @@ -9,15 +9,15 @@ from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter from smac.utils.io.traj_logging import TrajLogger -from smac.runhistory.runhistory import RunHistory from smac.scenario.scenario import Scenario -from smac.facade.smac_facade import SMAC +from smac.facade.smac_ac_facade import SMAC4AC from smac.configspace import convert_configurations_to_array __author__ = "Marius Lindauer" __copyright__ = "Copyright 2017, ML4AAD" __license__ = "3-clause BSD" + def setup_SMAC_from_file(smac_out_dn: str, add_dn:typing.List[str]): ''' @@ -44,7 +44,7 @@ def setup_SMAC_from_file(smac_out_dn: str, scenario_fn = os.path.join(smac_out_dn, "scenario.txt") scenario = Scenario(scenario_fn, {"output_dir": ""}) - smac = SMAC(scenario=scenario) + smac = SMAC4AC(scenario=scenario) rh = smac.solver.runhistory rh.load_json(os.path.join(smac_out_dn, "runhistory.json"), cs=scenario.cs) diff --git a/setup.py b/setup.py index 8cb5c8890..6cf4bd4f9 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 +import json import os from setuptools import setup with open('requirements.txt') as fh: - requirements = fh.read() -requirements = requirements.split('\n') -requirements = [requirement.strip() for requirement in requirements] + requirements = [line.strip() for line in fh.readlines()] +with open('extras_require.json') as fh: + extras_require = json.load(fh) + extras_require['all'] = set(sum(extras_require.values(), [])) def get_version(): @@ -30,6 +32,8 @@ def get_author(): setup( python_requires=">=3.5.2", install_requires=requirements, + extras_require=extras_require, + package_data={'smac': ['requirements.txt', 'extras_require.json']}, author=get_author(), version=get_version(), test_suite="nose.collector", diff --git a/smac/__init__.py b/smac/__init__.py index e85a9ec82..010b08ea4 100644 --- a/smac/__init__.py +++ b/smac/__init__.py @@ -1,23 +1,29 @@ +import json import os import sys +import lazy_import from smac.utils import dependencies -__version__ = '0.10.0' +__version__ = '0.11.0' __author__ = 'Marius Lindauer, Matthias Feurer, Katharina Eggensperger, Joshua Marben, André Biedenkapp, Aaron Klein, Stefan Falkner and Frank Hutter' -__MANDATORY_PACKAGES__ = """ -numpy>=1.7.1 -scipy>=0.18.1 -six -psutil -pynisher>=0.4.1 -ConfigSpace>=0.4.6,<0.5 -scikit-learn>=0.18.0 -pyrfr>=0.5.0 -joblib -""" -dependencies.verify_packages(__MANDATORY_PACKAGES__) + +with open(os.path.join(os.path.dirname(__file__), 'requirements.txt')) as fh: + dependencies.verify_packages(fh.read()) + +with open(os.path.join(os.path.dirname(__file__), 'extras_require.json')) as fh: + extras_require = json.load(fh) + +extras_installed = set() +for name, requirements in extras_require.items(): + if dependencies.are_valid_packages(requirements): + extras_installed.add(name) + for requirement in requirements: + package_name = dependencies.RE_PATTERN.match(requirement).group('name') + if package_name == 'scikit-optimize': + package_name = 'skopt' + lazy_import.lazy_module(package_name) if sys.version_info < (3, 5, 2): raise ValueError("SMAC requires Python 3.5.2 or newer.") diff --git a/smac/configspace/util.py b/smac/configspace/util.py index bfda79f9f..13c17a68e 100644 --- a/smac/configspace/util.py +++ b/smac/configspace/util.py @@ -2,7 +2,7 @@ import numpy as np -from smac.configspace import Configuration, ConfigurationSpace +from smac.configspace import Configuration def convert_configurations_to_array(configs: List[Configuration]) -> np.ndarray: @@ -18,40 +18,5 @@ def convert_configurations_to_array(configs: List[Configuration]) -> np.ndarray: Returns ------- np.ndarray - Array with configuration hyperparameters. Inactive values are imputed - with their default value. """ - configs_array = np.array([config.get_array() for config in configs], - dtype=np.float64) - configuration_space = configs[0].configuration_space - return impute_default_values(configuration_space, configs_array) - - -def impute_default_values( - configuration_space: ConfigurationSpace, - configs_array: np.ndarray -) -> np.ndarray: - """Impute inactive hyperparameters in configuration array with their default. - - Necessary to apply an EPM to the data. - - Parameters - ---------- - configuration_space : ConfigurationSpace - - configs_array : np.ndarray - Array of configurations. - - Returns - ------- - np.ndarray - Array with configuration hyperparameters. Inactive values are imputed - with their default value. - """ - for hp in configuration_space.get_hyperparameters(): - default = hp.normalized_default_value - idx = configuration_space.get_idx_by_hyperparameter_name(hp.name) - nonfinite_mask = ~np.isfinite(configs_array[:, idx]) - configs_array[nonfinite_mask, idx] = default - - return configs_array + return np.array([config.get_array() for config in configs], dtype=np.float64) diff --git a/smac/epm/base_epm.py b/smac/epm/base_epm.py index ce24c68bc..2f60e5f56 100644 --- a/smac/epm/base_epm.py +++ b/smac/epm/base_epm.py @@ -6,6 +6,10 @@ from sklearn.preprocessing import MinMaxScaler from sklearn.exceptions import NotFittedError +from smac.configspace import ConfigurationSpace +from smac.utils.constants import VERY_SMALL_NUMBER +from smac.utils.logging import PickableLoggerAdapter + __author__ = "Marius Lindauer" __copyright__ = "Copyright 2016, ML4AAD" __license__ = "3-clause BSD" @@ -45,8 +49,10 @@ class AbstractEPM(object): """ def __init__(self, + configspace: ConfigurationSpace, types: np.ndarray, bounds: typing.List[typing.Tuple[float, float]], + seed: int, instance_features: np.ndarray=None, pca_components: float=None, ): @@ -54,14 +60,18 @@ def __init__(self, Parameters ---------- + configspace : ConfigurationSpace + Configuration space to tune for. types : np.ndarray (D) Specifies the number of categorical values of an input dimension where the i-th entry corresponds to the i-th input dimension. Let's say we have 2 dimension where the first dimension consists of 3 different categorical choices and the second dimension is continuous than we - have to pass np.array([2, 0]). Note that we count starting from 0. + have to pass np.array([3, 0]). Note that we count starting from 0. bounds : list - Specifies the bounds for continuous features. + bounds of input dimensions: (lower, uppper) for continuous dims; (n_cat, np.nan) for categorical dims + seed : int + The seed that is passed to the model library. instance_features : np.ndarray (I, K) Contains the K dimensional instance features of the I different instances @@ -70,6 +80,8 @@ def __init__(self, dimensionality of instance features. Requires to set n_feats (> pca_dims). """ + self.configspace = configspace + self.seed = seed self.instance_features = instance_features self.pca_components = pca_components @@ -87,13 +99,15 @@ def __init__(self, self.scaler = MinMaxScaler() # Never use a lower variance than this - self.var_threshold = 10 ** -5 + self.var_threshold = VERY_SMALL_NUMBER self.bounds = bounds self.types = types # Initial types array which is used to reset the type array at every call to train() self._initial_types = types.copy() + self.logger = PickableLoggerAdapter(self.__module__ + "." + self.__class__.__name__) + def train(self, X: np.ndarray, Y: np.ndarray) -> 'AbstractEPM': """Trains the EPM on X and Y. @@ -114,7 +128,7 @@ def train(self, X: np.ndarray, Y: np.ndarray) -> 'AbstractEPM': if len(X.shape) != 2: raise ValueError('Expected 2d array, got %dd array!' % len(X.shape)) if X.shape[1] != len(self.types): - raise ValueError('Feature mismatch: X should have %d features, but has %d' % (X.shape[1], len(self.types))) + raise ValueError('Feature mismatch: X should have %d features, but has %d' % (len(self.types), X.shape[1])) if X.shape[0] != Y.shape[0]: raise ValueError('X.shape[0] (%s) != y.shape[0] (%s)' % (X.shape[0], Y.shape[0])) @@ -238,8 +252,10 @@ def predict_marginalized_over_instances(self, X: np.ndarray) -> typing.Tuple[np. if len(X.shape) != 2: raise ValueError('Expected 2d array, got %dd array!' % len(X.shape)) - if X.shape[1] != len(self.types): - raise ValueError('Rows in X should have %d entries but have %d!' % (len(self.types), X.shape[1])) + if X.shape[1] != self.bounds.shape[0]: + raise ValueError('Rows in X should have %d entries but have %d!' % + (self.bounds.shape[0], + X.shape[1])) if self.instance_features is None or \ len(self.instance_features) == 0: diff --git a/smac/epm/base_gp.py b/smac/epm/base_gp.py index 72cf70868..9c8fa9eec 100644 --- a/smac/epm/base_gp.py +++ b/smac/epm/base_gp.py @@ -1,24 +1,121 @@ +from typing import List, Optional, Tuple, Union + import numpy as np +import sklearn.gaussian_process.kernels from smac.epm.base_epm import AbstractEPM +import smac.epm.gp_base_prior class BaseModel(AbstractEPM): - def __init__(self, types, bounds): + def __init__(self, configspace, types, bounds, seed, **kwargs): """ Abstract base class for all Gaussian process models. """ - super().__init__(types=types, bounds=bounds, instance_features=None, pca_components=None) + super().__init__(configspace=configspace, types=types, bounds=bounds, seed=seed, **kwargs) + + self.rng = np.random.RandomState(seed) + + def _normalize_y(self, y: np.ndarray) -> np.ndarray: + """Normalize data to zero mean unit standard deviation. + + Parameters + ---------- + y : np.ndarray + Targets for the Gaussian process + + Returns + ------- + np.ndarray + """ + self.mean_y_ = np.mean(y) + self.std_y_ = np.std(y) + if self.std_y_ == 0: + self.std_y_ = 1 + return (y - self.mean_y_) / self.std_y_ + + def _untransform_y( + self, + y: np.ndarray, + var: Optional[np.ndarray] = None, + ) -> Union[np.ndarray, Tuple[np.ndarray, np.ndarray]]: + """Transform zeromean unit standard deviation data into the regular space. + + This function should be used after a prediction with the Gaussian process which was trained on normalized data. + + Parameters + ---------- + y : np.ndarray + Normalized data. + var : np.ndarray (optional) + Normalized variance + + Returns + ------- + np.ndarray on Tuple[np.ndarray, np.ndarray] + """ + y = y * self.std_y_ + self.mean_y_ + if var is not None: + var = var * self.std_y_ ** 2 + return y, var + return y - lower = [] - upper = [] - for bound in bounds: - lower.append(bound[0]) - upper.append(bound[1]) + def _get_all_priors( + self, + add_bound_priors: bool = True, + add_soft_bounds: bool = False, + ) -> List[List[smac.epm.gp_base_prior.Prior]]: + # Obtain a list of all priors for each tunable hyperparameter of the kernel + all_priors = [] + to_visit = [] + to_visit.append(self.gp.kernel.k1) + to_visit.append(self.gp.kernel.k2) + while len(to_visit) > 0: + current_param = to_visit.pop(0) + if isinstance(current_param, sklearn.gaussian_process.kernels.KernelOperator): + to_visit.insert(0, current_param.k1) + to_visit.insert(1, current_param.k2) + continue + elif isinstance(current_param, sklearn.gaussian_process.kernels.Kernel): + hps = current_param.hyperparameters + assert len(hps) == 1 + hp = hps[0] + if hp.fixed: + continue + bounds = hps[0].bounds + for i in range(hps[0].n_elements): + priors_for_hp = [] + if current_param.prior is not None: + priors_for_hp.append(current_param.prior) + if add_bound_priors: + if add_soft_bounds: + priors_for_hp.append(smac.epm.gp_base_prior.SoftTopHatPrior( + lower_bound=bounds[i][0], upper_bound=bounds[i][1], rng=self.rng, + )) + else: + priors_for_hp.append(smac.epm.gp_base_prior.TophatPrior( + lower_bound=bounds[i][0], upper_bound=bounds[i][1], rng=self.rng, + )) + all_priors.append(priors_for_hp) + return all_priors - self.lower = np.array(lower) - self.upper = np.array(upper) + def _set_has_conditions(self): + has_conditions = len(self.configspace.get_conditions()) > 0 + to_visit = [] + to_visit.append(self.kernel) + while len(to_visit) > 0: + current_param = to_visit.pop(0) + if isinstance(current_param, sklearn.gaussian_process.kernels.KernelOperator): + to_visit.insert(0, current_param.k1) + to_visit.insert(1, current_param.k2) + current_param.has_conditions = has_conditions + elif isinstance(current_param, sklearn.gaussian_process.kernels.Kernel): + current_param.has_conditions = has_conditions + else: + raise ValueError(current_param) - self.X = None - self.y = None + def _impute_inactive(self, X: np.ndarray) -> np.ndarray: + X = X.copy() + X[~np.isfinite(X)] = -1 + return X diff --git a/smac/epm/base_rf.py b/smac/epm/base_rf.py new file mode 100644 index 000000000..94ff5d654 --- /dev/null +++ b/smac/epm/base_rf.py @@ -0,0 +1,43 @@ +import numpy as np + +from smac.epm.base_epm import AbstractEPM +from smac.configspace import ( + CategoricalHyperparameter, + UniformFloatHyperparameter, + UniformIntegerHyperparameter, + Constant, +) + + +class BaseModel(AbstractEPM): + + def __init__(self, configspace, types, bounds, seed, **kwargs): + """ + Abstract base class for all random forest models. + """ + super().__init__(configspace=configspace, types=types, bounds=bounds, seed=seed, **kwargs) + + self.rng = np.random.RandomState(seed) + self.impute_values = dict() + + def _impute_inactive(self, X: np.ndarray) -> np.ndarray: + X = X.copy() + for idx, hp in enumerate(self.configspace.get_hyperparameters()): + if idx not in self.impute_values: + parents = self.configspace.get_parents_of(hp.name) + if len(parents) == 0: + self.impute_values[idx] = None + else: + if isinstance(hp, CategoricalHyperparameter): + self.impute_values[idx] = len(hp.choices) + elif isinstance(hp, (UniformFloatHyperparameter, UniformIntegerHyperparameter)): + self.impute_values[idx] = -1 + elif isinstance(hp, Constant): + self.impute_values[idx] = 1 + else: + raise ValueError + + nonfinite_mask = ~np.isfinite(X[:, idx]) + X[nonfinite_mask, idx] = self.impute_values[idx] + + return X diff --git a/smac/epm/gaussian_process.py b/smac/epm/gaussian_process.py index e0a1eda68..f50ae9c4d 100644 --- a/smac/epm/gaussian_process.py +++ b/smac/epm/gaussian_process.py @@ -1,15 +1,18 @@ import logging import typing -import george import numpy as np +from lazy_import import lazy_callable from scipy import optimize -from smac.epm import normalization +from smac.configspace import ConfigurationSpace from smac.epm.base_gp import BaseModel -from smac.epm.gp_base_prior import BasePrior +from smac.utils.constants import VERY_SMALL_NUMBER logger = logging.getLogger(__name__) +Kernel = lazy_callable('skopt.learning.gaussian_process.kernels.Kernel') +GaussianProcessRegressor = lazy_callable( + 'skopt.learning.gaussian_process.GaussianProcessRegressor') class GaussianProcess(BaseModel): @@ -34,56 +37,43 @@ class GaussianProcess(BaseModel): have to pass np.array([2, 0]). Note that we count starting from 0. bounds : list Specifies the bounds for continuous features. + seed : int + Model seed. kernel : george kernel object Specifies the kernel that is used for all Gaussian Process prior : prior object Defines a prior for the hyperparameters of the GP. Make sure that it implements the Prior interface. - noise : float - Noise term that is added to the diagonal of the covariance matrix - for the Cholesky decomposition. - use_gradients : bool - Use gradient information to optimize the negative log likelihood - normalize_output : bool + normalize_y : bool Zero mean unit variance normalization of the output values - normalize_input : bool - Normalize all inputs to be in [0, 1]. This is important to define good priors for the - length scales. rng: np.random.RandomState Random number generator """ def __init__( self, + configspace: ConfigurationSpace, types: np.ndarray, bounds: typing.List[typing.Tuple[float, float]], - kernel: george.kernels.Kernel, - prior: BasePrior=None, - noise: float=1e-3, - use_gradients: bool=False, - normalize_output: bool=True, - normalize_input: bool=True, - rng: typing.Optional[np.random.RandomState]=None, + seed: int, + kernel: Kernel, + normalize_y: bool=True, + n_opt_restarts=10, + **kwargs ): - super().__init__(types=types, bounds=bounds) - - if rng is None: - self.rng = np.random.RandomState(np.random.randint(0, 10000)) - else: - self.rng = rng + super().__init__(configspace=configspace, types=types, bounds=bounds, seed=seed, **kwargs) self.kernel = kernel self.gp = None - self.prior = prior - self.noise = noise - self.use_gradients = use_gradients - self.normalize_output = normalize_output - self.normalize_input = normalize_input - self.X = None - self.y = None + self.normalize_y = normalize_y + self.n_opt_restarts = n_opt_restarts + self.hypers = [] self.is_trained = False + self._n_ll_evals = 0 + + self._set_has_conditions() def _train(self, X: np.ndarray, y: np.ndarray, do_optimize: bool=True): """ @@ -104,49 +94,43 @@ def _train(self, X: np.ndarray, y: np.ndarray, do_optimize: bool=True): the default hyperparameters of the kernel are used. """ - if self.normalize_input: - # Normalize input to be in [0, 1] - self.X, self.lower, self.upper = normalization.zero_one_normalization(X, self.lower, self.upper) - else: - self.X = X - - if len(y.shape) > 1: - y = y.flatten() - if len(y) != len(X): - raise ValueError('Shape mismatch: %s vs %s' % (y.shape, X.shape)) - - if self.normalize_output: - # Normalize output to have zero mean and unit standard deviation - self.y, self.y_mean, self.y_std = normalization.zero_mean_unit_var_normalization(y) - if self.y_std == 0: - raise ValueError("Cannot normalize output. All targets have the same value") - else: - self.y = y + X = self._impute_inactive(X) + if self.normalize_y: + y = self._normalize_y(y) - # Use the empirical mean of the data as mean for the GP - self.mean = np.mean(self.y, axis=0) - self.gp = george.GP(self.kernel, mean=self.mean) + n_tries = 10 + for i in range(n_tries): + try: + self.gp = GaussianProcessRegressor( + kernel=self.kernel, + normalize_y=False, + optimizer=None, + n_restarts_optimizer=-1, # Do not use scikit-learn's optimization routine + alpha=0, # Governed by the kernel + noise=None, + random_state=self.rng, + ) + self.gp.fit(X, y) + break + except np.linalg.LinAlgError as e: + if i == n_tries: + raise e + # Assume that the last entry of theta is the noise + theta = np.exp(self.kernel.theta) + theta[-1] += 1 + self.kernel.theta = np.log(theta) if do_optimize: + self._all_priors = self._get_all_priors(add_bound_priors=False) self.hypers = self._optimize() - self.gp.kernel.set_parameter_vector(self.hypers[:-1]) - self.noise = np.exp(self.hypers[-1]) # sigma^2 + self.gp.kernel.theta = self.hypers + self.gp.fit(X, y) else: - self.hypers = self.gp.kernel.get_parameter_vector() - self.hypers = np.append(self.hypers, np.log(self.noise)) - - try: - self.gp.compute(self.X, yerr=np.sqrt(self.noise)) - except np.linalg.LinAlgError: - self.noise *= 10 - self.gp.compute(self.X, yerr=np.sqrt(self.noise)) + self.hypers = self.gp.kernel.theta self.is_trained = True - def _get_noise(self): - return self.noise - - def _nll(self, theta: np.ndarray) -> float: + def _nll(self, theta: np.ndarray) -> typing.Tuple[float, np.ndarray]: """ Returns the negative marginal log likelihood (+ the prior) for a hyperparameter configuration theta. @@ -163,52 +147,23 @@ def _nll(self, theta: np.ndarray) -> float: float lnlikelihood + prior """ - # Specify bounds to keep things sane - if np.any((-20 > theta) + (theta > 20)): - return 1e25 - - # The last entry of theta is always the noise - self.gp.kernel.set_parameter_vector(theta[:-1]) - noise = np.exp(theta[-1]) # sigma^2 + self._n_ll_evals += 1 try: - self.gp.compute(self.X, yerr=np.sqrt(noise)) + lml, grad = self.gp.log_marginal_likelihood(theta, eval_gradient=True) except np.linalg.LinAlgError: - return 1e25 - - ll = self.gp.lnlikelihood(self.y, quiet=True) + return 1e25, np.zeros(theta.shape) - # Add prior - if self.prior is not None: - ll += self.prior.lnprob(theta) + for dim, priors in enumerate(self._all_priors): + for prior in priors: + lml += prior.lnprob(theta[dim]) + grad[dim] += prior.gradient(theta[dim]) # We add a minus here because scipy is minimizing - return -ll if np.isfinite(ll) else 1e25 - - def _grad_nll(self, theta: np.ndarray): - - self.gp.kernel.set_parameter_vector(theta[:-1]) - noise = np.exp(theta[-1]) - - self.gp.compute(self.X, yerr=np.sqrt(noise)) - - self.gp._compute_alpha(self.y) - K_inv = self.gp.solver.apply_inverse(np.eye(self.gp._alpha.size), - in_place=True) - - # The gradients of the Gram matrix, for the noise this is just - # the identity matrix - Kg = self.gp.kernel.gradient(self.gp._x) - Kg = np.concatenate((Kg, np.eye(Kg.shape[0])[:, :, None]), axis=2) - - # Calculate the gradient. - A = np.outer(self.gp._alpha, self.gp._alpha) - K_inv - g = 0.5 * np.einsum('ijk,ij', Kg, A) - - if self.prior is not None: - g += self.prior.gradient(theta) - - return -g + if not np.isfinite(lml).all() or not np.all(np.isfinite(grad)): + return 1e25, np.zeros(theta.shape) + else: + return -lml, -grad def _optimize(self) -> np.ndarray: """ @@ -220,23 +175,43 @@ def _optimize(self) -> np.ndarray: theta : np.ndarray(H) Hyperparameter vector that maximizes the marginal log likelihood """ - # Start optimization from the previous hyperparameter configuration - p0 = self.gp.kernel.get_parameter_vector() - p0 = np.append(p0, np.log(self.noise)) - if self.use_gradients: - theta, _, _ = optimize.minimize(self._nll, p0, - method="BFGS", - jac=self._grad_nll) - else: - try: - results = optimize.minimize(self._nll, p0, method='L-BFGS-B') - theta = results.x - except ValueError: - logging.error("Could not find a valid hyperparameter configuration! Use initial configuration") - theta = p0 + log_bounds = [(b[0], b[1]) for b in self.gp.kernel.bounds] - return theta + # Start optimization from the previous hyperparameter configuration + p0 = [self.gp.kernel.theta] + if self.n_opt_restarts > 0: + dim_samples = [] + for dim, hp_bound in enumerate(log_bounds): + prior = self._all_priors[dim] + # Always sample from the first prior + if isinstance(prior, list): + if len(prior) == 0: + prior = None + else: + prior = prior[0] + if prior is None: + try: + sample = self.rng.uniform( + low=hp_bound[0], + high=hp_bound[1], + size=(self.n_opt_restarts,), + ) + except OverflowError: + raise ValueError('OverflowError while sampling from (%f, %f)' % (hp_bound[0], hp_bound[1])) + dim_samples.append(sample.flatten()) + else: + dim_samples.append(prior.sample_from_prior(self.n_opt_restarts).flatten()) + p0 += list(np.vstack(dim_samples).transpose()) + + theta_star = None + f_opt_star = np.inf + for i, start_point in enumerate(p0): + theta, f_opt, _ = optimize.fmin_l_bfgs_b(self._nll, start_point, bounds=log_bounds) + if f_opt < f_opt_star: + f_opt_star = f_opt + theta_star = theta + return theta_star def _predict(self, X_test: np.ndarray, full_cov: bool=False): r""" @@ -262,26 +237,16 @@ def _predict(self, X_test: np.ndarray, full_cov: bool=False): if not self.is_trained: raise Exception('Model has to be trained first!') - if self.normalize_input: - X_test_norm, _, _ = normalization.zero_one_normalization(X_test, self.lower, self.upper) - else: - X_test_norm = X_test - - mu, var = self.gp.predict(self.y, X_test_norm) - - if self.normalize_output: - mu = normalization.zero_mean_unit_var_unnormalization(mu, self.y_mean, self.y_std) - var *= self.y_std ** 2 - if not full_cov: - var = np.diag(var) + X_test = self._impute_inactive(X_test) + mu, var = self.gp.predict(X_test, return_cov=True) + var = np.diag(var) # Clip negative variances and set them to the smallest # positive float value - if var.shape[0] == 1: - var = np.clip(var, np.finfo(var.dtype).eps, np.inf) - else: - var = np.clip(var, np.finfo(var.dtype).eps, np.inf) - var[np.where((var < np.finfo(var.dtype).eps) & (var > -np.finfo(var.dtype).eps))] = 0 + var = np.clip(var, VERY_SMALL_NUMBER, np.inf) + + if self.normalize_y: + mu, var = self._untransform_y(mu, var) return mu, var @@ -303,18 +268,15 @@ def sample_functions(self, X_test: np.ndarray, n_funcs: int=1) -> np.ndarray: The F function values drawn at the N test points. """ - if self.normalize_input: - X_test_norm, _, _ = normalization.zero_one_normalization(X_test, self.lower, self.upper) - else: - X_test_norm = X_test - if not self.is_trained: raise Exception('Model has to be trained first!') - funcs = self.gp.sample_conditional(self.y, X_test_norm, n_funcs) + X_test = self._impute_inactive(X_test) + funcs = self.gp.sample_y(X_test, n_samples=n_funcs, random_state=self.rng) + funcs = np.squeeze(funcs, axis=1) - if self.normalize_output: - funcs = normalization.zero_mean_unit_var_unnormalization(funcs, self.y_mean, self.y_std) + if self.normalize_y: + funcs = self._untransform_y(funcs) if len(funcs.shape) == 1: return funcs[None, :] diff --git a/smac/epm/gaussian_process_mcmc.py b/smac/epm/gaussian_process_mcmc.py index b1d7ccf35..e64abecc7 100644 --- a/smac/epm/gaussian_process_mcmc.py +++ b/smac/epm/gaussian_process_mcmc.py @@ -2,33 +2,36 @@ import logging import typing -import george import emcee import numpy as np +from lazy_import import lazy_callable +from smac.configspace import ConfigurationSpace from smac.epm.base_gp import BaseModel from smac.epm.gaussian_process import GaussianProcess -from smac.epm import normalization -from smac.epm.gp_base_prior import BasePrior logger = logging.getLogger(__name__) +Kernel = lazy_callable('skopt.learning.gaussian_process.kernels.Kernel') +GaussianProcessRegressor = lazy_callable( + 'skopt.learning.gaussian_process.GaussianProcessRegressor') class GaussianProcessMCMC(BaseModel): def __init__( self, + configspace: ConfigurationSpace, types: np.ndarray, bounds: typing.List[typing.Tuple[float, float]], - kernel: george.kernels.Kernel, - prior: BasePrior=None, - n_hypers: int=20, - chain_length: int=2000, - burnin_steps: int=2000, - normalize_output: bool=True, - normalize_input: bool=True, - rng: typing.Optional[np.random.RandomState]=None, - noise: int=-8, + seed: int, + kernel: Kernel, + n_mcmc_walkers: int = 20, + chain_length: int = 50, + burnin_steps: int = 50, + normalize_y: bool = True, + mcmc_sampler: str = 'emcee', + average_samples: bool = False, + **kwargs ): """ Gaussian process model. @@ -53,54 +56,50 @@ def __init__( have to pass np.array([2, 0]). Note that we count starting from 0. bounds : list Specifies the bounds for continuous features. + seed : int + Model seed. kernel : george kernel object Specifies the kernel that is used for all Gaussian Process - prior : prior object - Defines a prior for the hyperparameters of the GP. Make sure that - it implements the Prior interface. During MCMC sampling the - lnlikelihood is multiplied with the prior. - n_hypers : int + n_mcmc_walkers : int The number of hyperparameter samples. This also determines the number of walker for MCMC sampling as each walker will return one hyperparameter sample. chain_length : int - The length of the MCMC chain. We start n_hypers walker for + The length of the MCMC chain. We start n_mcmc_walkers walker for chain_length steps and we use the last sample in the chain as a hyperparameter sample. burnin_steps : int The number of burnin steps before the actual MCMC sampling starts. - normalize_output : bool + normalize_y : bool Zero mean unit variance normalization of the output values - normalize_input : bool - Normalize all inputs to be in [0, 1]. This is important to define good priors for the - length scales. + mcmc_sampler : str + Choose a self-tuning MCMC sampler. Can be either ``emcee`` or ``nuts``. + average_samples : bool + Average the sampled hyperparameters if ``True``, uses the samples independently if ``False``. + This is equivalent to the posterior mean as used by + Letham et al. (2018, http://lethalletham.com/ConstrainedBO.pdf). rng: np.random.RandomState Random number generator - noise : float - Noise term that is added to the diagonal of the covariance matrix - for the Cholesky decomposition. """ - super().__init__(types=types, bounds=bounds) - - if rng is None: - self.rng = np.random.RandomState(np.random.randint(0, 10000)) - else: - self.rng = rng + super().__init__(configspace=configspace, types=types, bounds=bounds, seed=seed, **kwargs) self.kernel = kernel - self.prior = prior - self.noise = noise - self.n_hypers = n_hypers + self.n_mcmc_walkers = n_mcmc_walkers self.chain_length = chain_length self.burned = False self.burnin_steps = burnin_steps self.models = [] - self.normalize_output = normalize_output - self.normalize_input = normalize_input - self.X = None - self.y = None + self.normalize_y = normalize_y + self.mcmc_sampler = mcmc_sampler + self.average_samples = average_samples + self.is_trained = False + self._set_has_conditions() + + # Internal statistics + self._n_ll_evals = 0 + def _train(self, X: np.ndarray, y: np.ndarray, do_optimize: bool=True): """ Performs MCMC sampling to sample hyperparameter configurations from the @@ -117,92 +116,165 @@ def _train(self, X: np.ndarray, y: np.ndarray, do_optimize: bool=True): If set to true we perform MCMC sampling otherwise we just use the hyperparameter specified in the kernel. """ - - if self.normalize_input: - # Normalize input to be in [0, 1] - self.X, self.lower, self.upper = normalization.zero_one_normalization(X, self.lower, self.upper) - else: - self.X = X - - if len(y.shape) > 1: - y = y.flatten() - if len(y) != len(X): - raise ValueError('Shape mismatch: %s vs %s' % (y.shape, X.shape)) - - if self.normalize_output: - # Normalize output to have zero mean and unit standard deviation - self.y, self.y_mean, self.y_std = normalization.zero_mean_unit_var_normalization(y) - if self.y_std == 0: - raise ValueError("Cannot normalize output. All targets have the same value") - else: - self.y = y - - # Use the mean of the data as mean for the GP - self.mean = np.mean(self.y, axis=0) - self.gp = george.GP(self.kernel, mean=self.mean) + X = self._impute_inactive(X) + if self.normalize_y: + # A note on normalization for the Gaussian process with MCMC: + # Scikit-learn uses a different "normalization" than we use in SMAC3. Scikit-learn normalizes the data to + # have zero mean, while we normalize it to have zero mean unit variance. To make sure the scikit-learn GP + # behaves the same when we use it directly or indirectly (through the gaussian_process.py file), we + # normalize the data here. Then, after the individual GPs are fit, we inject the statistics into them so + # they unnormalize the data at prediction time. + y = self._normalize_y(y) + + self.gp = GaussianProcessRegressor( + kernel=self.kernel, + normalize_y=False, + optimizer=None, + n_restarts_optimizer=-1, # Do not use scikit-learn's optimization routine + alpha=0, # Governed by the kernel + noise=None, + ) if do_optimize: - # We have one walker for each hyperparameter configuration - sampler = emcee.EnsembleSampler(self.n_hypers, - len(self.kernel) + 1, - self._loglikelihood) - sampler.random_state = self.rng.get_state() - # Do a burn-in in the first iteration - if not self.burned: - # Initialize the walkers by sampling from the prior - if self.prior is None: - self.p0 = self.rng.rand(self.n_hypers, len(self.kernel) + 1) - else: - self.p0 = self.prior.sample_from_prior(self.n_hypers) - # Run MCMC sampling - self.p0, _, _ = sampler.run_mcmc(self.p0, - self.burnin_steps, - rstate0=self.rng) - - self.burned = True - - # Start sampling - pos, _, _ = sampler.run_mcmc(self.p0, - self.chain_length, - rstate0=self.rng) - - # Save the current position, it will be the start point in - # the next iteration - self.p0 = pos + self.gp.fit(X, y) + self._all_priors = self._get_all_priors( + add_bound_priors=True, + add_soft_bounds=True if self.mcmc_sampler == 'nuts' else False, + ) - # Take the last samples from each walker - self.hypers = sampler.chain[:, -1] + if self.mcmc_sampler == 'emcee': + sampler = emcee.EnsembleSampler(self.n_mcmc_walkers, + len(self.kernel.theta), + self._ll) + sampler.random_state = self.rng.get_state() + # Do a burn-in in the first iteration + if not self.burned: + # Initialize the walkers by sampling from the prior + dim_samples = [] + for dim, prior in enumerate(self._all_priors): + # Always sample from the first prior + if isinstance(prior, list): + if len(prior) == 0: + prior = None + else: + prior = prior[0] + if prior is None: + raise NotImplementedError() + else: + dim_samples.append(prior.sample_from_prior(self.n_mcmc_walkers).flatten()) + self.p0 = np.vstack(dim_samples).transpose() + + # Run MCMC sampling + self.p0, _, _ = sampler.run_mcmc(self.p0, + self.burnin_steps, + rstate0=self.rng) + + self.burned = True + + # Start sampling & save the current position, it will be the start point in the next iteration + self.p0, _, _ = sampler.run_mcmc(self.p0, self.chain_length, rstate0=self.rng) + + # Take the last samples from each walker + self.hypers = sampler.chain[:, -1] + print('hypers', self.hypers.mean(axis=0), np.exp(self.hypers.mean(axis=0))) + elif self.mcmc_sampler == 'nuts': + # Originally published as: + # http://www.stat.columbia.edu/~gelman/research/published/nuts.pdf + # A good explanation of HMC: + # https://theclevermachine.wordpress.com/2012/11/18/mcmc-hamiltonian-monte-carlo-a-k-a-hybrid-monte-carlo/ + # A good explanation of HMC and NUTS can be found in: + # https://besjournals.onlinelibrary.wiley.com/doi/full/10.1111/2041-210X.12681 + + # Do not require the installation of NUTS for SMAC + # This requires NUTS from https://github.com/mfeurer/NUTS + import nuts.nuts + + # Perform initial fit to the data to obtain theta0 + if not self.burned: + theta0 = self.gp.kernel.theta + self.burned = True + else: + theta0 = self.p0 + samples, _, _ = nuts.nuts.nuts6( + f=self._ll_w_grad, + Madapt=self.burnin_steps, + M=self.chain_length, + theta0=theta0, + # Increasing this value results in longer running times + delta=0.5, + adapt_mass=False, + # Rather low max depth to keep the number of required gradient steps low + max_depth=10, + rng=self.rng, + ) + indices = [int(np.rint(ind)) for ind in np.linspace(start=0, stop=len(samples) - 1, num=10)] + self.hypers = samples[indices] + self.p0 = self.hypers.mean(axis=0) + print('hypers', 'log space', self.p0, 'regular space', np.exp(self.p0)) + else: + raise ValueError(self.mcmc_sampler) + + if self.average_samples: + self.hypers = [self.hypers.mean(axis=0)] else: - self.hypers = self.gp.kernel.get_parameter_vector().tolist() - self.hypers.append(self.noise) + self.hypers = self.gp.kernel.theta self.hypers = [self.hypers] self.models = [] for sample in self.hypers: + if (sample < -50).any(): + sample[sample < -50] = -50 + if (sample > 50).any(): + sample[sample > 50] = 50 + # Instantiate a GP for each hyperparameter configuration kernel = deepcopy(self.kernel) - kernel.set_parameter_vector(sample[:-1]) - noise = np.exp(sample[-1]) + kernel.theta = sample + model = GaussianProcess( + configspace=self.configspace, + types=self.types, + bounds=self.bounds, + kernel=kernel, + normalize_y=False, + seed=self.rng.randint(low=0, high=10000), + ) + try: + model._train(X, y, do_optimize=False) + self.models.append(model) + except np.linalg.LinAlgError: + pass + + if len(self.models) == 0: + kernel = deepcopy(self.kernel) + kernel.theta = self.p0 model = GaussianProcess( + configspace=self.configspace, types=self.types, bounds=self.bounds, kernel=kernel, - normalize_output=self.normalize_output, - normalize_input=self.normalize_input, - noise=noise, - rng=self.rng, + normalize_y=False, + seed=self.rng.randint(low=0, high=10000), ) model._train(X, y, do_optimize=False) self.models.append(model) + if self.normalize_y: + # Inject the normalization statistics into the individual models. Setting normalize_y to True makes the + # individual GPs unnormalize the data at predict time. + for model in self.models: + model.normalize_y = True + model.mean_y_ = self.mean_y_ + model.std_y_ = self.std_y_ + self.is_trained = True + print('#LL evaluations', self._n_ll_evals) - def _loglikelihood(self, theta: np.ndarray): + def _ll(self, theta: np.ndarray) -> float: """ - Return the loglikelihood (+ the prior) for a hyperparameter - configuration theta. + Returns the marginal log likelihood (+ the prior) for + a hyperparameter configuration theta. Parameters ---------- @@ -215,26 +287,79 @@ def _loglikelihood(self, theta: np.ndarray): float lnlikelihood + prior """ + self._n_ll_evals += 1 # Bound the hyperparameter space to keep things sane. Note all # hyperparameters live on a log scale - if np.any((-20 > theta) + (theta > 20)): + if (theta < -50).any(): + theta[theta < -50] = -50 + if (theta > 50).any(): + theta[theta > 50] = 50 + + try: + lml = self.gp.log_marginal_likelihood(theta) + except ValueError as e: return -np.inf - # The last entry is always the noise - sigma_2 = np.exp(theta[-1]) - # Update the kernel and compute the lnlikelihood. - self.gp.kernel.set_parameter_vector(theta[:-1]) + # Add prior + for dim, priors in enumerate(self._all_priors): + for prior in priors: + lml += prior.lnprob(theta[dim]) - try: - self.gp.compute(self.X, yerr=np.sqrt(sigma_2)) - except: + if not np.isfinite(lml): return -np.inf + else: + return lml + + def _ll_w_grad(self, theta: np.ndarray) -> typing.Tuple[float, np.ndarray]: + """ + Returns the marginal log likelihood (+ the prior) for + a hyperparameter configuration theta. + + Parameters + ---------- + theta : np.ndarray(H) + Hyperparameter vector. Note that all hyperparameter are + on a log scale. - if self.prior is not None: - return self.prior.lnprob(theta) + self.gp.lnlikelihood(self.y, quiet=True) + Returns + ---------- + float + lnlikelihood + prior + """ + self._n_ll_evals += 1 + + # Bound the hyperparameter space to keep things sane. Note all hyperparameters live on a log scale + if (theta < -50).any(): + theta[theta < -50] = -50 + if (theta > 50).any(): + theta[theta > 50] = 50 + + lml = 0 + grad = np.zeros(theta.shape) + + # Add prior + for dim, priors in enumerate(self._all_priors): + for prior in priors: + lml += prior.lnprob(theta[dim]) + grad[dim] += prior.gradient(theta[dim]) + + # Check if one of the priors is invalid, if so, no need to compute the log marginal likelihood + if lml < -1e24: + return -1e25, np.zeros(theta.shape) + + try: + lml_, grad_ = self.gp.log_marginal_likelihood(theta, eval_gradient=True) + lml += lml_ + grad += grad_ + except ValueError: + return -1e25, np.zeros(theta.shape) + + # We add a minus here because scipy is minimizing + if not np.isfinite(lml) or (~np.isfinite(grad)).any(): + return -1e25, np.zeros(theta.shape) else: - return self.gp.lnlikelihood(self.y, quiet=True) + return lml, grad def _predict(self, X_test: np.ndarray): r""" @@ -261,6 +386,8 @@ def _predict(self, X_test: np.ndarray): if not self.is_trained: raise Exception('Model has to be trained first!') + X = self._impute_inactive(X_test) + mu = np.zeros([len(self.models), X_test.shape[0]]) var = np.zeros([len(self.models), X_test.shape[0]]) for i, model in enumerate(self.models): diff --git a/smac/epm/gp_base_prior.py b/smac/epm/gp_base_prior.py index cc4cc3aa1..0f081c6d7 100644 --- a/smac/epm/gp_base_prior.py +++ b/smac/epm/gp_base_prior.py @@ -1,21 +1,27 @@ +import math +import warnings import numpy as np import scipy.stats as sps +from smac.utils.constants import VERY_SMALL_NUMBER -class BasePrior(object): - def __init__(self, rng: np.random.RandomState=None): +class Prior(object): + + def __init__(self, rng: np.random.RandomState = None): """ Abstract base class to define the interface for priors of GP hyperparameter. - This class is a verbatim copy of the implementation of RoBO: + This class is adapted from RoBO: Klein, A. and Falkner, S. and Mansur, N. and Hutter, F. RoBO: A Flexible and Robust Bayesian Optimization Framework in Python In: NIPS 2017 Bayesian Optimization Workshop + [16.04.2019]: Whenever lnprob or the gradient is computed for a scalar input, we use math.* rather than np.* + Parameters ---------- rng: np.random.RandomState @@ -27,26 +33,48 @@ def __init__(self, rng: np.random.RandomState=None): else: self.rng = rng - def lnprob(self, theta: np.ndarray): + def lnprob(self, theta: float) -> float: """ - Returns the log probability of theta. Note: theta should - be on a log scale. + Return the log probability of theta. + + Theta must be on a log scale! This method exponentiates theta and calls ``self._lnprob``. Parameters ---------- - theta : (D,) numpy array - A hyperparameter configuration in log space. + theta : float + Hyperparameter configuration in log space. Returns ------- float The log probability of theta """ - pass + return self._lnprob(np.exp(theta)) - def sample_from_prior(self, n_samples: int): + def _lnprob(self, theta: float) -> float: """ - Returns N samples from the prior. + Return the log probability of theta. + + Theta must be on the original scale. + + Parameters + ---------- + theta : float + Hyperparameter configuration on the original scale. + + Returns + ------- + float + The log probability of theta + """ + raise NotImplementedError() + + def sample_from_prior(self, n_samples: int) -> np.ndarray: + """ + Returns ``n_samples`` from the prior. + + All samples are on a log scale. This method calls ``self._sample_from_prior`` and applies a log transformation + to the obtained values. Parameters ---------- @@ -55,36 +83,80 @@ def sample_from_prior(self, n_samples: int): Returns ------- - (N, D) np.array - The samples from the prior. + np.ndarray """ - pass - def gradient(self, theta: np.ndarray): + if np.ndim(n_samples) != 0: + raise ValueError('argument n_samples needs to be a scalar (is %s)' % n_samples) + if n_samples <= 0: + raise ValueError('argument n_samples needs to be positive (is %d)' % n_samples) + + sample = np.log(self._sample_from_prior(n_samples=n_samples)) + + if np.any(~np.isfinite(sample)): + raise ValueError('Sample %s from prior %s contains infinite values!' % (sample, self)) + + return sample + + def _sample_from_prior(self, n_samples: int) -> np.ndarray: """ - Computes the gradient of the prior with - respect to theta. + Returns ``n_samples`` from the prior. + + All samples are on a original scale. Parameters ---------- - theta : (D,) numpy array + n_samples : int + The number of samples that will be drawn. + + Returns + ------- + np.ndarray + """ + raise NotImplementedError() + + def gradient(self, theta: float) -> float: + """ + Computes the gradient of the prior with respect to theta. + + Theta must be on the original scale. + + Parameters + ---------- + theta : float Hyperparameter configuration in log space Returns ------- - (D) np.array + float The gradient of the prior at theta. """ - pass + return self._gradient(np.exp(theta)) + def _gradient(self, theta: float) -> float: + """ + Computes the gradient of the prior with respect to theta. -class TophatPrior(BasePrior): + Parameters + ---------- + theta : float + Hyperparameter configuration in the original space space - def __init__(self, l_bound: float, u_bound: float, rng: np.random.RandomState=None): + Returns + ------- + float + The gradient of the prior at theta. + """ + raise NotImplementedError() + + +class TophatPrior(Prior): + + def __init__(self, lower_bound: float, upper_bound: float, rng: np.random.RandomState = None): """ Tophat prior as it used in the original spearmint code. - This class is a verbatim copy of the implementation of RoBO: + This class is adapted from RoBO: Klein, A. and Falkner, S. and Mansur, N. and Hutter, F. RoBO: A Flexible and Robust Bayesian Optimization Framework in Python @@ -92,10 +164,10 @@ def __init__(self, l_bound: float, u_bound: float, rng: np.random.RandomState=No Parameters ---------- - l_bound : float - Lower bound of the prior. Note the log scale. - u_bound : float - Upper bound of the prior. Note the log scale. + lower_bound : float + Lower bound of the prior. In original scale. + upper_bound : float + Upper bound of the prior. In original scale. rng: np.random.RandomState Random number generator """ @@ -103,36 +175,34 @@ def __init__(self, l_bound: float, u_bound: float, rng: np.random.RandomState=No self.rng = np.random.RandomState(np.random.randint(0, 10000)) else: self.rng = rng - self.min = l_bound - self.max = u_bound + self.min = lower_bound + self._log_min = np.log(lower_bound) + self.max = upper_bound + self._log_max = np.log(upper_bound) if not (self.max > self.min): - raise Exception("Upper bound of Tophat prior must be greater \ - than the lower bound!") + raise Exception("Upper bound of Tophat prior must be greater than the lower bound!") - def lnprob(self, theta: np.ndarray): + def _lnprob(self, theta: float) -> float: """ - Returns the log probability of theta. Note: theta should - be on a log scale. + Return the log probability of theta. Parameters ---------- - theta : (D,) numpy array - A hyperparameter configuration in log space. + theta : float + A hyperparameter configuration Returns ------- float - The log probability of theta """ - - if np.any(theta < self.min) or np.any(theta > self.max): + if theta < self.min or theta > self.max: return -np.inf else: return 0 - def sample_from_prior(self, n_samples: int): + def _sample_from_prior(self, n_samples: int) -> np.ndarray: """ - Returns N samples from the prior. + Return ``n_samples`` from the prior. Parameters ---------- @@ -141,21 +211,24 @@ def sample_from_prior(self, n_samples: int): Returns ------- - (N, D) np.array - The samples from the prior. + np.ndarray """ - p0 = self.min + self.rng.rand(n_samples) * (self.max - self.min) - return p0[:, np.newaxis] + if np.ndim(n_samples) != 0: + raise ValueError('argument n_samples needs to be a scalar (is %s)' % n_samples) + if n_samples <= 0: + raise ValueError('argument n_samples needs to be positive (is %d)' % n_samples) - def gradient(self, theta: np.ndarray): + p0 = np.exp(self.rng.uniform(low=self._log_min, high=self._log_max, size=(n_samples, ))) + return p0 + + def gradient(self, theta: float) -> float: """ - Computes the gradient of the prior with - respect to theta. + Computes the gradient of the prior with respect to theta. Parameters ---------- - theta : (D,) numpy array + theta : float Hyperparameter configuration in log space Returns @@ -164,16 +237,16 @@ def gradient(self, theta: np.ndarray): The gradient of the prior at theta. """ - return np.zeros([theta.shape[0]]) + return 0 -class HorseshoePrior(BasePrior): +class HorseshoePrior(Prior): def __init__(self, scale: float=0.1, rng: np.random.RandomState=None): """ Horseshoe Prior as it is used in spearmint - This class is a verbatim copy of the implementation of RoBO: + This class is adapted from RoBO: Klein, A. and Falkner, S. and Mansur, N. and Hutter, F. RoBO: A Flexible and Robust Bayesian Optimization Framework in Python @@ -182,8 +255,7 @@ def __init__(self, scale: float=0.1, rng: np.random.RandomState=None): Parameters ---------- scale: float - Scaling parameter. See below how it is influenced - the distribution. + Scaling parameter. See below how it is influencing the distribution. rng: np.random.RandomState Random number generator """ @@ -192,28 +264,35 @@ def __init__(self, scale: float=0.1, rng: np.random.RandomState=None): else: self.rng = rng self.scale = scale + self.scale_square = scale ** 2 - def lnprob(self, theta: np.ndarray): + def _lnprob(self, theta: float) -> float: """ - Returns the log probability of theta. Note: theta should - be on a log scale. + Return the log probability of theta. Parameters ---------- theta : (D,) numpy array - A hyperparameter configuration in log space. + A hyperparameter configuration Returns ------- float - The log probability of theta """ - # We computed it exactly as in the original spearmint code - if np.any(theta == 0.0): - return np.inf - return np.log(np.log(1 + 3.0 * (self.scale / np.exp(theta)) ** 2)) + # We computed it exactly as in the original spearmint code, they basically say that there's no analytical form + # of the horseshoe prior, but that the multiplier is bounded between 2 and 4 and that they used the middle + # See "The horseshoe estimator for sparse signals" by Carvalho, Poloson and Scott (2010), Equation 1. + # https://www.jstor.org/stable/25734098 + # Compared to the paper by Carvalho, there's a constant multiplicator missing + # Compared to Spearmint we first have to undo the log space transformation of the theta + # Note: "undo log space transformation" is done in parent class + if theta == 0: + return np.inf # POSITIVE infinity (this is the "spike") + else: + a = math.log(1 + 3.0 * (self.scale_square / theta**2)) + return math.log(a + VERY_SMALL_NUMBER) - def sample_from_prior(self, n_samples: int): + def _sample_from_prior(self, n_samples: int) -> np.ndarray: """ Returns N samples from the prior. @@ -224,16 +303,15 @@ def sample_from_prior(self, n_samples: int): Returns ------- - (N, D) np.array - The samples from the prior. + np.ndarray """ + # This is copied from RoBO - scale is most likely the tau parameter lamda = np.abs(self.rng.standard_cauchy(size=n_samples)) + p0 = np.abs(self.rng.randn() * lamda * self.scale) + return p0 - p0 = np.log(np.abs(self.rng.randn() * lamda * self.scale)) - return p0[:, np.newaxis] - - def gradient(self, theta: np.ndarray): + def _gradient(self, theta: float) -> float: """ Computes the gradient of the prior with respect to theta. @@ -241,25 +319,30 @@ def gradient(self, theta: np.ndarray): Parameters ---------- theta : (D,) numpy array - Hyperparameter configuration in log space + Hyperparameter configuration Returns ------- (D) np.array The gradient of the prior at theta. """ - a = -(6 * self.scale ** 2) - b = (3 * self.scale ** 2 + np.exp(2 * theta)) - b *= np.log(3 * self.scale ** 2 * np.exp(- 2 * theta) + 1) - return a / b + if theta == 0: + return np.inf # POSITIVE infinity (this is the "spike") + else: + a = -(6 * self.scale_square) + b = 3 * self.scale_square + theta**2 + b *= math.log(3 * self.scale_square * theta ** (-2) + 1) + b = max(b, 1e-14) + return a / b -class LognormalPrior(BasePrior): - def __init__(self, sigma: float, mean: float=0, rng: np.random.RandomState=None): +class LognormalPrior(Prior): + + def __init__(self, sigma: float, mean: float=0, rng: np.random.RandomState = None): """ Log normal prior - This class is a verbatim copy of the implementation of RoBO: + This class is adapted from RoBO: Klein, A. and Falkner, S. and Mansur, N. and Hutter, F. RoBO: A Flexible and Robust Bayesian Optimization Framework in Python @@ -280,28 +363,37 @@ def __init__(self, sigma: float, mean: float=0, rng: np.random.RandomState=None) else: self.rng = rng + if mean != 0: + raise NotImplementedError(mean) + self.sigma = sigma + self.sigma_square = sigma ** 2 self.mean = mean + self.sqrt_2_pi = np.sqrt(2 * np.pi) - def lnprob(self, theta: np.ndarray): + def _lnprob(self, theta: float) -> float: """ - Returns the log probability of theta. Note: theta should - be on a log scale. + Return the log probability of theta Parameters ---------- - theta : (D,) numpy array - A hyperparameter configuration in log space. + theta : float + A hyperparameter configuration Returns ------- float - The log probability of theta """ + if theta <= self.mean: + return -1e25 + else: + rval = ( + -(math.log(theta) - self.mean) ** 2 / (2 * self.sigma_square) + - math.log(self.sqrt_2_pi * self.sigma * theta) + ) + return rval - return sps.lognorm.logpdf(theta, self.sigma, loc=self.mean) - - def sample_from_prior(self, n_samples: int): + def _sample_from_prior(self, n_samples: int) -> np.ndarray: """ Returns N samples from the prior. @@ -312,16 +404,12 @@ def sample_from_prior(self, n_samples: int): Returns ------- - (N, D) np.array - The samples from the prior. + np.ndarray """ - p0 = self.rng.lognormal(mean=self.mean, - sigma=self.sigma, - size=n_samples) - return p0[:, np.newaxis] + return self.rng.lognormal(mean=self.mean, sigma=self.sigma, size=n_samples) - def gradient(self, theta: np.ndarray): + def _gradient(self, theta: float) -> float: """ Computes the gradient of the prior with respect to theta. @@ -336,57 +424,128 @@ def gradient(self, theta: np.ndarray): (D) np.array The gradient of the prior at theta. """ - pass + if theta <= 0: + return 0 + else: + # derivative of log(1 / (x * s^2 * sqrt(2 pi)) * exp( - 0.5 * (log(x ) / s^2))^2)) + # This is without the mean!!! + return -(self.sigma_square + math.log(theta)) / (self.sigma_square * (theta)) * theta + + +class SoftTopHatPrior(Prior): + def __init__(self, lower_bound=-10, upper_bound=10, exponent=2, rng: np.random.RandomState = None): + super().__init__(rng=rng) + + with warnings.catch_warnings(): + warnings.simplefilter('error') + self.lower_bound = lower_bound + try: + self._log_lower_bound = np.log(lower_bound) + except RuntimeWarning as w: + if 'invalid value encountered in log' in w.args[0]: + raise ValueError('Invalid lower bound %f (cannot compute log)' % lower_bound) + else: + raise w + self.upper_bound = upper_bound + try: + self._log_upper_bound = np.log(upper_bound) + except RuntimeWarning as w: + if 'invalid value encountered in log' in w.args[0]: + raise ValueError('Invalid lower bound %f (cannot compute log)' % lower_bound) + else: + raise w + + if exponent <= 0: + raise ValueError('Exponent cannot be less or equal than zero (but is %f)' % exponent) + self.exponent = exponent + + def lnprob(self, theta: float) -> float: + # We need to use lnprob here instead of _lnprob to have the squared function work in the logarithmic space, + # too. + if np.ndim(theta) == 0 or (np.ndim(theta) == 1 and len(theta) == 1): + if theta < self._log_lower_bound: + return - ((theta - self._log_lower_bound) ** self.exponent) + elif theta > self._log_upper_bound: + return - (self._log_upper_bound - theta) ** self.exponent + else: + return 0 + else: + raise NotImplementedError() + def _sample_from_prior(self, n_samples: int) -> np.ndarray: + """ + Returns N samples from the prior. + + Parameters + ---------- + n_samples : int + The number of samples that will be drawn. -class NormalPrior(BasePrior): - def __init__(self, sigma: float, mean: float=0, rng: np.random.RandomState=None): + Returns + ------- + np.ndarray """ - Normal prior - This class is a verbatim copy of the implementation of RoBO: + return np.exp(self.rng.uniform(self._log_lower_bound, self._log_upper_bound, size=(n_samples, ))) - Klein, A. and Falkner, S. and Mansur, N. and Hutter, F. - RoBO: A Flexible and Robust Bayesian Optimization Framework in Python - In: NIPS 2017 Bayesian Optimization Workshop + def gradient(self, theta: float) -> float: + if np.ndim(theta) == 0 or (np.ndim(theta) == 1 and len(theta) == 1): + if theta < self._log_lower_bound: + return - self.exponent * (theta - self._log_lower_bound) + elif theta > self._log_upper_bound: + return self.exponent * ( self._log_upper_bound - theta) + else: + return 0 + else: + raise NotImplementedError() + + def __repr__(self): + return 'SoftTopHatPrior(lower_bound=%f, upper_bound=%f)' % (self.lower_bound, self.upper_bound) + + +class GammaPrior(Prior): + + def __init__(self, a: float, scale: float, loc: float=0, rng: np.random.RandomState = None): + """ + Gamma prior + + f(x) = (x-loc)**(a-1) * e**(-(x-loc)) * (1/scale)**a / gamma(a) Parameters ---------- - sigma: float - Specifies the standard deviation of the normal - distribution. - mean: float - Specifies the mean of the normal distribution + a: float > 0 + shape parameter + scale: float > 0 + scale parameter (1/scale corresponds to parameter p in canonical form) + loc: float + mean parameter for the distribution rng: np.random.RandomState Random number generator """ - if rng is None: - self.rng = np.random.RandomState(np.random.randint(0, 10000)) - else: - self.rng = rng + super().__init__(rng=rng) - self.sigma = sigma - self.mean = mean + self.a = a + self.loc = loc + self.scale = scale - def lnprob(self, theta: np.ndarray): + def _lnprob(self, theta: float) -> float: """ - Returns the pdf of theta. Note: theta should - be on a log scale. + Returns the logpdf of theta. Parameters ---------- - theta : (D,) numpy array - A hyperparameter configuration in log space. + theta : float + Hyperparameter configuration Returns ------- float - The log probability of theta """ + if np.ndim(theta) != 0: + raise NotImplementedError() + return sps.gamma.logpdf(theta, a=self.a, scale=self.scale, loc=self.loc) - return sps.norm.pdf(theta, scale=self.sigma, loc=self.mean) - - def sample_from_prior(self, n_samples: int): + def _sample_from_prior(self, n_samples: int) -> np.ndarray: """ Returns N samples from the prior. @@ -397,29 +556,26 @@ def sample_from_prior(self, n_samples: int): Returns ------- - (N, D) np.array - The samples from the prior. + np.ndarray """ - p0 = self.rng.normal(loc=self.mean, - scale=self.sigma, - size=n_samples) - return p0[:, np.newaxis] + return self.rng.gamma(shape=self.a, scale=self.scale, size=n_samples) - def gradient(self, theta: np.ndarray): + def _gradient(self, theta: float) -> float: """ - Computes the gradient of the prior with - respect to theta. + As computed by Wolfram Alpha Parameters ---------- - theta : (D,) numpy array - Hyperparameter configuration in log space + theta: float + A hyperparameter configuration Returns ------- - (D) np.array - The gradient of the prior at theta. + float """ - return (1 / (self.sigma * np.sqrt(2 * np.pi))) *\ - (- theta / (self.sigma ** 2) * np.exp(- (theta ** 2) / (2 * self.sigma ** 2))) + if np.ndim(theta) == 0: + # Multiply by theta because of the chain rule... + return ((self.a - 1) / theta - (1 / self.scale)) * theta + else: + raise NotImplementedError() diff --git a/smac/epm/gp_default_priors.py b/smac/epm/gp_default_priors.py deleted file mode 100644 index 7325e3e81..000000000 --- a/smac/epm/gp_default_priors.py +++ /dev/null @@ -1,60 +0,0 @@ - -import numpy as np - -from smac.epm.gp_base_prior import BasePrior, TophatPrior, \ - LognormalPrior, HorseshoePrior - - -class DefaultPrior(BasePrior): - - def __init__(self, n_dims: int, rng: np.random.RandomState=None): - """ - This class is a verbatim copy of the implementation of RoBO: - - Klein, A. and Falkner, S. and Mansur, N. and Hutter, F. - RoBO: A Flexible and Robust Bayesian Optimization Framework in Python - In: NIPS 2017 Bayesian Optimization Workshop - """ - if rng is None: - self.rng = np.random.RandomState(np.random.randint(0, 10000)) - else: - self.rng = rng - - # The number of hyperparameters - self.n_dims = n_dims - - # Prior for the Matern52 lengthscales - self.tophat = TophatPrior(-10, 2, rng=self.rng) - - # Prior for the covariance amplitude - self.ln_prior = LognormalPrior(mean=0.0, sigma=1.0, rng=self.rng) - - # Prior for the noise - self.horseshoe = HorseshoePrior(scale=0.1, rng=self.rng) - - def lnprob(self, theta: np.ndarray): - lp = 0 - # Covariance amplitude - lp += self.ln_prior.lnprob(theta[0]) - # Lengthscales - lp += self.tophat.lnprob(theta[1:-1]) - # Noise - lp += self.horseshoe.lnprob(theta[-1]) - - return lp - - def sample_from_prior(self, n_samples: int): - p0 = np.zeros([n_samples, self.n_dims]) - # Covariance amplitude - p0[:, 0] = self.ln_prior.sample_from_prior(n_samples)[:, 0] - # Lengthscales - ls_sample = np.array([self.tophat.sample_from_prior(n_samples)[:, 0] - for _ in range(1, (self.n_dims - 1))]).T - p0[:, 1:(self.n_dims - 1)] = ls_sample - # Noise - p0[:, -1] = self.horseshoe.sample_from_prior(n_samples)[:, 0] - return p0 - - def gradient(self, theta: np.ndarray): - # TODO: Implement real gradient here - return np.zeros([theta.shape[0]]) diff --git a/smac/epm/gp_kernels.py b/smac/epm/gp_kernels.py new file mode 100644 index 000000000..6e28fb037 --- /dev/null +++ b/smac/epm/gp_kernels.py @@ -0,0 +1,658 @@ +from inspect import signature +import math +from typing import Optional + +import numpy as np +import sklearn.gaussian_process.kernels +import scipy.optimize +import scipy.spatial.distance +import scipy.special + +from lazy_import import lazy_module +kernels = lazy_module('skopt.learning.gaussian_process.kernels') + +# This file contains almost no type annotations to simplify comparing it to the original scikit-learn version! + + +def get_conditional_hyperparameters(X: np.ndarray, Y: Optional[np.ndarray]) -> np.ndarray: + # Taking care of conditional hyperparameters according to Levesque et al. + X_cond = X <= -1 + if Y is not None: + Y_cond = Y <= -1 + else: + Y_cond = X <= -1 + active = ~((np.expand_dims(X_cond, axis=1) != Y_cond).any(axis=2)) + return active + + +class MagicMixin: + + prior = None + + def __call__(self, X, Y=None, eval_gradient=False, active=None): + + if active is None and self.has_conditions: + if self.operate_on is None: + active = get_conditional_hyperparameters(X, Y) + else: + if Y is None: + active = get_conditional_hyperparameters(X[:, self.operate_on], None) + else: + active = get_conditional_hyperparameters(X[:, self.operate_on], Y[:, self.operate_on]) + + if self.operate_on is None: + rval = self._call(X, Y, eval_gradient, active) + else: + if Y is None: + rval = self._call( + X=X[:, self.operate_on].reshape([-1, self.len_active]), + Y=None, + eval_gradient=eval_gradient, + active=active, + ) + X = X[:, self.operate_on].reshape((-1, self.len_active)) + else: + rval = self._call( + X=X[:, self.operate_on].reshape([-1, self.len_active]), + Y=Y[:, self.operate_on].reshape([-1, self.len_active]), + eval_gradient=eval_gradient, + active=active, + ) + X = X[:, self.operate_on].reshape((-1, self.len_active)) + Y = Y[:, self.operate_on].reshape((-1, self.len_active)) + + return rval + + def __add__(self, b): + if not isinstance(b, kernels.Kernel): + return Sum(self, ConstantKernel(b)) + return Sum(self, b) + + def __radd__(self, b): + if not isinstance(b, kernels.Kernel): + return Sum(ConstantKernel(b), self) + return Sum(b, self) + + def __mul__(self, b): + if not isinstance(b, kernels.Kernel): + return Product(self, ConstantKernel(b)) + return Product(self, b) + + def __rmul__(self, b): + if not isinstance(b, kernels.Kernel): + return Product(ConstantKernel(b), self) + return Product(b, self) + + def _signature(self, func): + try: + sig = self._signature_cache.get(func) + except AttributeError: + self._signature_cache = dict() + sig = None + if sig is None: + sig = signature(func) + self._signature_cache[func] = sig + return sig + + def get_params(self, deep=True): + """Get parameters of this kernel. + + Parameters + ---------- + deep : boolean, optional + If True, will return the parameters for this estimator and + contained subobjects that are estimators. + + Returns + ------- + params : mapping of string to any + Parameter names mapped to their values. + """ + params = dict() + + try: + args = self._args_cache + except AttributeError: + tmp = super().get_params(deep) + args = list(tmp.keys()) + # Sum and Product do not clone the 'has_conditions' attribute by default. Instead of changing their + # get_params() method, we simply add the attribute here! + if 'has_conditions' not in args: + args.append('has_conditions') + self._args_cache = args + + for arg in args: + params[arg] = getattr(self, arg, None) + return params + + @property + def hyperparameters(self): + """Returns a list of all hyperparameter specifications.""" + try: + return self._hyperparameters_cache + except AttributeError: + pass + + r = super().hyperparameters + self._hyperparameters_cache = r + + return r + + @property + def n_dims(self): + """Returns the number of non-fixed hyperparameters of the kernel.""" + + try: + return self._n_dims_cache + except AttributeError: + pass + + self._n_dims_cache = super().n_dims + return self._n_dims_cache + + def clone_with_theta(self, theta): + """Returns a clone of self with given hyperparameters theta. + + Parameters + ---------- + theta : array, shape (n_dims,) + The hyperparameters + """ + self.theta = theta + return self + + def set_active_dims(self, operate_on=None): + """Sets dimensions this kernel should work on + + Parameters + ---------- + operate_on : None, list or array, shape (n_dims,) + """ + if operate_on is not None and type(operate_on) in (list, np.ndarray): + if not isinstance(operate_on, np.ndarray): + raise TypeError('argument operate_on needs to be of type np.ndarray, but is %s' % type(operate_on)) + if operate_on.dtype != np.int: + raise ValueError('dtype of argument operate_on needs to be np.int, but is %s' % operate_on.dtype) + self.operate_on = operate_on + self.len_active = len(operate_on) + else: + self.operate_on = None + self.len_active = None + + +class Sum(MagicMixin, kernels.Sum): + + def __init__(self, k1, k2, operate_on=None, has_conditions=False): + super(Sum, self).__init__(k1=k1, k2=k2) + self.set_active_dims(operate_on) + self.has_conditions = has_conditions + + def _call(self, X, Y=None, eval_gradient=False, active=None): + """Return the kernel k(X, Y) and optionally its gradient. + + Parameters + ---------- + X : array, shape (n_samples_X, n_features) + Left argument of the returned kernel k(X, Y) + + Y : array, shape (n_samples_Y, n_features), (optional, default=None) + Right argument of the returned kernel k(X, Y). If None, k(X, X) + if evaluated instead. + + eval_gradient : bool (optional, default=False) + Determines whether the gradient with respect to the kernel + hyperparameter is determined. + + active : np.ndarray (n_samples_X, n_features) (optional) + Boolean array specifying which hyperparameters are active. + + Returns + ------- + K : array, shape (n_samples_X, n_samples_Y) + Kernel k(X, Y) + + K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims) + The gradient of the kernel k(X, X) with respect to the + hyperparameter of the kernel. Only returned when eval_gradient + is True. + """ + if eval_gradient: + K1, K1_gradient = self.k1(X, Y, eval_gradient=True, active=active) + K2, K2_gradient = self.k2(X, Y, eval_gradient=True, active=active) + return K1 + K2, np.dstack((K1_gradient, K2_gradient)) + else: + return self.k1(X, Y, active=active) + self.k2(X, Y, active=active) + + +class Product(MagicMixin, kernels.Product): + + def __init__(self, k1, k2, operate_on=None, has_conditions=False): + super(Product, self).__init__(k1=k1, k2=k2) + self.set_active_dims(operate_on) + self.has_conditions = has_conditions + + def _call(self, X, Y=None, eval_gradient=False, active=None): + """Return the kernel k(X, Y) and optionally its gradient. + + Parameters + ---------- + X : array, shape (n_samples_X, n_features) + Left argument of the returned kernel k(X, Y) + + Y : array, shape (n_samples_Y, n_features), (optional, default=None) + Right argument of the returned kernel k(X, Y). If None, k(X, X) + if evaluated instead. + + eval_gradient : bool (optional, default=False) + Determines whether the gradient with respect to the kernel + hyperparameter is determined. + + active : np.ndarray (n_samples_X, n_features) (optional) + Boolean array specifying which hyperparameters are active. + + Returns + ------- + K : array, shape (n_samples_X, n_samples_Y) + Kernel k(X, Y) + + K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims) + The gradient of the kernel k(X, X) with respect to the + hyperparameter of the kernel. Only returned when eval_gradient + is True. + """ + if eval_gradient: + K1, K1_gradient = self.k1(X, Y, eval_gradient=True, active=active) + K2, K2_gradient = self.k2(X, Y, eval_gradient=True, active=active) + return K1 * K2, np.dstack((K1_gradient * K2[:, :, np.newaxis], + K2_gradient * K1[:, :, np.newaxis])) + else: + return self.k1(X, Y, active=active) * self.k2(X, Y, active=active) + + +class ConstantKernel(MagicMixin, kernels.ConstantKernel): + + def __init__( + self, + constant_value=1.0, + constant_value_bounds=(1e-5, 1e5), + operate_on=None, + prior=None, + has_conditions=False, + ): + super(ConstantKernel, self).__init__(constant_value=constant_value, constant_value_bounds=constant_value_bounds) + self.set_active_dims(operate_on) + self.prior = prior + self.has_conditions = has_conditions + + def _call(self, X, Y=None, eval_gradient=False, active=None): + """Return the kernel k(X, Y) and optionally its gradient. + + Parameters + ---------- + X : array, shape (n_samples_X, n_features) + Left argument of the returned kernel k(X, Y) + + Y : array, shape (n_samples_Y, n_features), (optional, default=None) + Right argument of the returned kernel k(X, Y). If None, k(X, X) + if evaluated instead. + + eval_gradient : bool (optional, default=False) + Determines whether the gradient with respect to the kernel + hyperparameter is determined. Only supported when Y is None. + + active : np.ndarray (n_samples_X, n_features) (optional) + Boolean array specifying which hyperparameters are active. + + Returns + ------- + K : array, shape (n_samples_X, n_samples_Y) + Kernel k(X, Y) + + K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims) + The gradient of the kernel k(X, X) with respect to the + hyperparameter of the kernel. Only returned when eval_gradient + is True. + """ + X = np.atleast_2d(X) + if Y is None: + Y = X + elif eval_gradient: + raise ValueError("Gradient can only be evaluated when Y is None.") + + K = np.full((X.shape[0], Y.shape[0]), self.constant_value, + dtype=np.array(self.constant_value).dtype) + if eval_gradient: + if not self.hyperparameter_constant_value.fixed: + return (K, np.full((X.shape[0], X.shape[0], 1), + self.constant_value, + dtype=np.array(self.constant_value).dtype)) + else: + return K, np.empty((X.shape[0], X.shape[0], 0)) + else: + return K + + +class Matern(MagicMixin, kernels.Matern): + + def __init__( + self, + length_scale=1.0, + length_scale_bounds=(1e-5, 1e5), + nu=1.5, + operate_on=None, + prior=None, + has_conditions=False + ): + super(Matern, self).__init__(length_scale=length_scale, length_scale_bounds=length_scale_bounds, nu=nu) + self.set_active_dims(operate_on) + self.prior = prior + self.has_conditions = has_conditions + + def _call(self, X, Y=None, eval_gradient=False, active=None): + """Return the kernel k(X, Y) and optionally its gradient. + Parameters + ---------- + X : array, shape (n_samples_X, n_features) + Left argument of the returned kernel k(X, Y) + Y : array, shape (n_samples_Y, n_features), (optional, default=None) + Right argument of the returned kernel k(X, Y). If None, k(X, X) + if evaluated instead. + eval_gradient : bool (optional, default=False) + Determines whether the gradient with respect to the kernel + hyperparameter is determined. Only supported when Y is None. + active : np.ndarray (n_samples_X, n_features) (optional) + Boolean array specifying which hyperparameters are active. + Returns + ------- + K : array, shape (n_samples_X, n_samples_Y) + Kernel k(X, Y) + K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims) + The gradient of the kernel k(X, X) with respect to the + hyperparameter of the kernel. Only returned when eval_gradient + is True. + """ + + X = np.atleast_2d(X) + length_scale = sklearn.gaussian_process.kernels._check_length_scale(X, self.length_scale) + + if Y is None: + dists = scipy.spatial.distance.pdist(X / length_scale, metric='euclidean') + else: + if eval_gradient: + raise ValueError( + "Gradient can only be evaluated when Y is None.") + dists = scipy.spatial.distance.cdist(X / length_scale, Y / length_scale, metric='euclidean') + + if self.nu == 0.5: + K = np.exp(-dists) + elif self.nu == 1.5: + K = dists * math.sqrt(3) + K = (1. + K) * np.exp(-K) + elif self.nu == 2.5: + K = dists * math.sqrt(5) + K = (1. + K + K ** 2 / 3.0) * np.exp(-K) + else: # general case; expensive to evaluate + K = dists + K[K == 0.0] += np.finfo(float).eps # strict zeros result in nan + tmp = (math.sqrt(2 * self.nu) * K) + K.fill((2 ** (1. - self.nu)) / scipy.special.gamma(self.nu)) + K *= tmp ** self.nu + K *= scipy.special.kv(self.nu, tmp) + + if Y is None: + # convert from upper-triangular matrix to square matrix + K = scipy.spatial.distance.squareform(K) + np.fill_diagonal(K, 1) + + if active is not None: + K = K * active + + if eval_gradient: + if self.hyperparameter_length_scale.fixed: + # Hyperparameter l kept fixed + K_gradient = np.empty((X.shape[0], X.shape[0], 0)) + return K, K_gradient + + # We need to recompute the pairwise dimension-wise distances + if self.anisotropic: + D = (X[:, np.newaxis, :] - X[np.newaxis, :, :]) ** 2 / (length_scale ** 2) + else: + D = scipy.spatial.distance.squareform(dists ** 2)[:, :, np.newaxis] + + if self.nu == 0.5: + K_gradient = K[..., np.newaxis] * D / np.sqrt(D.sum(2))[:, :, np.newaxis] + K_gradient[~np.isfinite(K_gradient)] = 0 + elif self.nu == 1.5: + K_gradient = 3 * D * np.exp(-np.sqrt(3 * D.sum(-1)))[..., np.newaxis] + elif self.nu == 2.5: + tmp = np.sqrt(5 * D.sum(-1))[..., np.newaxis] + K_gradient = 5.0 / 3.0 * D * (tmp + 1) * np.exp(-tmp) + else: + # original sklearn code would approximate gradient numerically, but this would violate our assumption + # that the kernel hyperparameters are not changed within __call__ + raise ValueError(self.nu) + + if not self.anisotropic: + return K, K_gradient[:, :].sum(-1)[:, :, np.newaxis] + else: + return K, K_gradient + else: + return K + + +class RBF(MagicMixin, kernels.RBF): + + def __init__( + self, + length_scale=1.0, + length_scale_bounds=(1e-5, 1e5), + operate_on=None, + prior=None, + has_conditions=False, + ): + super(RBF, self).__init__(length_scale=length_scale, length_scale_bounds=length_scale_bounds) + self.set_active_dims(operate_on) + self.prior = prior + self.has_conditions = has_conditions + + def _call(self, X, Y=None, eval_gradient=False, active=None): + """Return the kernel k(X, Y) and optionally its gradient. + Parameters + ---------- + X : array, shape (n_samples_X, n_features) + Left argument of the returned kernel k(X, Y) + Y : array, shape (n_samples_Y, n_features), (optional, default=None) + Right argument of the returned kernel k(X, Y). If None, k(X, X) + if evaluated instead. + eval_gradient : bool (optional, default=False) + Determines whether the gradient with respect to the kernel + hyperparameter is determined. Only supported when Y is None. + active : np.ndarray (n_samples_X, n_features) (optional) + Boolean array specifying which hyperparameters are active. + Returns + ------- + K : array, shape (n_samples_X, n_samples_Y) + Kernel k(X, Y) + K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims) + The gradient of the kernel k(X, X) with respect to the + hyperparameter of the kernel. Only returned when eval_gradient + is True. + """ + + X = np.atleast_2d(X) + length_scale = sklearn.gaussian_process.kernels._check_length_scale(X, self.length_scale) + + if Y is None: + dists = scipy.spatial.distance.pdist(X / length_scale, metric='sqeuclidean') + K = np.exp(-.5 * dists) + # convert from upper-triangular matrix to square matrix + K = scipy.spatial.distance.squareform(K) + np.fill_diagonal(K, 1) + else: + if eval_gradient: + raise ValueError( + "Gradient can only be evaluated when Y is None.") + dists = scipy.spatial.distance.cdist(X / length_scale, Y / length_scale, metric='sqeuclidean') + K = np.exp(-.5 * dists) + + if active is not None: + K = K * active + + if eval_gradient: + if self.hyperparameter_length_scale.fixed: + # Hyperparameter l kept fixed + return K, np.empty((X.shape[0], X.shape[0], 0)) + elif not self.anisotropic or length_scale.shape[0] == 1: + K_gradient = (K * scipy.spatial.distance.squareform(dists))[:, :, np.newaxis] + return K, K_gradient + elif self.anisotropic: + # We need to recompute the pairwise dimension-wise distances + K_gradient = (X[:, np.newaxis, :] - X[np.newaxis, :, :]) ** 2 \ + / (length_scale ** 2) + K_gradient *= K[..., np.newaxis] + return K, K_gradient + else: + return K + + +class WhiteKernel(MagicMixin, kernels.WhiteKernel): + + def __init__( + self, + noise_level=1.0, + noise_level_bounds=(1e-5, 1e5), + operate_on=None, + prior=None, + has_conditions=False, + ): + super(WhiteKernel, self).__init__(noise_level=noise_level, noise_level_bounds=noise_level_bounds) + self.set_active_dims(operate_on) + self.prior = prior + self.has_conditions = has_conditions + + def _call(self, X, Y=None, eval_gradient=False, active=None): + """Return the kernel k(X, Y) and optionally its gradient. + Parameters + ---------- + X : array, shape (n_samples_X, n_features) + Left argument of the returned kernel k(X, Y) + Y : array, shape (n_samples_Y, n_features), (optional, default=None) + Right argument of the returned kernel k(X, Y). If None, k(X, X) + if evaluated instead. + eval_gradient : bool (optional, default=False) + Determines whether the gradient with respect to the kernel + hyperparameter is determined. Only supported when Y is None. + active : np.ndarray (n_samples_X, n_features) (optional) + Boolean array specifying which hyperparameters are active. + Returns + ------- + K : array, shape (n_samples_X, n_samples_Y) + Kernel k(X, Y) + K_gradient : array (opt.), shape (n_samples_X, n_samples_X, n_dims) + The gradient of the kernel k(X, X) with respect to the + hyperparameter of the kernel. Only returned when eval_gradient + is True. + """ + + X = np.atleast_2d(X) + + if Y is not None and eval_gradient: + raise ValueError("Gradient can only be evaluated when Y is None.") + + if Y is None: + K = self.noise_level * np.eye(X.shape[0]) + + if active is not None: + K = K * active + + if eval_gradient: + if not self.hyperparameter_noise_level.fixed: + return (K, self.noise_level + * np.eye(X.shape[0])[:, :, np.newaxis]) + else: + return K, np.empty((X.shape[0], X.shape[0], 0)) + else: + return K + else: + return np.zeros((X.shape[0], Y.shape[0])) + + +class HammingKernel(MagicMixin, kernels.HammingKernel): + + def __init__( + self, + length_scale=1.0, + length_scale_bounds=(1e-5, 1e5), + operate_on=None, + prior=None, + has_conditions=False, + ): + super(HammingKernel, self).__init__(length_scale=length_scale, length_scale_bounds=length_scale_bounds) + self.set_active_dims(operate_on) + self.prior = prior + self.has_conditions = has_conditions + + def _call(self, X, Y=None, eval_gradient=False, active=None): + """Return the kernel k(X, Y) and optionally its gradient. + + Parameters + ---------- + X : [array-like, shape=(n_samples_X, n_features)] + Left argument of the returned kernel k(X, Y) + Y : [array-like, shape=(n_samples_Y, n_features) or None(default)] + Right argument of the returned kernel k(X, Y). If None, k(X, X) + if evaluated instead. + eval_gradient : [bool, False(default)] + Determines whether the gradient with respect to the kernel + hyperparameter is determined. Only supported when Y is None. + active : np.ndarray (n_samples_X, n_features) (optional) + Boolean array specifying which hyperparameters are active. + + Returns + ------- + K : [array-like, shape=(n_samples_X, n_samples_Y)] + Kernel k(X, Y) + + K_gradient : [array-like, shape=(n_samples_X, n_samples_X, n_dims)] + The gradient of the kernel k(X, X) with respect to the + hyperparameter of the kernel. Only returned when eval_gradient + is True. + + Note + ---- + Code partially copied from skopt (https://github.com/scikit-optimize). + Made small changes to only compute necessary values and use scikit-learn helper functions. + """ + + X = np.atleast_2d(X) + length_scale = sklearn.gaussian_process.kernels._check_length_scale(X, self.length_scale) + + if Y is None: + Y = X + elif eval_gradient: + raise ValueError("gradient can be evaluated only when Y != X") + else: + Y = np.atleast_2d(Y) + + indicator = np.expand_dims(X, axis=1) != Y + K = (-1/(2*length_scale**2) * indicator).sum(axis=2) + K = np.exp(K) + + if active is not None: + K = K * active + + if eval_gradient: + # dK / d theta = (dK / dl) * (dl / d theta) + # theta = log(l) => dl / d (theta) = e^theta = l + # dK / d theta = l * dK / dl + + # dK / dL computation + if np.iterable(length_scale) and length_scale.shape[0] > 1: + grad = (np.expand_dims(K, axis=-1) * np.array(indicator, dtype=np.float32)) + else: + grad = np.expand_dims(K * np.sum(indicator, axis=2), axis=-1) + + grad *= (1 / length_scale ** 3) + + return K, grad + return K diff --git a/smac/epm/normalization.py b/smac/epm/normalization.py deleted file mode 100644 index 7d84748c6..000000000 --- a/smac/epm/normalization.py +++ /dev/null @@ -1,32 +0,0 @@ -import numpy as np - - -def zero_one_normalization(X: np.ndarray, lower=None, upper=None): - - if lower is None: - lower = np.min(X, axis=0) - if upper is None: - upper = np.max(X, axis=0) - - X_normalized = np.true_divide((X - lower), (upper - lower)) - - return X_normalized, lower, upper - - -def zero_one_unnormalization(X_normalized: np.ndarray, lower, upper): - return lower + (upper - lower) * X_normalized - - -def zero_mean_unit_var_normalization(X: np.ndarray, mean=None, std=None): - if mean is None: - mean = np.mean(X, axis=0) - if std is None: - std = np.std(X, axis=0) - - X_normalized = (X - mean) / std - - return X_normalized, mean, std - - -def zero_mean_unit_var_unnormalization(X_normalized: np.ndarray, mean, std): - return X_normalized * std + mean diff --git a/smac/epm/random_epm.py b/smac/epm/random_epm.py index 3f7df9721..da0386a8d 100644 --- a/smac/epm/random_epm.py +++ b/smac/epm/random_epm.py @@ -12,25 +12,12 @@ class RandomEPM(AbstractEPM): - """EPM which returns random values on a call to ``fit``. + """EPM which returns random values on a call to ``fit``.""" - Attributes - ---------- - logger : logging.Logger - rng : np.random.RandomState - """ + def __init__(self, **kwargs): - def __init__(self, rng: np.random.RandomState, **kwargs): - """Constructor - - Parameters - ---------- - rng: np.random.RandomState - """ super().__init__(**kwargs) - - self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) - self.rng = rng + self.rng = np.random.RandomState(self.seed) def _train(self, X: np.ndarray, Y: np.ndarray, **kwargs): """ diff --git a/smac/epm/rf_with_instances.py b/smac/epm/rf_with_instances.py index 80a9470f3..7f75c1c3f 100644 --- a/smac/epm/rf_with_instances.py +++ b/smac/epm/rf_with_instances.py @@ -4,8 +4,9 @@ import numpy as np from pyrfr import regression -from smac.epm.base_epm import AbstractEPM -from smac.utils.constants import N_TREES +from smac.configspace import ConfigurationSpace +from smac.epm.base_rf import BaseModel +from smac.utils.constants import N_TREES, VERY_SMALL_NUMBER __author__ = "Aaron Klein" @@ -16,7 +17,7 @@ __version__ = "0.0.1" -class RandomForestWithInstances(AbstractEPM): +class RandomForestWithInstances(BaseModel): """Random forest that takes instance features into account. @@ -37,20 +38,24 @@ class RandomForestWithInstances(AbstractEPM): logger : logging.logger """ - def __init__(self, types: np.ndarray, - bounds: typing.List[typing.Tuple[float, float]], - log_y: bool=False, - num_trees: int=N_TREES, - do_bootstrapping: bool=True, - n_points_per_tree: int=-1, - ratio_features: float=5. / 6., - min_samples_split: int=3, - min_samples_leaf: int=3, - max_depth: int=2**20, - eps_purity: float=1e-8, - max_num_nodes: int=2**20, - seed: int=42, - **kwargs): + def __init__( + self, + configspace: ConfigurationSpace, + types: np.ndarray, + bounds: typing.List[typing.Tuple[float, float]], + seed: int, + log_y: bool = False, + num_trees: int = N_TREES, + do_bootstrapping: bool = True, + n_points_per_tree: int = -1, + ratio_features: float = 5. / 6., + min_samples_split: int = 3, + min_samples_leaf: int = 3, + max_depth: int = 2**20, + eps_purity: float = 1e-8, + max_num_nodes: int = 2**20, + **kwargs + ): """ Parameters ---------- @@ -62,6 +67,8 @@ def __init__(self, types: np.ndarray, have to pass np.array([2, 0]). Note that we count starting from 0. bounds : list Specifies the bounds for continuous features. + seed : int + The seed that is passed to the random_forest_run library. log_y: bool y values (passed to this RF) are expected to be log(y) transformed; this will be considered during predicting @@ -85,10 +92,8 @@ def __init__(self, types: np.ndarray, different max_num_nodes : int The maxmimum total number of nodes in a tree - seed : int - The seed that is passed to the random_forest_run library. """ - super().__init__(types, bounds, **kwargs) + super().__init__(configspace, types, bounds, seed, **kwargs) self.log_y = log_y self.rng = regression.default_random_engine(seed) @@ -112,8 +117,7 @@ def __init__(self, types: np.ndarray, # This list well be read out by save_iteration() in the solver self.hypers = [num_trees, max_num_nodes, do_bootstrapping, n_points_per_tree, ratio_features, min_samples_split, - min_samples_leaf, max_depth, eps_purity, seed] - self.seed = seed + min_samples_leaf, max_depth, eps_purity, self.seed] self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) @@ -125,14 +129,14 @@ def _train(self, X: np.ndarray, y: np.ndarray): ---------- X : np.ndarray [n_samples, n_features (config + instance features)] Input data points. - Y : np.ndarray [n_samples, ] + y : np.ndarray [n_samples, ] The corresponding target values. Returns ------- self """ - + X = self._impute_inactive(X) self.X = X self.y = y.flatten() @@ -196,21 +200,38 @@ def _predict(self, X: np.ndarray) -> typing.Tuple[np.ndarray, np.ndarray]: if X.shape[1] != self.types.shape[0]: raise ValueError('Rows in X should have %d entries but have %d!' % (self.types.shape[0], X.shape[1])) - means, vars_ = [], [] - for row_X in X: - if self.log_y: + X = self._impute_inactive(X) + + if self.log_y: + all_preds = [] + third_dimension = 0 + + # Gather data in a list of 2d arrays and get statistics about the required size of the 3d array + for row_X in X: preds_per_tree = self.rf.all_leaf_values(row_X) - means_per_tree = [] - for preds in preds_per_tree: - # within one tree, we want to use the - # arithmetic mean and not the geometric mean - means_per_tree.append(np.log(np.mean(np.exp(preds)))) - mean = np.mean(means_per_tree) - var = np.var(means_per_tree) # variance over trees as uncertainty estimate - else: - mean, var = self.rf.predict_mean_var(row_X) - means.append(mean) - vars_.append(var) + all_preds.append(preds_per_tree) + max_num_leaf_data = max(map(len, preds_per_tree)) + third_dimension = max(max_num_leaf_data, third_dimension) + + # Transform list of 2d arrays into a 3d array + preds_as_array = np.zeros((X.shape[0], self.rf_opts.num_trees, third_dimension)) * np.NaN + for i, preds_per_tree in enumerate(all_preds): + for j, pred in enumerate(preds_per_tree): + preds_as_array[i, j, :len(pred)] = pred + + # Do all necessary computation with vectorized functions + preds_as_array = np.log(np.nanmean(np.exp(preds_as_array), axis=2) + VERY_SMALL_NUMBER) + + # Compute the mean and the variance across the different trees + means = preds_as_array.mean(axis=1) + vars_ = preds_as_array.var(axis=1) + else: + means, vars_ = [], [] + for row_X in X: + mean_, var = self.rf.predict_mean_var(row_X) + means.append(mean_) + vars_.append(var) + means = np.array(means) vars_ = np.array(vars_) @@ -245,20 +266,22 @@ def predict_marginalized_over_instances(self, X: np.ndarray): if self.instance_features is None or \ len(self.instance_features) == 0: - mean, var = self.predict(X) + mean_, var = self.predict(X) var[var < self.var_threshold] = self.var_threshold var[np.isnan(var)] = self.var_threshold - return mean, var + return mean_, var if len(X.shape) != 2: raise ValueError( 'Expected 2d array, got %dd array!' % len(X.shape)) - if X.shape[1] != self.bounds.shape[0]: + if X.shape[1] != len(self.bounds): raise ValueError('Rows in X should have %d entries but have %d!' % - (self.bounds.shape[0], + (len(self.bounds), X.shape[1])) - mean = np.zeros(X.shape[0]) + X = self._impute_inactive(X) + + mean_ = np.zeros(X.shape[0]) var = np.zeros(X.shape[0]) for i, x in enumerate(X): @@ -287,11 +310,11 @@ def predict_marginalized_over_instances(self, X: np.ndarray): var_x = self.var_threshold var[i] = var_x - mean[i] = mean_x + mean_[i] = mean_x - if len(mean.shape) == 1: - mean = mean.reshape((-1, 1)) + if len(mean_.shape) == 1: + mean_ = mean_.reshape((-1, 1)) if len(var.shape) == 1: var = var.reshape((-1, 1)) - return mean, var + return mean_, var diff --git a/smac/epm/rf_with_instances_hpo.py b/smac/epm/rf_with_instances_hpo.py index 565502894..5c8533377 100644 --- a/smac/epm/rf_with_instances_hpo.py +++ b/smac/epm/rf_with_instances_hpo.py @@ -45,13 +45,14 @@ class RandomForestWithInstancesHPO(RandomForestWithInstances): def __init__( self, + configspace: ConfigurationSpace, types: np.ndarray, bounds: typing.List[typing.Tuple[float, float]], - log_y: bool=False, - bootstrap: bool=False, - n_iters: int=50, - n_splits: int=10, - seed: int=42, + seed: int, + log_y: bool = False, + bootstrap: bool = False, + n_iters: int = 50, + n_splits: int = 10, ): """Parameters ---------- @@ -76,9 +77,11 @@ def __init__( The seed that is passed to the random_forest_run library. """ super().__init__( - types, - bounds, - log_y, + configspace=configspace, + types=types, + seed=seed, + bounds=bounds, + log_y=log_y, num_trees=N_TREES, do_bootstrapping=bootstrap, n_points_per_tree=N_POINTS_PER_TREE, @@ -88,11 +91,8 @@ def __init__( max_depth=MAX_DEPTH, eps_purity=EPSILON_IMPURITY, max_num_nodes=MAX_NUM_NODES, - seed=seed, ) - self.types = types - self.bounds = bounds self.log_y = log_y self.n_iters = n_iters self.n_splits = n_splits @@ -136,6 +136,7 @@ def _train(self, X: np.ndarray, y: np.ndarray) -> 'RandomForestWithInstancesHPO' self """ + X = self._impute_inactive(X) self.X = X self.y = y.flatten() diff --git a/smac/epm/uncorrelated_mo_rf_with_instances.py b/smac/epm/uncorrelated_mo_rf_with_instances.py index 33f26cf0c..ef2a7a5c0 100644 --- a/smac/epm/uncorrelated_mo_rf_with_instances.py +++ b/smac/epm/uncorrelated_mo_rf_with_instances.py @@ -1,9 +1,10 @@ import numpy as np +from smac.configspace import ConfigurationSpace from smac.epm.base_epm import AbstractEPM from smac.epm.rf_with_instances import RandomForestWithInstances -from typing import List, Dict, Any, Optional +from typing import List, Dict, Any, Optional, Tuple class UncorrelatedMultiObjectiveRandomForestWithInstances(AbstractEPM): @@ -23,12 +24,14 @@ class UncorrelatedMultiObjectiveRandomForestWithInstances(AbstractEPM): """ def __init__( - self, - target_names: List[str], - bounds: List[float], - types: np.ndarray, - rf_kwargs: Optional[Dict[str, Any]]=None, - **kwargs + self, + target_names: List[str], + configspace: ConfigurationSpace, + bounds: List[Tuple[float, float]], + types: np.ndarray, + seed: int, + rf_kwargs: Optional[Dict[str, Any]]=None, + **kwargs ): """Constructor @@ -47,14 +50,15 @@ def __init__( kwargs See :class:`~smac.epm.rf_with_instances.RandomForestWithInstances` documentation. """ - super().__init__(bounds=bounds, types=types, **kwargs) + super().__init__(configspace=configspace, bounds=bounds, types=types, seed=seed, **kwargs) if rf_kwargs is None: rf_kwargs = {} self.target_names = target_names self.num_targets = len(self.target_names) - self.estimators = [RandomForestWithInstances(types, bounds, **rf_kwargs) - for i in range(self.num_targets)] + print(seed, rf_kwargs) + self.estimators = [RandomForestWithInstances(configspace, types, bounds, **rf_kwargs) + for _ in range(self.num_targets)] def _train(self, X: np.ndarray, Y: np.ndarray, **kwargs): """Trains the random forest on X and y. diff --git a/smac/utils/util_funcs.py b/smac/epm/util_funcs.py similarity index 74% rename from smac/utils/util_funcs.py rename to smac/epm/util_funcs.py index c9d7c3c5d..3887dbc07 100644 --- a/smac/utils/util_funcs.py +++ b/smac/epm/util_funcs.py @@ -16,36 +16,59 @@ def get_types(config_space, instance_features=None): bounds = [(np.nan, np.nan)]*types.shape[0] for i, param in enumerate(config_space.get_hyperparameters()): + parents = config_space.get_parents_of(param.name) + if len(parents) == 0: + can_be_inactive = False + else: + can_be_inactive = True + if isinstance(param, (CategoricalHyperparameter)): n_cats = len(param.choices) + if can_be_inactive: + n_cats = len(param.choices) + 1 types[i] = n_cats bounds[i] = (int(n_cats), np.nan) elif isinstance(param, (OrdinalHyperparameter)): n_cats = len(param.sequence) types[i] = 0 - bounds[i] = (0, int(n_cats) - 1) + if can_be_inactive: + bounds[i] = (0, int(n_cats)) + else: + bounds[i] = (0, int(n_cats) - 1) elif isinstance(param, Constant): - # for constants we simply set types to 0 - # which makes it a numerical parameter - types[i] = 0 - bounds[i] = (0, np.nan) + # for constants we simply set types to 0 which makes it a numerical + # parameter + if can_be_inactive: + bounds[i] = (2, np.nan) + types[i] = 2 + else: + bounds[i] = (0, np.nan) + types[i] = 0 # and we leave the bounds to be 0 for now - elif isinstance(param, UniformFloatHyperparameter): # Are sampled on the unit hypercube thus the bounds - # bounds[i] = (float(param.lower), float(param.upper)) # are always 0.0, 1.0 - bounds[i] = (0.0, 1.0) + elif isinstance(param, UniformFloatHyperparameter): + # Are sampled on the unit hypercube thus the bounds + # are always 0.0, 1.0 + if can_be_inactive: + bounds[i] = (-1.0, 1.0) + else: + bounds[i] = (0, 1.0) elif isinstance(param, UniformIntegerHyperparameter): - # bounds[i] = (int(param.lower), int(param.upper)) - bounds[i] = (0.0, 1.0) + if can_be_inactive: + bounds[i] = (-1.0, 1.0) + else: + bounds[i] = (0, 1.0) elif not isinstance(param, (UniformFloatHyperparameter, UniformIntegerHyperparameter, - OrdinalHyperparameter)): + OrdinalHyperparameter, + CategoricalHyperparameter)): raise TypeError("Unknown hyperparameter type %s" % type(param)) if instance_features is not None: types = np.hstack( - (types, np.zeros((instance_features.shape[1])))) + (types, np.zeros((instance_features.shape[1]))) + ) types = np.array(types, dtype=np.uint) bounds = np.array(bounds, dtype=object) diff --git a/smac/extras_require.json b/smac/extras_require.json new file mode 120000 index 000000000..ab1b36701 --- /dev/null +++ b/smac/extras_require.json @@ -0,0 +1 @@ +../extras_require.json \ No newline at end of file diff --git a/smac/facade/bogp_facade.py b/smac/facade/bogp_facade.py deleted file mode 100644 index 75ac45a2f..000000000 --- a/smac/facade/bogp_facade.py +++ /dev/null @@ -1,107 +0,0 @@ -import numpy as np -import george - -from smac.facade.smac_facade import SMAC -from smac.epm.gp_default_priors import DefaultPrior -from smac.epm.gaussian_process_mcmc import GaussianProcessMCMC, GaussianProcess -from smac.utils.util_funcs import get_types, get_rng - - -__author__ = "Marius Lindauer" -__copyright__ = "Copyright 2018, ML4AAD" -__license__ = "3-clause BSD" - - -class BOGP(SMAC): - """ - Facade to use BORF default mode - - see smac.facade.smac_Facade for API - This facade overwrites option available via the SMAC facade - - Attributes - ---------- - logger - stats : Stats - solver : SMBO - runhistory : RunHistory - List with information about previous runs - trajectory : list - List of all incumbents - - """ - - def __init__(self, model_type='gp_mcmc', **kwargs): - """ - Constructor - see ~smac.facade.smac_facade for documentation - """ - scenario = kwargs['scenario'] - if scenario.initial_incumbent not in ['LHD', 'FACTORIAL', 'SOBOL']: - scenario.initial_incumbent = 'SOBOL' - - if scenario.transform_y is 'NONE': - scenario.transform_y = "LOGS" - - if kwargs.get('model') is None: - _, rng = get_rng(rng=kwargs.get("rng", None), run_id=kwargs.get("run_id", None), logger=None) - - cov_amp = 2 - types, bounds = get_types(kwargs['scenario'].cs, instance_features=None) - n_dims = len(types) - - initial_ls = np.ones([n_dims]) - exp_kernel = george.kernels.Matern52Kernel(initial_ls, ndim=n_dims) - kernel = cov_amp * exp_kernel - - prior = DefaultPrior(len(kernel) + 1, rng=rng) - - n_hypers = 3 * len(kernel) - if n_hypers % 2 == 1: - n_hypers += 1 - - if model_type == "gp": - model = GaussianProcess( - types=types, - bounds=bounds, - kernel=kernel, - prior=prior, - rng=rng, - normalize_output=True, - normalize_input=True, - ) - elif model_type == "gp_mcmc": - model = GaussianProcessMCMC( - types=types, - bounds=bounds, - kernel=kernel, - prior=prior, - n_hypers=n_hypers, - chain_length=200, - burnin_steps=100, - normalize_input=True, - normalize_output=True, - rng=rng, - ) - kwargs['model'] = model - super().__init__(**kwargs) - - if self.solver.scenario.n_features > 0: - raise NotImplementedError("BOGP cannot handle instances") - - self.logger.info(self.__class__) - - self.solver.random_configuration_chooser.prob = 0.0 - - # only 1 configuration per SMBO iteration - self.solver.scenario.intensification_percentage = 1e-10 - self.solver.intensifier.min_chall = 1 - - # better improve acqusition function optimization - # 1. increase number of sls iterations - self.solver.acq_optimizer.n_sls_iterations = 100 - # 2. more randomly sampled configurations - self.solver.scenario.acq_opt_challengers = 1000 - - # activate predict incumbent - self.solver.predict_incumbent = True diff --git a/smac/facade/borf_facade.py b/smac/facade/borf_facade.py deleted file mode 100644 index 83058a4dd..000000000 --- a/smac/facade/borf_facade.py +++ /dev/null @@ -1,101 +0,0 @@ -import logging -import os -import typing - -import numpy as np - -from smac.facade.smac_facade import SMAC -from smac.optimizer.random_configuration_chooser import ChooserNoCoolDown, \ - RandomConfigurationChooser, ChooserCosineAnnealing, \ - ChooserProb -from smac.runhistory.runhistory2epm import AbstractRunHistory2EPM, \ - RunHistory2EPM4LogCost, RunHistory2EPM4Cost, \ - RunHistory2EPM4LogScaledCost, RunHistory2EPM4SqrtScaledCost, \ - RunHistory2EPM4InvScaledCost, RunHistory2EPM4ScaledCost -from smac.optimizer.acquisition import EI, LogEI, AbstractAcquisitionFunction -from smac.optimizer.ei_optimization import InterleavedLocalAndRandomSearch, \ - AcquisitionFunctionMaximizer -from smac.tae.execute_ta_run import StatusType -from smac.epm.rf_with_instances_hpo import RandomForestWithInstancesHPO -from smac.utils.util_funcs import get_types -from smac.utils.constants import MAXINT -from smac.initial_design.latin_hypercube_design import LHDesign -from smac.initial_design.factorial_design import FactorialInitialDesign -from smac.initial_design.sobol_design import SobolDesign - -__author__ = "Marius Lindauer" -__copyright__ = "Copyright 2018, ML4AAD" -__license__ = "3-clause BSD" - - -class BORF(SMAC): - """ - Facade to use BORF default mode - - see smac.facade.smac_Facade for API - This facade overwrites option available via the SMAC facade - - Attributes - ---------- - logger - stats : Stats - solver : SMBO - runhistory : RunHistory - List with information about previous runs - trajectory : list - List of all incumbents - - """ - - def __init__(self,**kwargs): - """ - Constructor - see ~smac.facade.smac_facade for docu - """ - - scenario = kwargs['scenario'] - - if scenario.initial_incumbent not in ['LHD', 'FACTORIAL', 'SOBOL']: - scenario.initial_incumbent = 'SOBOL' - - if scenario.transform_y is 'NONE': - scenario.transform_y = "LOGS" - #scenario.logy = True - - super().__init__(**kwargs) - self.logger.info(self.__class__) - - #== static RF settings - self.solver.model.rf_opts.num_trees = 10 - self.solver.model.rf_opts.do_bootstrapping = True - self.solver.model.rf_opts.tree_opts.max_features = self.solver.model.types.shape[0] - self.solver.model.rf_opts.tree_opts.min_samples_to_split = 2 - self.solver.model.rf_opts.tree_opts.min_samples_in_leaf = 1 - - # RF with HPO - #======================================================================= - # scenario = self.solver.scenario - # types, bounds = get_types(scenario.cs, scenario.feature_array) - # model = RandomForestWithInstancesHPO(types=types, - # bounds=bounds, - # seed=self.solver.rng.randint(MAXINT), - # log_y=scenario.logy) - # self.solver.model = model - #======================================================================= - - # assumes random chooser for random configs - self.solver.random_configuration_chooser.prob = 0.0 - - # only 1 configuration per SMBO iteration - self.solver.scenario.intensification_percentage = 1e-10 - self.solver.intensifier.min_chall = 1 - - # better improve acquisition function optimization - # 1. increase number of sls iterations - self.solver.acq_optimizer.n_sls_iterations = 100 - # 2. more randomly sampled configurations - self.solver.scenario.acq_opt_challengers = 10000 - - # activate predict incumbent - self.solver.predict_incumbent = True - \ No newline at end of file diff --git a/test/test_integration/__init__.py b/smac/facade/experimental/__init__.py similarity index 100% rename from test/test_integration/__init__.py rename to smac/facade/experimental/__init__.py diff --git a/smac/facade/epils_facade.py b/smac/facade/experimental/epils_facade.py similarity index 89% rename from smac/facade/epils_facade.py rename to smac/facade/experimental/epils_facade.py index b795cb671..64a515983 100644 --- a/smac/facade/epils_facade.py +++ b/smac/facade/experimental/epils_facade.py @@ -14,11 +14,8 @@ from smac.runhistory.runhistory2epm import AbstractRunHistory2EPM, \ RunHistory2EPM4LogCost, RunHistory2EPM4Cost from smac.initial_design.initial_design import InitialDesign -from smac.initial_design.default_configuration_design import \ - DefaultConfiguration -from smac.initial_design.random_configuration_design import RandomConfiguration -from smac.initial_design.multi_config_initial_design import \ - MultiConfigInitialDesign +from smac.initial_design.default_configuration_design import DefaultConfiguration +from smac.initial_design.random_configuration_design import RandomConfigurations from smac.intensification.intensification import Intensifier from smac.optimizer.epils import EPILS_Solver from smac.optimizer.objective import average_cost @@ -27,7 +24,7 @@ from smac.epm.rf_with_instances import RandomForestWithInstances from smac.epm.rfr_imputator import RFRImputator from smac.epm.base_epm import AbstractEPM -from smac.utils.util_funcs import get_types +from smac.epm.util_funcs import get_types from smac.utils.io.traj_logging import TrajLogger from smac.utils.constants import MAXINT from smac.utils.io.output_directory import create_output_directory @@ -141,16 +138,19 @@ def __init__(self, types, bounds = get_types(scenario.cs, scenario.feature_array) if model is None: model = RandomForestWithInstances( - types=types, bounds=bounds, - instance_features=scenario.feature_array, - seed=rng.randint(MAXINT), - pca_components=scenario.PCA_DIM, - num_trees=scenario.rf_num_trees, - do_bootstrapping=scenario.rf_do_bootstrapping, - ratio_features=scenario.rf_ratio_features, - min_samples_split=scenario.rf_min_samples_split, - min_samples_leaf=scenario.rf_min_samples_leaf, - max_depth=scenario.rf_max_depth) + configspace=scenario.cs, + types=types, + bounds=bounds, + instance_features=scenario.feature_array, + seed=rng.randint(MAXINT), + pca_components=scenario.PCA_DIM, + num_trees=scenario.rf_num_trees, + do_bootstrapping=scenario.rf_do_bootstrapping, + ratio_features=scenario.rf_ratio_features, + min_samples_split=scenario.rf_min_samples_split, + min_samples_leaf=scenario.rf_min_samples_leaf, + max_depth=scenario.rf_max_depth, + ) # initial acquisition function if acquisition_function is None: if scenario.run_obj == "runtime": @@ -243,28 +243,36 @@ def __init__(self, "Either use initial_design or initial_configurations; but not both") if initial_configurations is not None: - initial_design = MultiConfigInitialDesign(tae_runner=tae_runner, + initial_design = InitialDesign(tae_runner=tae_runner, + scenario=scenario, + stats=self.stats, + traj_logger=traj_logger, + runhistory=runhistory, + rng=rng, + configs=initial_configurations, + intensifier=intensifier, + aggregate_func=aggregate_func) + elif initial_design is None: + if scenario.initial_incumbent == "DEFAULT": + initial_design = DefaultConfiguration(tae_runner=tae_runner, scenario=scenario, stats=self.stats, traj_logger=traj_logger, runhistory=runhistory, rng=rng, - configs=initial_configurations, intensifier=intensifier, - aggregate_func=aggregate_func) - elif initial_design is None: - if scenario.initial_incumbent == "DEFAULT": - initial_design = DefaultConfiguration(tae_runner=tae_runner, + aggregate_func=aggregate_func, + max_config_fracs=0.0) + elif scenario.initial_incumbent == "RANDOM": + initial_design = RandomConfigurations(tae_runner=tae_runner, scenario=scenario, stats=self.stats, traj_logger=traj_logger, - rng=rng) - elif scenario.initial_incumbent == "RANDOM": - initial_design = RandomConfiguration(tae_runner=tae_runner, - scenario=scenario, - stats=self.stats, - traj_logger=traj_logger, - rng=rng) + runhistory=runhistory, + rng=rng, + intensifier=intensifier, + aggregate_func=aggregate_func, + max_config_fracs=0.0) else: raise ValueError("Don't know what kind of initial_incumbent " "'%s' is" % scenario.initial_incumbent) @@ -288,8 +296,7 @@ def __init__(self, # the RFRImputator will already get # log transform data from the runhistory cutoff = np.log(scenario.cutoff) - threshold = np.log(scenario.cutoff * - scenario.par_factor) + threshold = np.log(scenario.cutoff * scenario.par_factor) imputor = RFRImputator(rng=rng, cutoff=cutoff, @@ -307,8 +314,7 @@ def __init__(self, elif scenario.run_obj == 'quality': runhistory2epm = RunHistory2EPM4Cost(scenario=scenario, num_params=num_params, - success_states=[ - StatusType.SUCCESS, ], + success_states=[StatusType.SUCCESS, ], impute_censored_data=False, impute_state=None) else: diff --git a/smac/facade/hydra_facade.py b/smac/facade/experimental/hydra_facade.py similarity index 90% rename from smac/facade/hydra_facade.py rename to smac/facade/experimental/hydra_facade.py index 6583e55a7..58f852858 100644 --- a/smac/facade/hydra_facade.py +++ b/smac/facade/experimental/hydra_facade.py @@ -7,7 +7,6 @@ from collections import defaultdict import pickle -from functools import partial import numpy as np @@ -17,12 +16,12 @@ from smac.tae.execute_ta_run_hydra import ExecuteTARunOld from smac.tae.execute_ta_run_hydra import ExecuteTARun from smac.scenario.scenario import Scenario -from smac.facade.smac_facade import SMAC -from smac.facade.psmac_facade import PSMAC +from smac.facade.smac_ac_facade import SMAC4AC +from smac.facade.experimental.psmac_facade import PSMAC from smac.utils.io.output_directory import create_output_directory from smac.runhistory.runhistory import RunHistory from smac.optimizer.objective import average_cost -from smac.utils.util_funcs import get_rng +from smac.epm.util_funcs import get_rng from smac.utils.constants import MAXINT from smac.optimizer.pSMAC import read @@ -52,12 +51,13 @@ class Hydra(object): def __init__(self, scenario: typing.Type[Scenario], n_iterations: int, - val_set: str='train', - incs_per_round: int=1, - n_optimizers: int=1, - rng: typing.Optional[typing.Union[np.random.RandomState, int]]=None, - run_id: int=1, - tae: typing.Type[ExecuteTARun]=ExecuteTARunOld, + val_set: str = 'train', + incs_per_round: int = 1, + n_optimizers: int = 1, + rng: typing.Optional[typing.Union[np.random.RandomState, int]] = None, + run_id: int = 1, + tae: typing.Type[ExecuteTARun] = ExecuteTARunOld, + tae_kwargs: typing.Union[dict, None] = None, **kwargs): """ Constructor @@ -82,6 +82,8 @@ def __init__(self, run_id for this hydra run tae: ExecuteTARun Target Algorithm Runner (supports old and aclib format as well as AbstractTAFunc) + tae_kwargs: Optional[dict] + arguments passed to constructor of '~tae' """ self.logger = logging.getLogger( @@ -97,7 +99,7 @@ def __init__(self, self.portfolio = None self.rh = RunHistory(average_cost) self._tae = tae - self.tae = tae(ta=self.scenario.ta, run_obj=self.scenario.run_obj) + self._tae_kwargs = tae_kwargs if incs_per_round <= 0: self.logger.warning('Invalid value in %s: %d. Setting to 1', 'incs_per_round', incs_per_round) self.incs_per_round = max(incs_per_round, 1) @@ -172,16 +174,27 @@ def optimize(self) -> typing.List[Configuration]: scen.output_dir_for_this_run = None scen.output_dir = None # parent process SMAC only used for validation purposes - self.solver = SMAC(scenario=scen, tae_runner=self.tae, rng=self.rng, run_id=self.run_id, **self.kwargs) + self.solver = SMAC4AC(scenario=scen, tae_runner=self._tae, rng=self.rng, run_id=self.run_id, **self.kwargs) for i in range(self.n_iterations): self.logger.info("="*120) self.logger.info("Hydra Iteration: %d", (i + 1)) + if i == 0: + tae = self._tae + tae_kwargs = self._tae_kwargs + else: + tae = ExecuteTARunHydra + if self._tae_kwargs: + tae_kwargs = self._tae_kwargs + else: + tae_kwargs = {} + tae_kwargs['cost_oracle'] = self.cost_per_inst self.optimizer = PSMAC( scenario=self.scenario, run_id=self.run_id, rng=self.rng, - tae=self._tae if i == 0 else partial(ExecuteTARunHydra, cost_oracle=self.cost_per_inst), + tae=tae, + tae_kwargs=tae_kwargs, shared_model=False, validate=True if self.val_set else False, n_optimizers=self.n_optimizers, @@ -211,10 +224,6 @@ def optimize(self) -> typing.List[Configuration]: portfolio_cost = cur_portfolio_cost self.logger.info("Current pertfolio cost: %f", portfolio_cost) - # modify TAE such that it return oracle performance - self.tae = ExecuteTARunHydra(ta=self.scenario.ta, run_obj=self.scenario.run_obj, - cost_oracle=self.cost_per_inst, tae=self._tae) - self.scenario.output_dir = os.path.join(self.top_dir, "psmac3-output_%s" % ( datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S_%f'))) self.output_dir = create_output_directory(self.scenario, run_id=self.run_id, logger=self.logger) diff --git a/smac/facade/psmac_facade.py b/smac/facade/experimental/psmac_facade.py similarity index 78% rename from smac/facade/psmac_facade.py rename to smac/facade/experimental/psmac_facade.py index 0f627d739..f83249682 100644 --- a/smac/facade/psmac_facade.py +++ b/smac/facade/experimental/psmac_facade.py @@ -5,9 +5,7 @@ import typing import copy -import pickle -import multiprocessing - +import joblib import numpy as np from ConfigSpace.configuration_space import Configuration @@ -15,12 +13,12 @@ from smac.tae.execute_ta_run_hydra import ExecuteTARunOld from smac.tae.execute_ta_run_hydra import ExecuteTARun from smac.scenario.scenario import Scenario -from smac.facade.smac_facade import SMAC +from smac.facade.smac_ac_facade import SMAC4AC from smac.optimizer.pSMAC import read from smac.utils.io.output_directory import create_output_directory from smac.runhistory.runhistory import RunHistory from smac.optimizer.objective import average_cost -from smac.utils.util_funcs import get_rng +from smac.epm.util_funcs import get_rng from smac.utils.constants import MAXINT __author__ = "Andre Biedenkapp" @@ -28,9 +26,9 @@ __license__ = "3-clause BSD" -def optimize(queue: multiprocessing.Queue, - scenario: typing.Type[Scenario], +def optimize(scenario: typing.Type[Scenario], tae: typing.Type[ExecuteTARun], + tae_kwargs: typing.Dict, rng: typing.Union[np.random.RandomState, int], output_dir: str, **kwargs) -> Configuration: @@ -39,12 +37,12 @@ def optimize(queue: multiprocessing.Queue, Parameters ---------- - queue: multiprocessing.Queue - incumbents (Configurations) of each SMAC call will be pushed to this queue scenario: Scenario smac.Scenario to initialize SMAC tae: ExecuteTARun Target Algorithm Runner (supports old and aclib format) + tae_runner_kwargs: Optional[dict] + arguments passed to constructor of '~tae' rng: int/np.random.RandomState The randomState/seed to pass to each smac run output_dir: str @@ -56,8 +54,7 @@ def optimize(queue: multiprocessing.Queue, The incumbent configuration of this run """ - tae = tae(ta=scenario.ta, run_obj=scenario.run_obj) - solver = SMAC(scenario=scenario, tae_runner=tae, rng=rng, **kwargs) + solver = SMAC4AC(scenario=scenario, tae_runner=tae, tae_runner_kwargs=tae_kwargs, rng=rng, **kwargs) solver.stats.start_timing() solver.stats.print_stats() @@ -68,8 +65,6 @@ def optimize(queue: multiprocessing.Queue, solver.solver.runhistory.save_json( fn=os.path.join(solver.output_dir, "runhistory.json") ) - queue.put((incumbent, rng)) - queue.close() return incumbent @@ -96,11 +91,12 @@ def __init__(self, rng: typing.Optional[typing.Union[np.random.RandomState, int]] = None, run_id: int = 1, tae: typing.Type[ExecuteTARun] = ExecuteTARunOld, + tae_kwargs: typing.Union[dict, None] = None, shared_model: bool = True, validate: bool = True, n_optimizers: int = 2, val_set: typing.Union[typing.List[str], None] = None, - n_incs: int=1, + n_incs: int = 1, **kwargs): """ Constructor @@ -117,6 +113,8 @@ def __init__(self, run_id for this hydra run tae: ExecuteTARun Target Algorithm Runner (supports old and aclib format as well as AbstractTAFunc) + tae_kwargs: Optional[dict] + arguments passed to constructor of '~tae' shared_model: bool Flag to indicate whether information is shared between SMAC runs or not validate: bool / None @@ -137,7 +135,7 @@ def __init__(self, self.output_dir = None self.rh = RunHistory(average_cost) self._tae = tae - self.tae = tae(ta=self.scenario.ta, run_obj=self.scenario.run_obj) + self._tae_kwargs = tae_kwargs if n_optimizers <= 1: self.logger.warning('Invalid value in %s: %d. Setting to 2', 'n_optimizers', n_optimizers) self.n_optimizers = max(n_optimizers, 2) @@ -177,42 +175,25 @@ def optimize(self): self.logger.info("+" * 120) self.logger.info("PSMAC run") - # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Multiprocessing part start ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # - q = multiprocessing.Queue() - procs = [] - for p in range(self.n_optimizers): - proc = multiprocessing.Process(target=optimize, - args=( - q, # Output queue - self.scenario, # Scenario object - self._tae, # type of tae to run target with - p, # process_id (used in output folder name) - self.output_dir, # directory to create outputs in - ), - kwargs=self.kwargs) - proc.start() - procs.append(proc) - for proc in procs: - proc.join() - incs = np.empty((self.n_optimizers,), dtype=Configuration) - pids = np.empty((self.n_optimizers,), dtype=int) - idx = 0 - while not q.empty(): - conf, pid = q.get_nowait() - incs[idx] = conf - pids[idx] = pid - idx += 1 - self.logger.info('Loading all runhistories') - read(self.rh, self.scenario.input_psmac_dirs, self.scenario.cs, self.logger) - q.close() - # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Multiprocessing part end ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # + + incs = joblib.Parallel(n_jobs=self.n_optimizers)( + joblib.delayed(optimize)( + self.scenario, # Scenario object + self._tae, # type of tae to run target with + self._tae_kwargs, + p, # seed for the rng/run_id + self.output_dir, # directory to create outputs in + **self.kwargs + ) for p in range(self.n_optimizers) + ) + if self.n_optimizers == self.n_incs: # no validation necessary just return all incumbents return incs else: _, val_ids, _, est_ids = self.get_best_incumbents_ids(incs) # determine the best incumbents if val_ids: - return incs[val_ids] - return incs[est_ids] + return [inc for i, inc in enumerate(incs) if i in val_ids] + return [inc for i, inc in enumerate(incs) if i in est_ids] def get_best_incumbents_ids(self, incs: typing.List[Configuration]): """ @@ -271,7 +252,7 @@ def validate_incs(self, incs: np.ndarray): """ Validation """ - solver = SMAC(scenario=self.scenario, tae_runner=self.tae, rng=self.rng, run_id=MAXINT, **self.kwargs) + solver = SMAC4AC(scenario=self.scenario, rng=self.rng, run_id=MAXINT, **self.kwargs) self.logger.info('*' * 120) self.logger.info('Validating') new_rh = solver.validate(config_mode=incs, diff --git a/smac/facade/func_facade.py b/smac/facade/func_facade.py index 0e665ccf9..61d06805f 100644 --- a/smac/facade/func_facade.py +++ b/smac/facade/func_facade.py @@ -3,7 +3,7 @@ import numpy as np -from smac.facade.borf_facade import BORF +from smac.facade.smac_hpo_facade import SMAC4HPO from smac.scenario.scenario import Scenario from smac.configspace import ConfigurationSpace from smac.runhistory.runhistory import RunKey @@ -18,15 +18,15 @@ def fmin_smac(func: typing.Callable, x0: typing.List[float], - bounds: typing.List[typing.List[float]], - maxfun: int=-1, - rng: np.random.RandomState=None, - scenario_args: typing.Mapping[str, typing.Any]=None, + bounds: typing.List[typing.Iterable[float]], + maxfun: int = -1, + rng: typing.Union[np.random.RandomState, int] = None, + scenario_args: typing.Mapping[str, typing.Any] = None, **kwargs): """ - Minimize a function func using the BORF facade + Minimize a function func using the SMAC4HPO facade (i.e., a modified version of SMAC). - This function is a convenience wrapper for the BORF class. + This function is a convenience wrapper for the SMAC4HPO class. Parameters ---------- @@ -54,7 +54,7 @@ def fmin_smac(func: typing.Callable, Estimated position of the minimum. f : float Value of `func` at the minimum. - s : :class:`smac.facade.smac_facade.SMAC` + s : :class:`smac.facade.smac_hpo_facade.SMAC4HPO` SMAC objects which enables the user to get e.g., the trajectory and runhistory. @@ -72,9 +72,6 @@ def fmin_smac(func: typing.Callable, default_value=x0[idx]) cs.add_hyperparameter(parameter) - # Create target algorithm runner - ta = ExecuteTAFuncArray(ta=func) - # create scenario scenario_dict = { "run_obj": "quality", @@ -90,7 +87,14 @@ def fmin_smac(func: typing.Callable, scenario_dict["runcount_limit"] = maxfun scenario = Scenario(scenario_dict) - smac = BORF(scenario=scenario, tae_runner=ta, rng=rng, **kwargs) + smac = SMAC4HPO( + scenario=scenario, + tae_runner=ExecuteTAFuncArray, + tae_runner_kwargs={'ta': func}, + rng=rng, + **kwargs + ) + smac.logger = logging.getLogger(smac.__module__ + "." + smac.__class__.__name__) incumbent = smac.optimize() config_id = smac.solver.runhistory.config_ids[incumbent] diff --git a/smac/facade/roar_facade.py b/smac/facade/roar_facade.py index e983c0658..ba644c1e6 100644 --- a/smac/facade/roar_facade.py +++ b/smac/facade/roar_facade.py @@ -3,25 +3,24 @@ import numpy as np -from smac.optimizer.objective import average_cost +from smac.configspace import Configuration +from smac.epm.random_epm import RandomEPM +from smac.facade.smac_ac_facade import SMAC4AC +from smac.initial_design.initial_design import InitialDesign +from smac.intensification.intensification import Intensifier from smac.optimizer.ei_optimization import RandomSearch -from smac.tae.execute_ta_run import StatusType, ExecuteTARun -from smac.stats.stats import Stats -from smac.scenario.scenario import Scenario from smac.runhistory.runhistory import RunHistory from smac.runhistory.runhistory2epm import RunHistory2EPM4Cost -from smac.initial_design.initial_design import InitialDesign -from smac.intensification.intensification import Intensifier -from smac.facade.smac_facade import SMAC -from smac.configspace import Configuration -from smac.utils.util_funcs import get_rng +from smac.stats.stats import Stats +from smac.scenario.scenario import Scenario +from smac.tae.execute_ta_run import ExecuteTARun __author__ = "Marius Lindauer" __copyright__ = "Copyright 2016, ML4AAD" __license__ = "3-clause BSD" -class ROAR(SMAC): +class ROAR(SMAC4AC): """ Facade to use ROAR mode @@ -78,31 +77,7 @@ def __init__(self, """ self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) - # initial random number generator - _, rng = get_rng(rng=rng, logger=self.logger) - - # initial conversion of runhistory into EPM data - # since ROAR does not really use it the converted data - # we simply use a cheap RunHistory2EPM here - num_params = len(scenario.cs.get_hyperparameters()) - runhistory2epm = RunHistory2EPM4Cost( - scenario=scenario, num_params=num_params, - success_states=[StatusType.SUCCESS, ], - impute_censored_data=False, impute_state=None) - - aggregate_func = average_cost - # initialize empty runhistory - if runhistory is None: - runhistory = RunHistory(aggregate_func=aggregate_func) - # inject aggr_func if necessary - if runhistory.aggregate_func is None: - runhistory.aggregate_func = aggregate_func - - self.stats = Stats(scenario) - rs = RandomSearch( - acquisition_function=None, - config_space=scenario.cs, - ) + scenario.acq_opt_challengers = 1 # use SMAC facade super().__init__( @@ -110,11 +85,12 @@ def __init__(self, tae_runner=tae_runner, runhistory=runhistory, intensifier=intensifier, - runhistory2epm=runhistory2epm, + runhistory2epm=RunHistory2EPM4Cost, initial_design=initial_design, initial_configurations=initial_configurations, - stats=stats, - rng=rng, run_id=run_id, - acquisition_function_optimizer=rs, + acquisition_function_optimizer=RandomSearch, + model=RandomEPM, + rng=rng, + stats=stats ) diff --git a/smac/facade/smac_facade.py b/smac/facade/smac_ac_facade.py similarity index 50% rename from smac/facade/smac_facade.py rename to smac/facade/smac_ac_facade.py index 98dcd2e17..228d73077 100644 --- a/smac/facade/smac_facade.py +++ b/smac/facade/smac_ac_facade.py @@ -1,6 +1,7 @@ +import inspect import logging import os -import typing +from typing import List, Union, Optional, Type, Callable import numpy as np @@ -21,43 +22,39 @@ from smac.initial_design.initial_design import InitialDesign from smac.initial_design.default_configuration_design import \ DefaultConfiguration -from smac.initial_design.random_configuration_design import RandomConfiguration +from smac.initial_design.random_configuration_design import RandomConfigurations from smac.initial_design.latin_hypercube_design import LHDesign from smac.initial_design.factorial_design import FactorialInitialDesign -from smac.initial_design.sobol_design import SobolDesign -from smac.initial_design.multi_config_initial_design import \ - MultiConfigInitialDesign +from smac.initial_design.sobol_design import SobolDesign + # intensification from smac.intensification.intensification import Intensifier # optimizer from smac.optimizer.smbo import SMBO from smac.optimizer.objective import average_cost -from smac.optimizer.acquisition import EI, LogEI, AbstractAcquisitionFunction +from smac.optimizer.acquisition import EI, LogEI, AbstractAcquisitionFunction, IntegratedAcquisitionFunction from smac.optimizer.ei_optimization import InterleavedLocalAndRandomSearch, \ AcquisitionFunctionMaximizer -from smac.optimizer.random_configuration_chooser import ChooserNoCoolDown, \ - RandomConfigurationChooser, ChooserCosineAnnealing, ChooserProb +from smac.optimizer.random_configuration_chooser import RandomConfigurationChooser, ChooserProb # epm from smac.epm.rf_with_instances import RandomForestWithInstances from smac.epm.rfr_imputator import RFRImputator from smac.epm.base_epm import AbstractEPM +from smac.epm.util_funcs import get_types, get_rng # utils -from smac.utils.util_funcs import get_types from smac.utils.io.traj_logging import TrajLogger -from smac.utils.constants import MAXINT, N_TREES -from smac.utils.util_funcs import get_rng +from smac.utils.constants import MAXINT from smac.utils.io.output_directory import create_output_directory from smac.configspace import Configuration - __author__ = "Marius Lindauer" __copyright__ = "Copyright 2018, ML4AAD" __license__ = "3-clause BSD" -class SMAC(object): +class SMAC4AC(object): """ - Facade to use SMAC default mode + Facade to use SMAC default mode for Algorithm configuration Attributes ---------- @@ -73,21 +70,32 @@ class SMAC(object): def __init__(self, scenario: Scenario, - tae_runner: typing.Optional[typing.Union[ExecuteTARun, typing.Callable]]=None, - runhistory: typing.Optional[RunHistory]=None, - intensifier: typing.Optional[Intensifier]=None, - acquisition_function: typing.Optional[AbstractAcquisitionFunction]=None, - acquisition_function_optimizer: typing.Optional[AcquisitionFunctionMaximizer]=None, - model: typing.Optional[AbstractEPM]=None, - runhistory2epm: typing.Optional[AbstractRunHistory2EPM]=None, - initial_design: typing.Optional[InitialDesign]=None, - initial_configurations: typing.Optional[typing.List[Configuration]]=None, - stats: typing.Optional[Stats]=None, - restore_incumbent: typing.Optional[Configuration]=None, - rng: typing.Optional[typing.Union[np.random.RandomState, int]]=None, - smbo_class: typing.Optional[SMBO]=None, - run_id: typing.Optional[int]=None, - random_configuration_chooser: typing.Optional[RandomConfigurationChooser]=None): + tae_runner: Optional[Union[Type[ExecuteTARun], Callable]] = None, + tae_runner_kwargs: Optional[dict] = None, + runhistory: Optional[Union[Type[RunHistory], RunHistory]] = None, + runhistory_kwargs: Optional[dict] = None, + intensifier: Optional[Type[Intensifier]] = None, + intensifier_kwargs: Optional[dict] = None, + acquisition_function: Optional[Type[AbstractAcquisitionFunction]] = None, + acquisition_function_kwargs: Optional[dict] = None, + integrate_acquisition_function: bool = False, + acquisition_function_optimizer: Optional[Type[AcquisitionFunctionMaximizer]] = None, + acquisition_function_optimizer_kwargs: Optional[dict] = None, + model: Optional[Type[AbstractEPM]] = None, + model_kwargs: Optional[dict] = None, + runhistory2epm: Optional[Type[AbstractRunHistory2EPM]] = None, + runhistory2epm_kwargs: Optional[dict] = None, + initial_design: Optional[Type[InitialDesign]] = None, + initial_design_kwargs: Optional[dict] = None, + initial_configurations: Optional[List[Configuration]] = None, + stats: Optional[Stats] = None, + restore_incumbent: Optional[Configuration] = None, + rng: Optional[Union[np.random.RandomState, int]] = None, + smbo_class: Optional[SMBO] = None, + run_id: Optional[int] = None, + random_configuration_chooser: Optional[Type[RandomConfigurationChooser]] = None, + random_configuration_chooser_kwargs: Optional[dict] = None + ): """ Constructor @@ -102,29 +110,51 @@ def __init__(self, :class:`~smac.tae.execute_func.ExecuteTAFuncDict`. If not set, it will be initialized with the :class:`~smac.tae.execute_ta_run_old.ExecuteTARunOld`. + tae_runner_kwargs: Optional[dict] + arguments passed to constructor of '~tae_runner' runhistory : RunHistory runhistory to store all algorithm runs + runhistory_kwargs : Optional[dict] + arguments passed to constructor of runhistory. + We strongly advise against changing the aggregation function, + since it will break some code assumptions intensifier : Intensifier intensification object to issue a racing to decide the current incumbent + intensifier_kwargs: Optional[dict] + arguments passed to the constructor of '~intensifier' acquisition_function : ~smac.optimizer.acquisition.AbstractAcquisitionFunction - Object that implements the :class:`~smac.optimizer.acquisition.AbstractAcquisitionFunction`. - Will use :class:`~smac.optimizer.acquisition.EI` if not set. + Class or object that implements the :class:`~smac.optimizer.acquisition.AbstractAcquisitionFunction`. + Will use :class:`~smac.optimizer.acquisition.EI` or :class:`~smac.optimizer.acquisition.LogEI` if not set. + `~acquisition_function_kwargs` is passed to the class constructor. + acquisition_function_kwargs : Optional[dict] + dictionary to pass specific arguments to ~acquisition_function + integrate_acquisition_function : bool, default=False + Whether to integrate the acquisition function. Works only with models which can sample their + hyperparameters (i.e. GaussianProcessMCMC). acquisition_function_optimizer : ~smac.optimizer.ei_optimization.AcquisitionFunctionMaximizer Object that implements the :class:`~smac.optimizer.ei_optimization.AcquisitionFunctionMaximizer`. Will use :class:`smac.optimizer.ei_optimization.InterleavedLocalAndRandomSearch` if not set. + acquisition_function_optimizer_kwargs: Optional[dict] + Arguments passed to constructor of '~acquisition_function_optimizer' model : AbstractEPM Model that implements train() and predict(). Will use a :class:`~smac.epm.rf_with_instances.RandomForestWithInstances` if not set. + model_kwargs : Optional[dict] + Arguments passed to constructor of '~model' runhistory2epm : ~smac.runhistory.runhistory2epm.RunHistory2EMP Object that implements the AbstractRunHistory2EPM. If None, will use :class:`~smac.runhistory.runhistory2epm.RunHistory2EPM4Cost` if objective is cost or :class:`~smac.runhistory.runhistory2epm.RunHistory2EPM4LogCost` if objective is runtime. + runhistory2epm_kwargs: Optional[dict] + Arguments passed to the constructor of '~runhistory2epm' initial_design : InitialDesign initial sampling design - initial_configurations : typing.List[Configuration] + initial_design_kwargs: Optional[dict] + arguments passed to constructor of `~initial_design' + initial_configurations : List[Configuration] list of initial configurations for initial design -- cannot be used together with initial_design stats : Stats @@ -141,6 +171,8 @@ def __init__(self, chosen. random_configuration_chooser : ~smac.optimizer.random_configuration_chooser.RandomConfigurationChooser How often to choose a random configuration during the intensification procedure. + random_configuration_chooser_kwargs : Optional[dict] + arguments of constructor for '~random_configuration_chooser' """ self.logger = logging.getLogger( @@ -171,10 +203,11 @@ def __init__(self, and getattr(scenario, 'tuner_timeout', None) is None and scenario.run_obj == 'quality' ): - self.logger.info('Optimizing a deterministic scenario for ' - 'quality without a tuner timeout - will make ' - 'SMAC deterministic!') + self.logger.info('Optimizing a deterministic scenario for quality without a tuner timeout - will make ' + 'SMAC deterministic and only evaluate one configuration per iteration!') scenario.intensification_percentage = 1e-10 + scenario.min_chall = 1 + scenario.write() # initialize stats object @@ -184,19 +217,35 @@ def __init__(self, self.stats = Stats(scenario) if self.scenario.run_obj == "runtime" and not self.scenario.transform_y == "LOG": - self.logger.warn("Runtime as objective automatically activates log(y) transformation") + self.logger.warning("Runtime as objective automatically activates log(y) transformation") self.scenario.transform_y = "LOG" # initialize empty runhistory + runhistory_def_kwargs = {'aggregate_func': aggregate_func} + if runhistory_kwargs is not None: + runhistory_def_kwargs.update(runhistory_kwargs) if runhistory is None: - runhistory = RunHistory(aggregate_func=aggregate_func) - # inject aggr_func if necessary - if runhistory.aggregate_func is None: - runhistory.aggregate_func = aggregate_func + runhistory = RunHistory(**runhistory_def_kwargs) + elif inspect.isclass(runhistory): + runhistory = runhistory(**runhistory_def_kwargs) + else: + if runhistory.aggregate_func is None: + runhistory.aggregate_func = aggregate_func - if not random_configuration_chooser: - random_configuration_chooser = ChooserProb(prob=scenario.rand_prob, - rng=rng) + rand_conf_chooser_kwargs = { + 'rng': rng + } + if random_configuration_chooser_kwargs is not None: + rand_conf_chooser_kwargs.update(random_configuration_chooser_kwargs) + if random_configuration_chooser is None: + if 'prob' not in rand_conf_chooser_kwargs: + rand_conf_chooser_kwargs['prob'] = scenario.rand_prob + random_configuration_chooser = ChooserProb(**rand_conf_chooser_kwargs) + elif inspect.isclass(random_configuration_chooser): + random_configuration_chooser = random_configuration_chooser(**rand_conf_chooser_kwargs) + elif not isinstance(random_configuration_chooser, RandomConfigurationChooser): + raise ValueError("random_configuration_chooser has to be" + " a class or object of RandomConfigurationChooser") # reset random number generator in config space to draw different # random configurations with each seed given to SMAC @@ -207,75 +256,110 @@ def __init__(self, # initial EPM types, bounds = get_types(scenario.cs, scenario.feature_array) + model_def_kwargs = { + 'types': types, + 'bounds': bounds, + 'instance_features': scenario.feature_array, + 'seed': rng.randint(MAXINT), + 'pca_components': scenario.PCA_DIM, + } + if model_kwargs is not None: + model_def_kwargs.update(model_kwargs) if model is None: - model = RandomForestWithInstances(types=types, - bounds=bounds, - instance_features=scenario.feature_array, - seed=rng.randint(MAXINT), - pca_components=scenario.PCA_DIM, - log_y=scenario.transform_y in ["LOG", "LOGS"], - num_trees=scenario.rf_num_trees, - do_bootstrapping=scenario.rf_do_bootstrapping, - ratio_features=scenario.rf_ratio_features, - min_samples_split=scenario.rf_min_samples_split, - min_samples_leaf=scenario.rf_min_samples_leaf, - max_depth=scenario.rf_max_depth) + for key, value in { + 'log_y': scenario.transform_y in ["LOG", "LOGS"], + 'num_trees': scenario.rf_num_trees, + 'do_bootstrapping': scenario.rf_do_bootstrapping, + 'ratio_features': scenario.rf_ratio_features, + 'min_samples_split': scenario.rf_min_samples_split, + 'min_samples_leaf': scenario.rf_min_samples_leaf, + 'max_depth': scenario.rf_max_depth, + }.items(): + if key not in model_def_kwargs: + model_def_kwargs[key] = value + model_def_kwargs['configspace'] = self.scenario.cs + model = RandomForestWithInstances(**model_def_kwargs) + elif inspect.isclass(model): + model_def_kwargs['configspace'] = self.scenario.cs + model = model(**model_def_kwargs) + else: + raise TypeError( + "Model not recognized: %s" %(type(model))) + # initial acquisition function + acq_def_kwargs = {'model': model} + if acquisition_function_kwargs is not None: + acq_def_kwargs.update(acquisition_function_kwargs) if acquisition_function is None: if scenario.transform_y in ["LOG", "LOGS"]: - acquisition_function = LogEI(model=model) + acquisition_function = LogEI(**acq_def_kwargs) else: - acquisition_function = EI(model=model) - - # inject model if necessary - if acquisition_function.model is None: - acquisition_function.model = model + acquisition_function = EI(**acq_def_kwargs) + elif inspect.isclass(acquisition_function): + acquisition_function = acquisition_function(**acq_def_kwargs) + else: + raise TypeError( + "Argument acquisition_function must be None or an object implementing the " + "AbstractAcquisitionFunction, not %s." + % type(acquisition_function) + ) + if integrate_acquisition_function: + acquisition_function = IntegratedAcquisitionFunction( + acquisition_function=acquisition_function, + **acq_def_kwargs + ) # initialize optimizer on acquisition function + acq_func_opt_kwargs = { + 'acquisition_function': acquisition_function, + 'config_space': scenario.cs, + 'rng': rng, + } + if acquisition_function_optimizer_kwargs is not None: + acq_func_opt_kwargs.update(acquisition_function_optimizer_kwargs) if acquisition_function_optimizer is None: - acquisition_function_optimizer = InterleavedLocalAndRandomSearch( - acquisition_function=acquisition_function, - config_space=scenario.cs, - rng=np.random.RandomState(seed=rng.randint(MAXINT)), - max_steps=scenario.sls_max_steps, - n_steps_plateau_walk=scenario.sls_n_steps_plateau_walk - ) - elif not isinstance( - acquisition_function_optimizer, - AcquisitionFunctionMaximizer, - ): - raise ValueError( - "Argument 'acquisition_function_optimizer' must be of type" - "'AcquisitionFunctionMaximizer', but is '%s'" % + for key, value in { + 'max_steps': scenario.sls_max_steps, + 'n_steps_plateau_walk': scenario.sls_n_steps_plateau_walk, + }.items(): + if key not in acq_func_opt_kwargs: + acq_func_opt_kwargs[key] = value + acquisition_function_optimizer = InterleavedLocalAndRandomSearch(**acq_func_opt_kwargs) + elif inspect.isclass(acquisition_function_optimizer): + acquisition_function_optimizer = acquisition_function_optimizer(**acq_func_opt_kwargs) + else: + raise TypeError( + "Argument acquisition_function_optimizer must be None or an object implementing the " + "AcquisitionFunctionMaximizer, but is '%s'" % type(acquisition_function_optimizer) ) # initialize tae_runner # First case, if tae_runner is None, the target algorithm is a call # string in the scenario file + tae_def_kwargs = { + 'stats': self.stats, + 'run_obj': scenario.run_obj, + 'runhistory': runhistory, + 'par_factor': scenario.par_factor, + 'cost_for_crash': scenario.cost_for_crash, + 'abort_on_first_run_crash': scenario.abort_on_first_run_crash + } + if tae_runner_kwargs is not None: + tae_def_kwargs.update(tae_runner_kwargs) + if 'ta' not in tae_def_kwargs: + tae_def_kwargs['ta'] = scenario.ta if tae_runner is None: - tae_runner = ExecuteTARunOld(ta=scenario.ta, - stats=self.stats, - run_obj=scenario.run_obj, - runhistory=runhistory, - par_factor=scenario.par_factor, - cost_for_crash=scenario.cost_for_crash, - abort_on_first_run_crash=scenario.abort_on_first_run_crash) - # Second case, the tae_runner is a function to be optimized + tae_def_kwargs['ta'] = scenario.ta + tae_runner = ExecuteTARunOld(**tae_def_kwargs) + elif inspect.isclass(tae_runner): + tae_runner = tae_runner(**tae_def_kwargs) elif callable(tae_runner): - tae_runner = ExecuteTAFuncDict(ta=tae_runner, - stats=self.stats, - run_obj=scenario.run_obj, - memory_limit=scenario.memory_limit, - runhistory=runhistory, - par_factor=scenario.par_factor, - cost_for_crash=scenario.cost_for_crash, - abort_on_first_run_crash=scenario.abort_on_first_run_crash) - # Third case, if it is an ExecuteTaRun we can simply use the - # instance. Otherwise, the next check raises an exception - elif not isinstance(tae_runner, ExecuteTARun): + tae_def_kwargs['ta'] = tae_runner + tae_runner = ExecuteTAFuncDict(**tae_def_kwargs) + else: raise TypeError("Argument 'tae_runner' is %s, but must be " - "either a callable or an instance of " + "either None, a callable or an object implementing " "ExecuteTaRun. Passing 'None' will result in the " "creation of target algorithm runner based on the " "call string in the scenario file." @@ -287,170 +371,139 @@ def __init__(self, "the scenario must be the same, but are '%s' and " "'%s'" % (tae_runner.run_obj, scenario.run_obj)) - # inject stats if necessary - if tae_runner.stats is None: - tae_runner.stats = self.stats - # inject runhistory if necessary - if tae_runner.runhistory is None: - tae_runner.runhistory = runhistory - # inject cost_for_crash - if tae_runner.crash_cost != scenario.cost_for_crash: - tae_runner.crash_cost = scenario.cost_for_crash - # initialize intensification + intensifier_def_kwargs = { + 'tae_runner': tae_runner, + 'stats': self.stats, + 'traj_logger': traj_logger, + 'rng': rng, + 'instances': scenario.train_insts, + 'cutoff': scenario.cutoff, + 'deterministic': scenario.deterministic, + 'run_obj_time': scenario.run_obj == "runtime", + 'always_race_against': scenario.cs.get_default_configuration() + if scenario.always_race_default else None, + 'use_ta_time_bound': scenario.use_ta_time, + 'instance_specifics': scenario.instance_specific, + 'minR': scenario.minR, + 'maxR': scenario.maxR, + 'adaptive_capping_slackfactor': scenario.intens_adaptive_capping_slackfactor, + 'min_chall': scenario.intens_min_chall + } + if intensifier_kwargs is not None: + intensifier_def_kwargs.update(intensifier_kwargs) if intensifier is None: - intensifier = Intensifier(tae_runner=tae_runner, - stats=self.stats, - traj_logger=traj_logger, - rng=rng, - instances=scenario.train_insts, - cutoff=scenario.cutoff, - deterministic=scenario.deterministic, - run_obj_time=scenario.run_obj == "runtime", - always_race_against=scenario.cs.get_default_configuration() - if scenario.always_race_default else None, - use_ta_time_bound=scenario.use_ta_time, - instance_specifics=scenario.instance_specific, - minR=scenario.minR, - maxR=scenario.maxR, - adaptive_capping_slackfactor=scenario.intens_adaptive_capping_slackfactor, - min_chall=scenario.intens_min_chall) - # inject deps if necessary - if intensifier.tae_runner is None: - intensifier.tae_runner = tae_runner - if intensifier.stats is None: - intensifier.stats = self.stats - if intensifier.traj_logger is None: - intensifier.traj_logger = traj_logger + intensifier = Intensifier(**intensifier_def_kwargs) + elif inspect.isclass(intensifier): + intensifier = intensifier(**intensifier_def_kwargs) + else: + raise TypeError( + "Argument intensifier must be None or an object implementing the Intensifier, but is '%s'" % + type(intensifier) + ) # initial design if initial_design is not None and initial_configurations is not None: raise ValueError( "Either use initial_design or initial_configurations; but not both") + init_design_def_kwargs = { + 'tae_runner': tae_runner, + 'scenario': scenario, + 'stats': self.stats, + 'traj_logger': traj_logger, + 'runhistory': runhistory, + 'rng': rng, + 'configs': initial_configurations, + 'intensifier': intensifier, + 'aggregate_func': aggregate_func, + 'n_configs_x_params': 0, + 'max_config_fracs': 0.0 + } + if initial_design_kwargs is not None: + init_design_def_kwargs.update(initial_design_kwargs) if initial_configurations is not None: - initial_design = MultiConfigInitialDesign(tae_runner=tae_runner, - scenario=scenario, - stats=self.stats, - traj_logger=traj_logger, - runhistory=runhistory, - rng=rng, - configs=initial_configurations, - intensifier=intensifier, - aggregate_func=aggregate_func) + initial_design = InitialDesign(**init_design_def_kwargs) elif initial_design is None: if scenario.initial_incumbent == "DEFAULT": - initial_design = DefaultConfiguration(tae_runner=tae_runner, - scenario=scenario, - stats=self.stats, - traj_logger=traj_logger, - rng=rng) + init_design_def_kwargs['max_config_fracs'] = 0.0 + initial_design = DefaultConfiguration(**init_design_def_kwargs) elif scenario.initial_incumbent == "RANDOM": - initial_design = RandomConfiguration(tae_runner=tae_runner, - scenario=scenario, - stats=self.stats, - traj_logger=traj_logger, - rng=rng) + init_design_def_kwargs['max_config_fracs'] = 0.0 + initial_design = RandomConfigurations(**init_design_def_kwargs) elif scenario.initial_incumbent == "LHD": - initial_design = LHDesign(runhistory=runhistory, - intensifier=intensifier, - aggregate_func=aggregate_func, - tae_runner=tae_runner, - scenario=scenario, - stats=self.stats, - traj_logger=traj_logger, - rng=rng) + initial_design = LHDesign(**init_design_def_kwargs) elif scenario.initial_incumbent == "FACTORIAL": - initial_design = FactorialInitialDesign(runhistory=runhistory, - intensifier=intensifier, - aggregate_func=aggregate_func, - tae_runner=tae_runner, - scenario=scenario, - stats=self.stats, - traj_logger=traj_logger, - rng=rng) + initial_design = FactorialInitialDesign(**init_design_def_kwargs) elif scenario.initial_incumbent == "SOBOL": - initial_design = SobolDesign(runhistory=runhistory, - intensifier=intensifier, - aggregate_func=aggregate_func, - tae_runner=tae_runner, - scenario=scenario, - stats=self.stats, - traj_logger=traj_logger, - rng=rng) + initial_design = SobolDesign(**init_design_def_kwargs) else: raise ValueError("Don't know what kind of initial_incumbent " "'%s' is" % scenario.initial_incumbent) - # inject deps if necessary - if initial_design.tae_runner is None: - initial_design.tae_runner = tae_runner - if initial_design.scenario is None: - initial_design.scenario = scenario - if initial_design.stats is None: - initial_design.stats = self.stats - if initial_design.traj_logger is None: - initial_design.traj_logger = traj_logger - - # initial conversion of runhistory into EPM data - if runhistory2epm is None: + elif inspect.isclass(initial_design): + initial_design = initial_design(**init_design_def_kwargs) + else: + raise TypeError( + "Argument initial_design must be None or an object implementing the InitialDesign, but is '%s'" % + type(initial_design) + ) - num_params = len(scenario.cs.get_hyperparameters()) + # if we log the performance data, + # the RFRImputator will already get + # log transform data from the runhistory + if scenario.transform_y in ["LOG", "LOGS"]: + cutoff = np.log(np.nanmin([np.inf, np.float_(scenario.cutoff)])) + threshold = cutoff + np.log(scenario.par_factor) + else: + cutoff = np.nanmin([np.inf, np.float_(scenario.cutoff)]) + threshold = cutoff * scenario.par_factor + num_params = len(scenario.cs.get_hyperparameters()) + imputor = RFRImputator(rng=rng, + cutoff=cutoff, + threshold=threshold, + model=model, + change_threshold=0.01, + max_iter=2) + + r2e_def_kwargs = { + 'scenario': scenario, + 'num_params': num_params, + 'success_states': [StatusType.SUCCESS, ], + 'impute_censored_data': True, + 'impute_state': [StatusType.CAPPED, ], + 'imputor': imputor, + 'scale_perc': 5 + } + if scenario.run_obj == 'quality': + r2e_def_kwargs.update({ + 'success_states': [StatusType.SUCCESS, StatusType.CRASHED], + 'impute_censored_data': False, + 'impute_state': None, + }) + if runhistory2epm_kwargs is not None: + r2e_def_kwargs.update(runhistory2epm_kwargs) + if runhistory2epm is None: if scenario.run_obj == 'runtime': - - # if we log the performance data, - # the RFRImputator will already get - # log transform data from the runhistory - cutoff = np.log(scenario.cutoff) - threshold = np.log(scenario.cutoff * - scenario.par_factor) - - imputor = RFRImputator(rng=rng, - cutoff=cutoff, - threshold=threshold, - model=model, - change_threshold=0.01, - max_iter=2) - - runhistory2epm = RunHistory2EPM4LogCost( - scenario=scenario, num_params=num_params, - success_states=[StatusType.SUCCESS, ], - impute_censored_data=True, - impute_state=[StatusType.CAPPED, ], - imputor=imputor) - + runhistory2epm = RunHistory2EPM4LogCost(**r2e_def_kwargs) elif scenario.run_obj == 'quality': if scenario.transform_y == "NONE": - runhistory2epm = RunHistory2EPM4Cost(scenario=scenario, num_params=num_params, - success_states=[ - StatusType.SUCCESS, - StatusType.CRASHED], - impute_censored_data=False, impute_state=None) + runhistory2epm = RunHistory2EPM4Cost(**r2e_def_kwargs) elif scenario.transform_y == "LOG": - runhistory2epm = RunHistory2EPM4LogCost(scenario=scenario, num_params=num_params, - success_states=[ - StatusType.SUCCESS, - StatusType.CRASHED], - impute_censored_data=False, impute_state=None) + runhistory2epm = RunHistory2EPM4LogCost(**r2e_def_kwargs) elif scenario.transform_y == "LOGS": - runhistory2epm = RunHistory2EPM4LogScaledCost(scenario=scenario, num_params=num_params, - success_states=[ - StatusType.SUCCESS, - StatusType.CRASHED], - impute_censored_data=False, impute_state=None) + runhistory2epm = RunHistory2EPM4LogScaledCost(**r2e_def_kwargs) elif scenario.transform_y == "INVS": - runhistory2epm = RunHistory2EPM4InvScaledCost(scenario=scenario, num_params=num_params, - success_states=[ - StatusType.SUCCESS, - StatusType.CRASHED], - impute_censored_data=False, impute_state=None) - + runhistory2epm = RunHistory2EPM4InvScaledCost(**r2e_def_kwargs) else: raise ValueError('Unknown run objective: %s. Should be either ' 'quality or runtime.' % self.scenario.run_obj) - - # inject scenario if necessary: - if runhistory2epm.scenario is None: - runhistory2epm.scenario = scenario + elif inspect.isclass(runhistory2epm): + runhistory2epm = runhistory2epm(**r2e_def_kwargs) + else: + raise TypeError( + "Argument runhistory2epm must be None or an object implementing the RunHistory2EPM, but is '%s'" % + type(runhistory2epm) + ) smbo_args = { 'scenario': scenario, @@ -504,9 +557,12 @@ def optimize(self): return incumbent def validate(self, - config_mode: typing.Union[typing.List[Configuration], np.ndarray, str]='inc', - instance_mode: typing.Union[typing.List[str], str]='train+test', - repetitions: int=1, use_epm: bool=False, n_jobs: int=-1, backend: str='threading'): + config_mode: Union[List[Configuration], np.ndarray, str] = 'inc', + instance_mode: Union[List[str], str] = 'train+test', + repetitions: int = 1, + use_epm: bool = False, + n_jobs: int = -1, backend: + str = 'threading'): """ Create validator-object and run validation, using scenario-information, runhistory from smbo and tae_runner from intensify diff --git a/smac/facade/smac_bo_facade.py b/smac/facade/smac_bo_facade.py new file mode 100644 index 000000000..68583aabb --- /dev/null +++ b/smac/facade/smac_bo_facade.py @@ -0,0 +1,151 @@ +import numpy as np + +from smac.facade.smac_ac_facade import SMAC4AC +from smac.epm.gaussian_process_mcmc import GaussianProcessMCMC, GaussianProcess +from smac.epm.gp_base_prior import HorseshoePrior, LognormalPrior +from smac.epm.util_funcs import get_types, get_rng +from smac.initial_design.sobol_design import SobolDesign +from smac.runhistory.runhistory2epm import RunHistory2EPM4LogScaledCost + + +__author__ = "Marius Lindauer" +__copyright__ = "Copyright 2018, ML4AAD" +__license__ = "3-clause BSD" + + +class SMAC4BO(SMAC4AC): + """ + Facade to use SMAC for BO using a GP + + see smac.facade.smac_Facade for API + This facade overwrites options available via the SMAC facade + + Attributes + ---------- + logger + stats : Stats + solver : SMBO + runhistory : RunHistory + List with information about previous runs + trajectory : list + List of all incumbents + + """ + + def __init__(self, model_type='gp_mcmc', **kwargs): + """ + Constructor + see ~smac.facade.smac_facade for documentation + """ + scenario = kwargs['scenario'] + + kwargs['initial_design'] = kwargs.get('initial_design', SobolDesign) + kwargs['runhistory2epm'] = kwargs.get('runhistory2epm', RunHistory2EPM4LogScaledCost) + + init_kwargs = kwargs.get('initial_design_kwargs', dict()) + init_kwargs['n_configs_x_params'] = init_kwargs.get('n_configs_x_params', 10) + init_kwargs['max_config_fracs'] = init_kwargs.get('max_config_fracs', 0.25) + kwargs['initial_design_kwargs'] = init_kwargs + + if kwargs.get('model') is None: + from smac.epm.gp_kernels import ConstantKernel, Matern, WhiteKernel, HammingKernel + + model_kwargs = kwargs.get('model_kwargs', dict()) + + _, rng = get_rng(rng=kwargs.get("rng", None), run_id=kwargs.get("run_id", None), logger=None) + + types, bounds = get_types(kwargs['scenario'].cs, instance_features=None) + + cov_amp = ConstantKernel( + 2.0, + constant_value_bounds=(np.exp(-10), np.exp(2)), + prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rng), + ) + + cont_dims = np.nonzero(types == 0)[0] + cat_dims = np.nonzero(types != 0)[0] + + if len(cont_dims) > 0: + exp_kernel = Matern( + np.ones([len(cont_dims)]), + [(np.exp(-10), np.exp(2)) for _ in range(len(cont_dims))], + nu=2.5, + operate_on=cont_dims, + ) + + if len(cat_dims) > 0: + ham_kernel = HammingKernel( + np.ones([len(cat_dims)]), + [(np.exp(-10), np.exp(2)) for _ in range(len(cat_dims))], + operate_on=cat_dims, + ) + + noise_kernel = WhiteKernel( + noise_level=1e-8, + noise_level_bounds=(np.exp(-25), np.exp(2)), + prior=HorseshoePrior(scale=0.1, rng=rng), + ) + + if len(cont_dims) > 0 and len(cat_dims) > 0: + # both + kernel = cov_amp * (exp_kernel*ham_kernel) + noise_kernel + elif len(cont_dims) > 0 and len(cat_dims) == 0: + # only cont + kernel = cov_amp * exp_kernel + noise_kernel + elif len(cont_dims) == 0 and len(cat_dims) > 0: + # only cont + kernel = cov_amp * ham_kernel + noise_kernel + else: + raise ValueError() + + if model_type == "gp": + model_class = GaussianProcess + kwargs['model'] = model_class + model_kwargs['kernel'] = kernel + model_kwargs['normalize_y'] = True + model_kwargs['seed'] = rng.randint(0, 2 ** 20) + elif model_type == "gp_mcmc": + model_class = GaussianProcessMCMC + kwargs['model'] = model_class + kwargs['integrate_acquisition_function'] = True + + model_kwargs['kernel'] = kernel + + n_mcmc_walkers = 3 * len(kernel.theta) + if n_mcmc_walkers % 2 == 1: + n_mcmc_walkers += 1 + model_kwargs['n_mcmc_walkers'] = n_mcmc_walkers + model_kwargs['chain_length'] = 250 + model_kwargs['burnin_steps'] = 250 + model_kwargs['normalize_y'] = True + model_kwargs['seed'] = rng.randint(0, 2**20) + else: + raise ValueError('Unknown model type %s' % model_type) + kwargs['model_kwargs'] = model_kwargs + + if kwargs.get('random_configuration_chooser') is None: + random_config_chooser_kwargs = kwargs.get('random_configuration_chooser_kwargs', dict()) + random_config_chooser_kwargs['prob'] = random_config_chooser_kwargs.get('prob', 0.0) + kwargs['random_configuration_chooser_kwargs'] = random_config_chooser_kwargs + + if kwargs.get('acquisition_function_optimizer') is None: + acquisition_function_optimizer_kwargs = kwargs.get('acquisition_function_optimizer_kwargs', dict()) + acquisition_function_optimizer_kwargs['n_sls_iterations'] = 10 + kwargs['acquisition_function_optimizer_kwargs'] = acquisition_function_optimizer_kwargs + + # only 1 configuration per SMBO iteration + intensifier_kwargs = kwargs.get('intensifier_kwargs', dict()) + intensifier_kwargs['min_chall'] = 1 + kwargs['intensifier_kwargs'] = intensifier_kwargs + scenario.intensification_percentage = 1e-10 + + super().__init__(**kwargs) + + if self.solver.scenario.n_features > 0: + raise NotImplementedError("BOGP cannot handle instances") + + self.logger.info(self.__class__) + + self.solver.scenario.acq_opt_challengers = 1000 + # activate predict incumbent + self.solver.predict_incumbent = True diff --git a/smac/facade/smac_hpo_facade.py b/smac/facade/smac_hpo_facade.py new file mode 100644 index 000000000..2e25fce8f --- /dev/null +++ b/smac/facade/smac_hpo_facade.py @@ -0,0 +1,90 @@ +from smac.facade.smac_ac_facade import SMAC4AC +from smac.runhistory.runhistory2epm import RunHistory2EPM4LogScaledCost +from smac.optimizer.acquisition import LogEI +from smac.epm.rf_with_instances import RandomForestWithInstances +from smac.initial_design.sobol_design import SobolDesign + +__author__ = "Marius Lindauer" +__copyright__ = "Copyright 2018, ML4AAD" +__license__ = "3-clause BSD" + + +class SMAC4HPO(SMAC4AC): + """ + Facade to use SMAC for hyperparameter optimization + + see smac.facade.smac_Facade for API + This facade overwrites options available via the SMAC facade + + Attributes + ---------- + logger + stats : Stats + solver : SMBO + runhistory : RunHistory + List with information about previous runs + trajectory : list + List of all incumbents + + """ + + def __init__(self, **kwargs): + """ + Constructor + see ~smac.facade.smac_facade for docu + """ + + scenario = kwargs['scenario'] + + kwargs['initial_design'] = kwargs.get('initial_design', SobolDesign) + kwargs['runhistory2epm'] = kwargs.get('runhistory2epm', RunHistory2EPM4LogScaledCost) + + init_kwargs = kwargs.get('initial_design_kwargs', dict()) + init_kwargs['n_configs_x_params'] = init_kwargs.get('n_configs_x_params', 10) + init_kwargs['max_config_fracs'] = init_kwargs.get('max_config_fracs', 0.25) + kwargs['initial_design_kwargs'] = init_kwargs + + # only 1 configuration per SMBO iteration + intensifier_kwargs = kwargs.get('intensifier_kwargs', dict()) + intensifier_kwargs['min_chall'] = 1 + kwargs['intensifier_kwargs'] = intensifier_kwargs + scenario.intensification_percentage = 1e-10 + + model_class = RandomForestWithInstances + kwargs['model'] = model_class + + # == static RF settings + model_kwargs = kwargs.get('model_kwargs', dict()) + model_kwargs['num_trees'] = model_kwargs.get('num_trees', 10) + model_kwargs['do_bootstrapping'] = model_kwargs.get('do_bootstrapping', True) + model_kwargs['ratio_features'] = model_kwargs.get('ratio_features', 1.0) + model_kwargs['min_samples_split'] = model_kwargs.get('min_samples_split', 2) + model_kwargs['min_samples_leaf'] = model_kwargs.get('min_samples_leaf', 1) + model_kwargs['log_y'] = model_kwargs.get('log_y', True) + kwargs['model_kwargs'] = model_kwargs + + # == Acquisition function + kwargs['acquisition_function'] = kwargs.get('acquisition_function', LogEI) + + kwargs['runhistory2epm'] = kwargs.get('runhistory2epm', RunHistory2EPM4LogScaledCost) + + # assumes random chooser for random configs + random_config_chooser_kwargs = kwargs.get('random_configuration_chooser_kwargs', dict()) + random_config_chooser_kwargs['prob'] = random_config_chooser_kwargs.get('prob', 0.2) + kwargs['random_configuration_chooser_kwargs'] = random_config_chooser_kwargs + + # better improve acquisition function optimization + # 1. increase number of sls iterations + acquisition_function_optimizer_kwargs = kwargs.get('acquisition_function_optimizer_kwargs', dict()) + acquisition_function_optimizer_kwargs['n_sls_iterations'] = 10 + kwargs['acquisition_function_optimizer_kwargs'] = acquisition_function_optimizer_kwargs + + super().__init__(**kwargs) + self.logger.info(self.__class__) + + # better improve acquisition function optimization + # 2. more randomly sampled configurations + self.solver.scenario.acq_opt_challengers = 10000 + + # activate predict incumbent + self.solver.predict_incumbent = True diff --git a/smac/initial_design/default_configuration_design.py b/smac/initial_design/default_configuration_design.py index 66922ac46..6253c194c 100644 --- a/smac/initial_design/default_configuration_design.py +++ b/smac/initial_design/default_configuration_design.py @@ -1,57 +1,21 @@ +from typing import List + from ConfigSpace import Configuration -import numpy as np -from smac.initial_design.single_config_initial_design import \ - SingleConfigInitialDesign -from smac.tae.execute_ta_run import ExecuteTARun -from smac.stats.stats import Stats -from smac.utils.io.traj_logging import TrajLogger -from smac.scenario.scenario import Scenario +from smac.initial_design.initial_design import InitialDesign + __author__ = "Marius Lindauer" __copyright__ = "Copyright 2016, ML4AAD" __license__ = "3-clause BSD" -class DefaultConfiguration(SingleConfigInitialDesign): - """ Initial design that evaluates default configuration - - Attributes - ---------- +class DefaultConfiguration(InitialDesign): - """ + """Initial design that evaluates default configuration""" - def __init__(self, - tae_runner: ExecuteTARun, - scenario: Scenario, - stats: Stats, - traj_logger: TrajLogger, - rng: np.random.RandomState - ): - """Constructor + def _select_configurations(self) -> List[Configuration]: - Parameters - ---------- - tae_runner: ExecuteTARun - Target algorithm execution object. - scenario: Scenario - Scenario with all meta information (including configuration space). - stats: Stats - Statistics of experiments; needed in case initial design already - exhausts the budget. - traj_logger: TrajLogger - Trajectory logging to add new incumbents found by the initial - design. - rng: np.random.RandomState - Random state - """ - super().__init__(tae_runner=tae_runner, - scenario=scenario, - stats=stats, - traj_logger=traj_logger, - rng=rng) - - def _select_configuration(self) -> Configuration: """Selects the default configuration. Returns @@ -59,6 +23,7 @@ def _select_configuration(self) -> Configuration: config: Configuration Initial incumbent configuration. """ + config = self.scenario.cs.get_default_configuration() config.origin = 'Default' - return config + return [config] diff --git a/smac/initial_design/factorial_design.py b/smac/initial_design/factorial_design.py index a2a776261..47901c1bf 100644 --- a/smac/initial_design/factorial_design.py +++ b/smac/initial_design/factorial_design.py @@ -6,8 +6,7 @@ from ConfigSpace.util import deactivate_inactive_hyperparameters import numpy as np -from smac.initial_design.multi_config_initial_design import \ - MultiConfigInitialDesign +from smac.initial_design.initial_design import InitialDesign __author__ = "Marius Lindauer" @@ -15,8 +14,8 @@ __license__ = "3-clause BSD" -class FactorialInitialDesign(MultiConfigInitialDesign): - """ Factorial initial design +class FactorialInitialDesign(InitialDesign): + """Factorial initial design Attributes ---------- diff --git a/smac/initial_design/initial_design.py b/smac/initial_design/initial_design.py index 639e2b24b..97f44a02d 100644 --- a/smac/initial_design/initial_design.py +++ b/smac/initial_design/initial_design.py @@ -1,31 +1,36 @@ import logging +import typing -from ConfigSpace import Configuration +from ConfigSpace.configuration_space import Configuration, ConfigurationSpace +from ConfigSpace.hyperparameters import NumericalHyperparameter, \ + Constant, CategoricalHyperparameter, OrdinalHyperparameter +from ConfigSpace.util import deactivate_inactive_hyperparameters import numpy as np +from smac.intensification.intensification import Intensifier from smac.tae.execute_ta_run import ExecuteTARun from smac.stats.stats import Stats from smac.utils.io.traj_logging import TrajLogger from smac.scenario.scenario import Scenario +from smac.runhistory.runhistory import RunHistory +from smac.tae.execute_ta_run import FirstRunCrashedException +from smac.utils import constants __author__ = "Marius Lindauer" -__copyright__ = "Copyright 2016, ML4AAD" +__copyright__ = "Copyright 2019, AutoML" __license__ = "3-clause BSD" -class InitialDesign(object): - """ Base class for initial designs, i.e. the configurations that are run - before optimization starts +class InitialDesign: + """Base class for initial design strategies that evaluates multiple configurations Attributes ---------- - tae_runner : ExecuteTARun - Target algorithm runner that will be used to run the initial design - scenario - stats - traj_logger - rng - logger + configs : typing.List[Configuration] + List of configurations to be evaluated + intensifier + runhistory + aggregate_func """ def __init__(self, @@ -33,7 +38,13 @@ def __init__(self, scenario: Scenario, stats: Stats, traj_logger: TrajLogger, - rng: np.random.RandomState + runhistory: RunHistory, + rng: np.random.RandomState, + intensifier: Intensifier, + aggregate_func: typing.Callable, + configs: typing.Optional[typing.List[Configuration]]=None, + n_configs_x_params: int=10, + max_config_fracs: float=0.25, ): """Constructor @@ -49,15 +60,50 @@ def __init__(self, traj_logger: TrajLogger Trajectory logging to add new incumbents found by the initial design. + runhistory: RunHistory + Runhistory with all target algorithm runs. rng: np.random.RandomState Random state + intensifier: Intensifier + Intensification object to issue a racing to decide the current + incumbent. + aggregate_func: typing:Callable + Function to aggregate performance of a configuration across + instances. + configs: typing.Optional[typing.List[Configuration]] + List of initial configurations. + n_configs_x_params: int + how many configurations will be used at most in the initial design (X*D) + max_config_fracs: float + use at most X*budget in the initial design. Not active if a time limit is given. """ + self.tae_runner = tae_runner - self.scenario = scenario self.stats = stats self.traj_logger = traj_logger + self.scenario = scenario self.rng = rng - self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) + self.configs = configs + self.intensifier = intensifier + self.runhistory = runhistory + self.aggregate_func = aggregate_func + + self.logger = self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) + + n_params = len(self.scenario.cs.get_hyperparameters()) + self.init_budget = int(max(1, min(n_configs_x_params * n_params, + (max_config_fracs * scenario.ta_run_limit)))) + self.logger.info("Running initial design for %d configurations" %(self.init_budget)) + + def select_configurations(self) -> typing.List[Configuration]: + + if self.configs is None: + return self._select_configurations() + else: + return self.configs + + def _select_configurations(self) -> typing.List[Configuration]: + raise NotImplementedError def run(self) -> Configuration: """Run the initial design. @@ -65,7 +111,117 @@ def run(self) -> Configuration: Returns ------- incumbent: Configuration - Initial incumbent configuration. + Initial incumbent configuration """ + configs = self.select_configurations() + for config in configs: + if config.origin is None: + config.origin = 'Initial design' - raise NotImplementedError + # run first design + # ensures that first design is part of trajectory file + inc = self._run_first_configuration(configs[0], self.scenario) + + if len(set(configs)) > 1: + # intensify will skip all challenger that are identical with the incumbent; + # if has only identical configurations, + # intensifiy will not do any configuration runs + # (also not on the incumbent) + # therefore, at least two different configurations have to be in + inc, _ = self.intensifier.intensify( + challengers=configs[1:], + incumbent=configs[0], + run_history=self.runhistory, + aggregate_func=self.aggregate_func, + ) + + return inc + + def _run_first_configuration(self, initial_incumbent, scenario): + """Runs the initial design by calling the target algorithm and adding new entries to the trajectory logger. + + Returns + ------- + incumbent: Configuration + Initial incumbent configuration + """ + if initial_incumbent.origin is None: + initial_incumbent.origin = 'Initial design' + + # add this incumbent right away to have an entry to time point 0 + self.traj_logger.add_entry(train_perf=2 ** 31, + incumbent_id=1, + incumbent=initial_incumbent) + + rand_inst = self.rng.choice(self.scenario.train_insts) + + if self.scenario.deterministic: + initial_seed = 0 + else: + initial_seed = self.rng.randint(0, constants.MAXINT) + + try: + status, cost, runtime, _ = self.tae_runner.start( + initial_incumbent, + instance=rand_inst, + cutoff=self.scenario.cutoff, + seed=initial_seed, + instance_specific=self.scenario.instance_specific.get(rand_inst, + "0")) + except FirstRunCrashedException as err: + if self.scenario.abort_on_first_run_crash: + raise err + else: + # TODO make it possible to add the failed run to the runhistory + if self.scenario.run_obj == "quality": + cost = self.scenario.cost_for_crash + else: + cost = self.scenario.cutoff * scenario.par_factor + + self.stats.inc_changed += 1 # first incumbent + + self.traj_logger.add_entry(train_perf=cost, + incumbent_id=self.stats.inc_changed, + incumbent=initial_incumbent) + + return initial_incumbent + + def _transform_continuous_designs(self, + design: np.ndarray, + origin: str, + cs: ConfigurationSpace) -> typing.List[Configuration]: + + params = cs.get_hyperparameters() + for idx, param in enumerate(params): + if isinstance(param, NumericalHyperparameter): + continue + elif isinstance(param, Constant): + # add a vector with zeros + design_ = np.zeros(np.array(design.shape) + np.array((0, 1))) + design_[:, :idx] = design[:, :idx] + design_[:, idx+1:] = design[:, idx:] + design = design_ + elif isinstance(param, CategoricalHyperparameter): + v_design = design[:, idx] + v_design[v_design == 1] = 1 - 10**-10 + design[:, idx] = np.array(v_design * len(param.choices), dtype=np.int) + elif isinstance(param, OrdinalHyperparameter): + v_design = design[:, idx] + v_design[v_design == 1] = 1 - 10**-10 + design[:, idx] = np.array(v_design * len(param.sequence), dtype=np.int) + else: + raise ValueError("Hyperparamer not supported in LHD") + + self.logger.debug("Initial Design") + configs = [] + for vector in design: + conf = deactivate_inactive_hyperparameters(configuration=None, + configuration_space=cs, + vector=vector) + conf.origin = origin + configs.append(conf) + self.logger.debug(conf) + + self.logger.debug("Size of initial design: %d" % (len(configs))) + + return configs diff --git a/smac/initial_design/latin_hypercube_design.py b/smac/initial_design/latin_hypercube_design.py index 6c7fe017c..c6e10b769 100644 --- a/smac/initial_design/latin_hypercube_design.py +++ b/smac/initial_design/latin_hypercube_design.py @@ -1,21 +1,21 @@ import typing -from pyDOE import lhs +import numpy as np + +import pyDOE from ConfigSpace.configuration_space import Configuration -from ConfigSpace.hyperparameters import FloatHyperparameter -from ConfigSpace.util import deactivate_inactive_hyperparameters +from ConfigSpace.hyperparameters import Constant -from smac.initial_design.multi_config_initial_design import \ - MultiConfigInitialDesign +from smac.initial_design.initial_design import InitialDesign __author__ = "Marius Lindauer" -__copyright__ = "Copyright 2018, ML4AAD" +__copyright__ = "Copyright 2019, AutoML" __license__ = "3-clause BSD" -class LHDesign(MultiConfigInitialDesign): - """ Latin Hypercube design +class LHDesign(InitialDesign): + """Latin Hypercube design Attributes ---------- @@ -40,25 +40,16 @@ def _select_configurations(self) -> typing.List[Configuration]: cs = self.scenario.cs params = cs.get_hyperparameters() - lhd = lhs(n=len(params), samples=self.init_budget) - - for idx, param in enumerate(params): - if isinstance(param, FloatHyperparameter): - lhd[:,idx] = lhd[:,idx] * (param.upper - param.lower) + param.lower - else: - raise ValueError("only FloatHyperparameters supported in LHD") - - self.logger.debug("Initial Design") - configs = [] - # add middle point in space + # seeding of lhd design + np.random.seed(self.rng.randint(1,2*20)) - for design in lhd: - conf_dict = dict([(p.name,v) for p,v in zip(params,design)]) - conf = deactivate_inactive_hyperparameters(conf_dict, cs) - conf.origin = "LHD" - configs.append(conf) - self.logger.debug(conf) + constants = 0 + for p in params: + if isinstance(p, Constant): + constants += 1 - self.logger.debug("Size of lhd: %d" %(len(configs))) + lhd = pyDOE.lhs(n=len(params)-constants, samples=self.init_budget) - return configs + return self._transform_continuous_designs(design=lhd, + origin='LHD', + cs=cs) diff --git a/smac/initial_design/multi_config_initial_design.py b/smac/initial_design/multi_config_initial_design.py deleted file mode 100644 index 56545ac2d..000000000 --- a/smac/initial_design/multi_config_initial_design.py +++ /dev/null @@ -1,139 +0,0 @@ -import typing -import numpy as np - -from ConfigSpace.configuration_space import Configuration - -from smac.initial_design.initial_design import InitialDesign -from smac.initial_design.single_config_initial_design import SingleConfigInitialDesign -from smac.intensification.intensification import Intensifier - -from smac.tae.execute_ta_run import ExecuteTARun -from smac.stats.stats import Stats -from smac.utils.io.traj_logging import TrajLogger -from smac.scenario.scenario import Scenario -from smac.runhistory.runhistory import RunHistory - -__author__ = "Marius Lindauer" -__copyright__ = "Copyright 2016, ML4AAD" -__license__ = "3-clause BSD" - - -class MultiConfigInitialDesign(InitialDesign): - """ Base class for initial design strategies that evaluates multiple - configurations - - Attributes - ---------- - configs : typing.List[Configuration] - List of configurations to be evaluated - intensifier - runhistory - aggregate_func - """ - - def __init__(self, - tae_runner: ExecuteTARun, - scenario: Scenario, - stats: Stats, - traj_logger: TrajLogger, - runhistory: RunHistory, - rng: np.random.RandomState, - intensifier: Intensifier, - aggregate_func: typing.Callable, - configs: typing.Optional[typing.List[Configuration]]=None, - n_configs_x_params: int=10, - max_config_fracs: float=0.25 - ): - """Constructor - - Parameters - --------- - tae_runner: ExecuteTARun - Target algorithm execution object. - scenario: Scenario - Scenario with all meta information (including configuration space). - stats: Stats - Statistics of experiments; needed in case initial design already - exhausts the budget. - traj_logger: TrajLogger - Trajectory logging to add new incumbents found by the initial - design. - runhistory: RunHistory - Runhistory with all target algorithm runs. - rng: np.random.RandomState - Random state - intensifier: Intensifier - Intensification object to issue a racing to decide the current - incumbent. - aggregate_func: typing:Callable - Function to aggregate performance of a configuration across - instances. - configs: typing.Optional[typing.List[Configuration]] - List of initial configurations. - n_configs_x_params: int - how many configurations will be used at most in the initial design (X*D) - max_config_fracs: float - use at most X*budget in the initial design. Not active if a time limit is given. - """ - super().__init__(tae_runner=tae_runner, - scenario=scenario, - stats=stats, - traj_logger=traj_logger, - rng=rng) - - self.configs = configs - self.intensifier = intensifier - self.runhistory = runhistory - self.aggregate_func = aggregate_func - - n_params = len(self.scenario.cs.get_hyperparameters()) - self.init_budget = int(max(2, min(n_configs_x_params * n_params, - (max_config_fracs * scenario.ta_run_limit)))) - - def select_configuration(self) -> typing.List[Configuration]: - - if self.configs is None: - return self._select_configurations() - else: - return self.configs - - def run(self) -> Configuration: - """Run the initial design. - - Returns - ------- - incumbent: Configuration - Initial incumbent configuration - """ - configs = self.select_configuration() - for config in configs: - if config.origin is None: - config.origin = 'Initial design' - - # run first design - # ensures that first design is part of trajectory file - scid = SingleConfigInitialDesign(tae_runner=self.tae_runner, - scenario=self.scenario, - stats=self.stats, - traj_logger=self.traj_logger, - rng=self.rng) - - def get_config(): - return configs[0] - scid._select_configuration = get_config - scid.run() - - if len(set(configs)) > 1: - # intensify will skip all challenger that are identical with the incumbent; - # if has only identical configurations, - # intensifiy will not do any configuration runs - # (also not on the incumbent) - # therefore, at least two different configurations have to be in - inc, inc_perf = self.intensifier.intensify(challengers=configs[1:], - incumbent=configs[0], - run_history=self.runhistory, - aggregate_func=self.aggregate_func) - else: - raise ValueError('Cannot use a multiple configuration initial design with only a single configuration!') - - return inc diff --git a/smac/initial_design/multi_rand_design.py b/smac/initial_design/multi_rand_design.py deleted file mode 100644 index 38fe799b8..000000000 --- a/smac/initial_design/multi_rand_design.py +++ /dev/null @@ -1,39 +0,0 @@ -import typing - -from ConfigSpace.configuration_space import Configuration - -from smac.initial_design.multi_config_initial_design import \ - MultiConfigInitialDesign - - -__author__ = "Marius Lindauer" -__copyright__ = "Copyright 2018, ML4AAD" -__license__ = "3-clause BSD" - - -class MultiRandDesign(MultiConfigInitialDesign): - """ Initial design that evaluates multiple random configurations - - Attributes - ---------- - configs : typing.List[Configuration] - List of configurations to be evaluated - Don't pass configs to the constructor; - otherwise factorial design is overwritten - intensifier - runhistory - aggregate_func - """ - - def _select_configurations(self) -> typing.List[Configuration]: - """Selects a single configuration to run - - Returns - ------- - config: Configuration - initial incumbent configuration - """ - - cs = self.scenario.cs - self.logger.debug("Sample %d random configs for initial design" %(self.init_budget)) - return cs.sample_configuration(size=self.init_budget) diff --git a/smac/initial_design/random_configuration_design.py b/smac/initial_design/random_configuration_design.py index 3ae91c7cf..60cabd103 100644 --- a/smac/initial_design/random_configuration_design.py +++ b/smac/initial_design/random_configuration_design.py @@ -1,57 +1,19 @@ +from typing import List + from ConfigSpace import Configuration -import numpy as np -from smac.initial_design.single_config_initial_design import\ - SingleConfigInitialDesign -from smac.tae.execute_ta_run import ExecuteTARun -from smac.stats.stats import Stats -from smac.utils.io.traj_logging import TrajLogger -from smac.scenario.scenario import Scenario +from smac.initial_design.initial_design import InitialDesign + __author__ = "Katharina Eggensperger" __copyright__ = "Copyright 2016, ML4AAD" __license__ = "3-clause BSD" -class RandomConfiguration(SingleConfigInitialDesign): - """ Initial design that evaluates a single random configuration - - Attributes - ---------- - - """ +class RandomConfigurations(InitialDesign): + """Initial design that evaluates random configurations.""" - def __init__(self, - tae_runner: ExecuteTARun, - scenario: Scenario, - stats: Stats, - traj_logger: TrajLogger, - rng: np.random.RandomState - ): - """Constructor - - Parameters - ---------- - tae_runner: ExecuteTARun - Target algorithm execution object. - scenario: Scenario - Scenario with all meta information (including configuration space). - stats: Stats - Statistics of experiments; needed in case initial design already - exhausts the budget. - traj_logger: TrajLogger - Trajectory logging to add new incumbents found by the initial - design. - rng: np.random.RandomState - Random state - """ - super().__init__(tae_runner=tae_runner, - scenario=scenario, - stats=stats, - traj_logger=traj_logger, - rng=rng) - - def _select_configuration(self) -> Configuration: + def _select_configurations(self) -> List[Configuration]: """Select a random configuration. Returns @@ -59,6 +21,10 @@ def _select_configuration(self) -> Configuration: config: Configuration() Initial incumbent configuration """ - config = self.scenario.cs.sample_configuration() - config.origin = 'Random initial design.' - return config + + configs = self.scenario.cs.sample_configuration(size=self.init_budget) + if self.init_budget == 1: + configs = [configs] + for config in configs: + config.origin = 'Random initial design.' + return configs diff --git a/smac/initial_design/single_config_initial_design.py b/smac/initial_design/single_config_initial_design.py deleted file mode 100644 index f77c91b90..000000000 --- a/smac/initial_design/single_config_initial_design.py +++ /dev/null @@ -1,114 +0,0 @@ -from ConfigSpace import Configuration -import numpy as np - -from smac.initial_design.initial_design import InitialDesign -from smac.tae.execute_ta_run import ExecuteTARun, StatusType -from smac.stats.stats import Stats -from smac.utils.io.traj_logging import TrajLogger -from smac.scenario.scenario import Scenario -from smac.utils import constants -from smac.tae.execute_ta_run import FirstRunCrashedException - -__author__ = "Marius Lindauer, Katharina Eggensperger" -__copyright__ = "Copyright 2016, ML4AAD" -__license__ = "3-clause BSD" - - -class SingleConfigInitialDesign(InitialDesign): - """ Base class for initial design strategies that evaluates multiple - configurations - - Attributes - ---------- - - """ - - def __init__(self, - tae_runner: ExecuteTARun, - scenario: Scenario, - stats: Stats, - traj_logger: TrajLogger, - rng: np.random.RandomState - ): - """Constructor - - Parameters - --------- - tae_runner: ExecuteTARun - target algorithm execution object - scenario: Scenario - scenario with all meta information (including configuration space) - stats: Stats - statistics of experiments; needed in case initial design already - exhaust the budget - traj_logger: TrajLogger - trajectory logging to add new incumbents found by the initial design - rng: np.random.RandomState - random state - """ - super().__init__(tae_runner=tae_runner, - scenario=scenario, - stats=stats, - traj_logger=traj_logger, - rng=rng) - - def run(self) -> Configuration: - """Runs the initial design by calling the target algorithm - and adding new entries to the trajectory logger. - - Returns - ------- - incumbent: Configuration - Initial incumbent configuration - """ - initial_incumbent = self._select_configuration() - if initial_incumbent.origin is None: - initial_incumbent.origin = 'Initial design' - - # add this incumbent right away to have an entry to time point 0 - self.traj_logger.add_entry(train_perf=2**31, - incumbent_id=1, - incumbent=initial_incumbent) - - rand_inst = self.rng.choice(self.scenario.train_insts) - - if self.scenario.deterministic: - initial_seed = 0 - else: - initial_seed = self.rng.randint(0, constants.MAXINT) - - try: - status, cost, runtime, additional_info = self.tae_runner.start( - initial_incumbent, - instance=rand_inst, - cutoff=self.scenario.cutoff, - seed=initial_seed, - instance_specific=self.scenario.instance_specific.get(rand_inst, - "0")) - except FirstRunCrashedException as err: - if self.scenario.abort_on_first_run_crash: - raise err - else: - # TODO make it possible to add the failed run to the runhistory - if self.scenario.run_obj == "quality": - cost = self.scenario.cost_for_crash - else: - cost = self.scenario.cutoff * scenario.par_factor - - self.stats.inc_changed += 1 # first incumbent - - self.traj_logger.add_entry(train_perf=cost, - incumbent_id=self.stats.inc_changed, - incumbent=initial_incumbent) - - return initial_incumbent - - def _select_configuration(self) -> Configuration: - """Selects a single configuration to run - - Returns - ------- - config: Configuration - initial incumbent configuration - """ - raise NotImplementedError diff --git a/smac/initial_design/sobol_design.py b/smac/initial_design/sobol_design.py index 39f872600..6664417af 100644 --- a/smac/initial_design/sobol_design.py +++ b/smac/initial_design/sobol_design.py @@ -1,19 +1,18 @@ import typing -from sobol_seq import i4_sobol_generate +import sobol_seq from ConfigSpace.configuration_space import Configuration -from ConfigSpace.hyperparameters import FloatHyperparameter -from ConfigSpace.util import deactivate_inactive_hyperparameters +from ConfigSpace.hyperparameters import Constant -from smac.initial_design.multi_config_initial_design import MultiConfigInitialDesign +from smac.initial_design.initial_design import InitialDesign __author__ = "Marius Lindauer" __copyright__ = "Copyright 2018, ML4AAD" __license__ = "3-clause BSD" -class SobolDesign(MultiConfigInitialDesign): +class SobolDesign(InitialDesign): """ Sobol sequence design Attributes @@ -39,25 +38,13 @@ def _select_configurations(self) -> typing.List[Configuration]: cs = self.scenario.cs params = cs.get_hyperparameters() - sobol = i4_sobol_generate(len(params), self.init_budget) + constants = 0 + for p in params: + if isinstance(p, Constant): + constants += 1 - for idx, param in enumerate(params): - if isinstance(param, FloatHyperparameter): - sobol[:,idx] = sobol[:,idx] * (param.upper - param.lower) + param.lower - else: - raise ValueError("only FloatHyperparameters supported in SOBOL") + sobol = sobol_seq.i4_sobol_generate(len(params) - constants, self.init_budget) - self.logger.debug("Initial Design") - configs = [] - # add middle point in space - - for design in sobol: - conf_dict = dict([(p.name,v) for p,v in zip(params,design)]) - conf = deactivate_inactive_hyperparameters(conf_dict, cs) - conf.origin = "Sobol" - configs.append(conf) - self.logger.debug(conf) - - self.logger.debug("Length of Sobol sequence: %d" %(len(configs))) - - return configs + return self._transform_continuous_designs(design=sobol, + origin='Sobol', + cs=cs) diff --git a/smac/intensification/intensification.py b/smac/intensification/intensification.py index 03e32e618..88f64f750 100644 --- a/smac/intensification/intensification.py +++ b/smac/intensification/intensification.py @@ -533,4 +533,4 @@ def _compare_configs(self, incumbent: Configuration, return challenger # undecided - return None \ No newline at end of file + return None diff --git a/smac/optimizer/acquisition.py b/smac/optimizer/acquisition.py index 46f703946..9aef84f65 100644 --- a/smac/optimizer/acquisition.py +++ b/smac/optimizer/acquisition.py @@ -1,6 +1,6 @@ # encoding=utf8 import abc -import logging +import copy from typing import List import numpy as np @@ -9,6 +9,7 @@ from smac.configspace import Configuration from smac.configspace.util import convert_configurations_to_array from smac.epm.base_epm import AbstractEPM +from smac.utils.logging import PickableLoggerAdapter __author__ = "Aaron Klein, Marius Lindauer" __copyright__ = "Copyright 2017, ML4AAD" @@ -27,7 +28,7 @@ class AbstractAcquisitionFunction(object, metaclass=abc.ABCMeta): def __str__(self): return type(self).__name__ + " (" + self.long_name + ")" - def __init__(self, model: AbstractEPM, **kwargs): + def __init__(self, model: AbstractEPM): """Constructor Parameters @@ -36,8 +37,7 @@ def __init__(self, model: AbstractEPM, **kwargs): Models the objective function. """ self.model = model - self.logger = logging.getLogger( - self.__module__ + "." + self.__class__.__name__) + self.logger = PickableLoggerAdapter(self.__module__ + "." + self.__class__.__name__) def update(self, **kwargs): """Update the acquisition functions values. @@ -63,7 +63,7 @@ def __call__(self, configurations: List[Configuration]): ---------- configurations : list The configurations where the acquisition function - should be evaluated. + should be evaluated. Returns ------- @@ -101,6 +101,76 @@ def _compute(self, X: np.ndarray): raise NotImplementedError() +class IntegratedAcquisitionFunction(AbstractAcquisitionFunction): + + r"""Marginalize over Model hyperparameters to compute the integrated acquisition function. + + See "Practical Bayesian Optimization of Machine Learning Algorithms" by Jasper Snoek et al. + (https://papers.nips.cc/paper/4522-practical-bayesian-optimization-of-machine-learning-algorithms.pdf) + for further details. + """ + + def __init__(self, model: AbstractEPM, acquisition_function: AbstractAcquisitionFunction, **kwargs): + """Constructor + + Parameters + ---------- + model : AbstractEPM + The model needs to implement an additional attribute ``models`` which contains the different models to + integrate over. + kwargs + Additional keyword arguments + """ + + super().__init__(model, **kwargs) + self.long_name = 'Integrated Acquisition Function (%s)' % acquisition_function.long_name + self.acq = acquisition_function + self._functions = None + self.eta = None + + def update(self, model: AbstractEPM, **kwargs): + """Update the acquisition functions values. + + This method will be called if the model is updated. E.g. entropy search uses it to update its approximation + of P(x=x_min), EI uses it to update the current fmin. + + This implementation creates an acquisition function object for each model to integrate over and sets the + respective attributes for each acquisition function object. + + Parameters + ---------- + model : AbstractEPM + The model needs to implement an additional attribute ``models`` which contains the different models to + integrate over. + kwargs + """ + if not hasattr(model, 'models') or len(model.models) == 0: + raise ValueError('IntegratedAcquisitionFunction requires at least one model to integrate!') + if self._functions is None or len(self._functions) != len(model.models): + self._functions = [copy.deepcopy(self.acq) for _ in model.models] + for model, func in zip(model.models, self._functions): + func.update(model=model, **kwargs) + + def _compute(self, X: np.ndarray, **kwargs): + """Computes the EI value and its derivatives. + + Parameters + ---------- + X: np.ndarray(N, D), The input points where the acquisition function + should be evaluated. The dimensionality of X is (N, D), with N as + the number of points to evaluate at and D is the number of + dimensions of one X. + + Returns + ------- + np.ndarray(N,1) + Expected Improvement of X + """ + if self._functions is None: + raise ValueError('Need to call update first!') + return np.array([func._compute(X) for func in self._functions]).mean(axis=0) + + class EI(AbstractAcquisitionFunction): r"""Computes for a given x the expected improvement as @@ -112,8 +182,7 @@ class EI(AbstractAcquisitionFunction): def __init__(self, model: AbstractEPM, - par: float=0.0, - **kwargs): + par: float=0.0): """Constructor Parameters @@ -184,8 +253,7 @@ def calculate_f(): class EIPS(EI): def __init__(self, model: AbstractEPM, - par: float=0.0, - **kwargs): + par: float=0.0): r"""Computes for a given x the expected improvement as acquisition value. :math:`EI(X) := \frac{\mathbb{E}\left[ \max\{0, f(\mathbf{X^+}) - f_{t+1}(\mathbf{X}) - \xi\right] \} ]} {np.log(r(x))}`, @@ -270,8 +338,7 @@ class LogEI(AbstractAcquisitionFunction): def __init__(self, model: AbstractEPM, - par: float=0.0, - **kwargs): + par: float=0.0): r"""Computes for a given x the logarithm expected improvement as acquisition value. diff --git a/smac/optimizer/ei_optimization.py b/smac/optimizer/ei_optimization.py index 821ece8ce..dbc626dcf 100644 --- a/smac/optimizer/ei_optimization.py +++ b/smac/optimizer/ei_optimization.py @@ -1,12 +1,17 @@ import abc +import itertools import logging import time import numpy as np from typing import Iterable, List, Union, Tuple, Optional -from smac.configspace import get_one_exchange_neighbourhood, \ - convert_configurations_to_array, Configuration, ConfigurationSpace +from smac.configspace import ( + get_one_exchange_neighbourhood, + Configuration, + ConfigurationSpace, + convert_configurations_to_array, +) from smac.runhistory.runhistory import RunHistory from smac.stats.stats import Stats from smac.optimizer.acquisition import AbstractAcquisitionFunction @@ -40,7 +45,7 @@ def __init__( self, acquisition_function: AbstractAcquisitionFunction, config_space: ConfigurationSpace, - rng: Union[bool, np.random.RandomState]=None + rng: Union[bool, np.random.RandomState] = None ): self.logger = logging.getLogger( self.__module__ + "." + self.__class__.__name__ @@ -160,6 +165,14 @@ class LocalSearch(AcquisitionFunctionMaximizer): n_steps_plateau_walk: int number of steps during a plateau walk before local search terminates + vectorization_min_obtain : int + Minimal number of neighbors to obtain at once for each local search for vectorized calls. Can be tuned to + reduce the overhead of SMAC + + vectorization_max_obtain : int + Maximal number of neighbors to obtain at once for each local search for vectorized calls. Can be tuned to + reduce the overhead of SMAC + """ def __init__( @@ -167,18 +180,23 @@ def __init__( acquisition_function: AbstractAcquisitionFunction, config_space: ConfigurationSpace, rng: Union[bool, np.random.RandomState] = None, - max_steps: Optional[int]=None, - n_steps_plateau_walk: int=10, + max_steps: Optional[int] = None, + n_steps_plateau_walk: int = 10, + vectorization_min_obtain: int = 2, + vectorization_max_obtain: int = 64, ): super().__init__(acquisition_function, config_space, rng) self.max_steps = max_steps self.n_steps_plateau_walk = n_steps_plateau_walk + self.vectorization_min_obtain = vectorization_min_obtain + self.vectorization_max_obtain = vectorization_max_obtain def _maximize( self, runhistory: RunHistory, stats: Stats, num_points: int, + additional_start_points: Optional[List[Tuple[float, Configuration]]] = None, **kwargs ) -> List[Tuple[float, Configuration]]: """Starts a local search from the given startpoint and quits @@ -193,6 +211,8 @@ def _maximize( current stats object num_points: int number of points to be sampled + additional_start_points : Optional[List[Tuple[float, Configuration]]] + Additional start point ***kwargs: Additional parameters that will be passed to the acquisition function @@ -206,116 +226,211 @@ def _maximize( """ - init_points = self._get_initial_points( - num_points, runhistory) - - configs_acq = [] - # Start N local search from different random start points - for start_point in init_points: - acq_val, configuration = self._one_iter( - start_point, **kwargs) - - configuration.origin = "Local Search" - configs_acq.append((acq_val, configuration)) + init_points = self._get_initial_points(num_points, runhistory, additional_start_points) + configs_acq = self._do_search(init_points) # shuffle for random tie-break self.rng.shuffle(configs_acq) # sort according to acq value configs_acq.sort(reverse=True, key=lambda x: x[0]) + for _, inc in configs_acq: + inc.origin = 'Local Search' return configs_acq - def _get_initial_points(self, num_points, runhistory): - + def _get_initial_points(self, num_points, runhistory, additional_start_points): + if runhistory.empty(): - init_points = self.config_space.sample_configuration( - size=num_points) + init_points = self.config_space.sample_configuration(size=num_points) else: - # initiate local search with best configurations from previous runs + # initiate local search configs_previous_runs = runhistory.get_all_configs() - configs_previous_runs_sorted = self._sort_configs_by_acq_value( - configs_previous_runs) - num_configs_local_search = int(min( - len(configs_previous_runs_sorted), - num_points) - ) - init_points = list( - map(lambda x: x[1], - configs_previous_runs_sorted[:num_configs_local_search]) - ) - + + # configurations with the highest previous EI + configs_previous_runs_sorted = self._sort_configs_by_acq_value(configs_previous_runs) + configs_previous_runs_sorted = [conf[1] for conf in configs_previous_runs_sorted[:num_points]] + + # configurations with the lowest predictive cost, check for None to make unit tests work + if self.acquisition_function.model is not None: + conf_array = convert_configurations_to_array(configs_previous_runs) + costs = self.acquisition_function.model.predict_marginalized_over_instances(conf_array)[0] + # From here + # http://stackoverflow.com/questions/20197990/how-to-make-argsort-result-to-be-random-between-equal-values + random = self.rng.rand(len(costs)) + # Last column is primary sort key! + indices = np.lexsort((random.flatten(), costs.flatten())) + + # Cannot use zip here because the indices array cannot index the + # rand_configs list, because the second is a pure python list + configs_previous_runs_sorted_by_cost = [configs_previous_runs[ind] for ind in indices][:num_points] + else: + configs_previous_runs_sorted_by_cost = [] + + if additional_start_points is not None: + additional_start_points = [asp[1] for asp in additional_start_points[:num_points]] + else: + additional_start_points = [] + + init_points = [] + init_points_as_set = set() + for cand in itertools.chain( + configs_previous_runs_sorted, + configs_previous_runs_sorted_by_cost, + additional_start_points, + ): + if cand not in init_points_as_set: + init_points.append(cand) + init_points_as_set.add(cand) + return init_points - def _one_iter( + def _do_search( self, - start_point: Configuration, + start_points: List[Configuration], **kwargs - ) -> Tuple[float, Configuration]: - - incumbent = start_point - # Compute the acquisition value of the incumbent - acq_val_incumbent = self.acquisition_function([incumbent], **kwargs)[0] - - local_search_steps = 0 - neighbors_looked_at = 0 - n_no_improvements = 0 - time_n = [] - while True: - - local_search_steps += 1 - if local_search_steps % 1000 == 0: - self.logger.warning( - "Local search took already %d iterations. Is it maybe " - "stuck in a infinite loop?", local_search_steps - ) - - # Get neighborhood of the current incumbent - # by randomly drawing configurations - changed_inc = False - - # Get one exchange neighborhood returns an iterator (in contrast of - # the previously returned list). - all_neighbors = get_one_exchange_neighbourhood( - incumbent, seed=self.rng.randint(MAXINT)) + ) -> List[Tuple[float, Configuration]]: + # Gather data strucuture for starting points + if isinstance(start_points, Configuration): + start_points = [start_points] + incumbents = start_points + # Compute the acquisition value of the incumbents + num_incumbents = len(incumbents) + acq_val_incumbents = self.acquisition_function(incumbents, **kwargs) + if num_incumbents == 1: + acq_val_incumbents = [acq_val_incumbents[0][0]] + else: + acq_val_incumbents = [a[0] for a in acq_val_incumbents] + + # Set up additional variables required to do vectorized local search: + # whether the i-th local search is still running + active = [True] * num_incumbents + # number of plateau walks of the i-th local search. Reaching the maximum number is the stopping criterion of + # the local search. + n_no_plateau_walk = [0] * num_incumbents + # tracking the number of steps for logging purposes + local_search_steps = [0] * num_incumbents + # tracking the number of neighbors looked at for logging purposes + neighbors_looked_at = [0] * num_incumbents + # tracking the number of neighbors generated for logging purposse + neighbors_generated = [0] * num_incumbents + # how many neighbors were obtained for the i-th local search. Important to map the individual acquisition + # function values to the correct local search run + obtain_n = [self.vectorization_min_obtain] * num_incumbents + # Tracking the time it takes to compute the acquisition function + times = [] + + # Set up the neighborhood generators + neighborhood_iterators = [] + for i, inc in enumerate(incumbents): + neighborhood_iterators.append(get_one_exchange_neighbourhood( + inc, seed=self.rng.randint(low=0, high=100000))) + local_search_steps[i] += 1 + # Keeping track of configurations with equal acquisition value for plateau walking + neighbors_w_equal_acq = [[]] * num_incumbents + + num_iters = 0 + while np.any(active): + + num_iters += 1 + # Whether the i-th local search improved. When a new neighborhood is generated, this is used to determine + # whether a step was made (improvement) or not (iterator exhausted) + improved = [False] * num_incumbents + # Used to request a new neighborhood for the incumbent of the i-th local search + new_neighborhood = [False] * num_incumbents + + # gather all neighbors neighbors = [] - for neighbor in all_neighbors: - s_time = time.time() - acq_val = self.acquisition_function([neighbor], **kwargs) - neighbors_looked_at += 1 - time_n.append(time.time() - s_time) - - if acq_val == acq_val_incumbent: - neighbors.append(neighbor) - if acq_val > acq_val_incumbent: - self.logger.debug("Switch to one of the neighbors") - incumbent = neighbor - acq_val_incumbent = acq_val - changed_inc = True - break - - if ( - not changed_inc - and n_no_improvements < self.n_steps_plateau_walk - and len(neighbors) > 0 - ): - n_no_improvements += 1 - incumbent = neighbors[0] - changed_inc = True + for i, neighborhood_iterator in enumerate(neighborhood_iterators): + if active[i]: + neighbors_for_i = [] + for j in range(obtain_n[i]): + try: + n = next(neighborhood_iterator) + neighbors_generated[i] += 1 + neighbors_for_i.append(n) + except StopIteration: + obtain_n[i] = len(neighbors_for_i) + new_neighborhood[i] = True + break + neighbors.extend(neighbors_for_i) + + if len(neighbors) != 0: + start_time = time.time() + acq_val = self.acquisition_function(neighbors, **kwargs) + end_time = time.time() + times.append(end_time - start_time) + if np.ndim(acq_val.shape) == 0: + acq_val = [acq_val] + + # Comparing the acquisition function of the neighbors with the acquisition value of the incumbent + acq_index = 0 + # Iterating the all i local searches + for i in range(num_incumbents): + if not active[i]: + continue + # And for each local search we know how many neighbors we obtained + for j in range(obtain_n[i]): + # The next line is only true if there was an improvement and we basically need to iterate to + # the i+1-th local search + if improved[i]: + acq_index += 1 + else: + neighbors_looked_at[i] += 1 + + # Found a better configuration + if acq_val[acq_index] > acq_val_incumbents[i]: + self.logger.debug( + "Local search %d: Switch to one of the neighbors (after %d configurations).", + i, + neighbors_looked_at[i], + ) + incumbents[i] = neighbors[acq_index] + acq_val_incumbents[i] = acq_val[acq_index] + new_neighborhood[i] = True + improved[i] = True + local_search_steps[i] += 1 + neighbors_w_equal_acq[i] = [] + obtain_n[i] = 1 + # Found an equally well performing configuration, keeping it for plateau walking + elif acq_val[acq_index] == acq_val_incumbents[i]: + neighbors_w_equal_acq[i].append(neighbors[acq_index]) + + acq_index += 1 + + # Now we check whether we need to create new neighborhoods and whether we need to increase the number of + # plateau walks for one of the local searches. Also disables local searches if the number of plateau walks + # is reached (and all being switched off is the termination criterion). + for i in range(num_incumbents): + if not active[i]: + continue + if obtain_n[i] == 0 or improved[i]: + obtain_n[i] = 2 + else: + obtain_n[i] = obtain_n[i] * 2 + obtain_n[i] = min(obtain_n[i], self.vectorization_max_obtain) + if new_neighborhood[i]: + if not improved[i] and n_no_plateau_walk[i] < self.n_steps_plateau_walk: + if len(neighbors_w_equal_acq[i]) != 0: + incumbents[i] = neighbors_w_equal_acq[i][0] + neighbors_w_equal_acq[i] = [] + n_no_plateau_walk[i] += 1 + if n_no_plateau_walk[i] >= self.n_steps_plateau_walk: + active[i] = False + continue + + neighborhood_iterators[i] = get_one_exchange_neighbourhood( + incumbents[i], seed=self.rng.randint(low=0, high=100000), + ) - if (not changed_inc) or \ - (self.max_steps is not None and - local_search_steps == self.max_steps): - self.logger.debug("Local search took %d steps and looked at %d " - "configurations. Computing the acquisition " - "value for one configuration took %f seconds" - " on average.", - local_search_steps, neighbors_looked_at, - np.mean(time_n)) - break + self.logger.debug( + "Local searches took %s steps and looked at %s configurations. Computing the acquisition function in " + "vectorized for took %f seconds on average.", + local_search_steps, neighbors_looked_at, np.mean(times), + ) - return acq_val_incumbent, incumbent + return [(a, i) for a, i in zip(acq_val_incumbents, incumbents)] class DiffOpt(AcquisitionFunctionMaximizer): @@ -535,18 +650,19 @@ def maximize( to be concrete: ~smac.ei_optimization.ChallengerList """ - next_configs_by_local_search = self.local_search._maximize( - runhistory, stats, self.n_sls_iterations, **kwargs - ) - # Get configurations sorted by EI next_configs_by_random_search_sorted = self.random_search._maximize( runhistory, stats, - num_points - len(next_configs_by_local_search), + num_points, _sorted=True, ) + next_configs_by_local_search = self.local_search._maximize( + runhistory, stats, self.n_sls_iterations, additional_start_points=next_configs_by_random_search_sorted, + **kwargs + ) + # Having the configurations from random search, sorted by their # acquisition function value is important for the first few iterations # of SMAC. As long as the random forest predicts constant value, we @@ -559,8 +675,8 @@ def maximize( ) next_configs_by_acq_value.sort(reverse=True, key=lambda x: x[0]) self.logger.debug( - "First 10 acq func (origin) values of selected configurations: %s", - str([[_[0], _[1].origin] for _ in next_configs_by_acq_value[:10]]) + "First 5 acq func (origin) values of selected configurations: %s", + str([[_[0], _[1].origin] for _ in next_configs_by_acq_value[:5]]) ) next_configs_by_acq_value = [_[1] for _ in next_configs_by_acq_value] diff --git a/smac/optimizer/epils.py b/smac/optimizer/epils.py index 90e3ec7ae..1840e8879 100644 --- a/smac/optimizer/epils.py +++ b/smac/optimizer/epils.py @@ -14,8 +14,7 @@ from smac.stats.stats import Stats from smac.initial_design.initial_design import InitialDesign from smac.scenario.scenario import Scenario -from smac.configspace import Configuration, convert_configurations_to_array, \ - get_one_exchange_neighbourhood +from smac.configspace import Configuration, get_one_exchange_neighbourhood from smac.tae.execute_ta_run import FirstRunCrashedException from smac.utils.constants import MAXINT diff --git a/smac/optimizer/pSMAC.py b/smac/optimizer/pSMAC.py index 1dc0c7d09..7f848cdce 100644 --- a/smac/optimizer/pSMAC.py +++ b/smac/optimizer/pSMAC.py @@ -55,17 +55,14 @@ def read(run_history: RunHistory, new_numruns_in_runhistory = len(run_history.data) difference = new_numruns_in_runhistory - numruns_in_runhistory - logger.debug('Shared model mode: Loaded %d new runs from %s' % - (difference, runhistory_file)) + logger.debug('Shared model mode: Loaded %d new runs from %s' % (difference, runhistory_file)) numruns_in_runhistory = new_numruns_in_runhistory difference = numruns_in_runhistory - initial_numruns_in_runhistory - logger.info('Shared model mode: Finished loading new runs, found %d new ' - 'runs.' % difference) + logger.info('Shared model mode: Finished loading new runs, found %d new runs.' % difference) -def write(run_history: RunHistory, output_directory: str, - logger: logging.Logger): +def write(run_history: RunHistory, output_directory: str, logger: logging.Logger): """Write the runhistory to the output directory. Overwrites previously outputted runhistories. @@ -82,7 +79,7 @@ def write(run_history: RunHistory, output_directory: str, output_filename = os.path.join(output_directory, RUNHISTORY_FILEPATTERN) - logging.debug("Saving runhistory to %s" %(output_filename)) + logging.debug("Saving runhistory to %s" % output_filename) with tempfile.NamedTemporaryFile('wb', dir=output_directory, delete=False) as fh: diff --git a/smac/optimizer/random_configuration_chooser.py b/smac/optimizer/random_configuration_chooser.py index 24495ab9d..070894ee9 100644 --- a/smac/optimizer/random_configuration_chooser.py +++ b/smac/optimizer/random_configuration_chooser.py @@ -17,6 +17,9 @@ class RandomConfigurationChooser(ABC): random configurations in a list of challengers. """ + def __init__(self, rng: np.random.RandomState): + self.rng = rng + @abstractmethod def next_smbo_iteration(self) -> None: """Indicate beginning of next SMBO iteration""" @@ -38,7 +41,9 @@ class ChooserNoCoolDown(RandomConfigurationChooser): """ - def __init__(self, modulus: float=2.0): + def __init__(self, rng: np.random.RandomState, modulus: float = 2.0): + super().__init__(rng) + self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) if modulus <= 1.0: self.logger.warning("Using SMAC with random configurations only." @@ -54,7 +59,13 @@ def check(self, iteration) -> bool: class ChooserLinearCoolDown(RandomConfigurationChooser): - def __init__(self, start_modulus: float=2.0, modulus_increment: float=0.3, end_modulus: float=np.inf): + def __init__( + self, + rng: np.random.RandomState, + start_modulus: float = 2.0, + modulus_increment: float = 0.3, + end_modulus: float = np.inf, + ): """Interleave a random configuration, decreasing the fraction of random configurations over time. Parameters @@ -68,6 +79,8 @@ def __init__(self, start_modulus: float=2.0, modulus_increment: float=0.3, end_m further increased. If it is not reached before the optimization is over, there will be no adjustment to make sure that the ``end_modulus`` is reached. """ + super().__init__(rng) + self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) if start_modulus <= 1.0 and modulus_increment <= 0.0: self.logger.warning("Using SMAC with random configurations only. ROAR is the better choice for this.") @@ -91,7 +104,7 @@ def check(self, iteration: int) -> bool: class ChooserProb(RandomConfigurationChooser): - def __init__(self, prob: float, rng: np.random.RandomState): + def __init__(self, rng: np.random.RandomState, prob: float): """Interleave a random configuration according to a given probability. Parameters @@ -101,8 +114,8 @@ def __init__(self, prob: float, rng: np.random.RandomState): rng : np.random.RandomState Random state """ + super().__init__(rng) self.prob = prob - self.rng = rng def next_smbo_iteration(self) -> None: pass @@ -116,7 +129,7 @@ def check(self, iteration: int) -> bool: class ChooserProbCoolDown(RandomConfigurationChooser): - def __init__(self, prob: float, cool_down_fac: float, rng: np.random.RandomState): + def __init__(self, rng: np.random.RandomState, prob: float, cool_down_fac: float): """Interleave a random configuration according to a given probability which is decreased over time. Parameters @@ -128,8 +141,8 @@ def __init__(self, prob: float, cool_down_fac: float, rng: np.random.RandomState rng : np.random.RandomState Random state """ + super().__init__(rng) self.prob = prob - self.rng = rng self.cool_down_fac = cool_down_fac def next_smbo_iteration(self) -> None: @@ -160,11 +173,12 @@ class ChooserCosineAnnealing(RandomConfigurationChooser): def __init__( self, + rng: np.random.RandomState, prob_max: float, prob_min: float, restart_iteration: int, - rng: np.random.RandomState, ): + super().__init__(rng) self.logger = logging.getLogger( self.__module__ + "." + self.__class__.__name__) self.prob_max = prob_max @@ -172,14 +186,13 @@ def __init__( self.restart_iteration = restart_iteration self.iteration = 0 self.prob = prob_max - self.rng = rng def next_smbo_iteration(self) -> None: self.prob = ( self.prob_min + (0.5 * (self.prob_max - self.prob_min) * (1 + np.cos(self.iteration * np.pi / self.restart_iteration))) ) - self.logger.error("Probability for random configs: %f" % (self.prob)) + self.logger.error("Probability for random configs: %f" % self.prob) self.iteration += 1 if self.iteration > self.restart_iteration: self.iteration = 0 diff --git a/smac/optimizer/smbo.py b/smac/optimizer/smbo.py index 0b733400d..915592490 100644 --- a/smac/optimizer/smbo.py +++ b/smac/optimizer/smbo.py @@ -4,16 +4,16 @@ import time import typing -import george - +import smac from smac.configspace import ConfigurationSpace, Configuration, Constant,\ CategoricalHyperparameter, UniformFloatHyperparameter, \ UniformIntegerHyperparameter, InCondition from smac.configspace.util import convert_configurations_to_array from smac.epm.base_epm import AbstractEPM from smac.epm.rf_with_instances import RandomForestWithInstances -from smac.epm.gp_default_priors import DefaultPrior from smac.epm.gaussian_process_mcmc import GaussianProcessMCMC +from smac.epm.gp_base_prior import LognormalPrior, HorseshoePrior +from smac.epm.util_funcs import get_types from smac.initial_design.initial_design import InitialDesign from smac.intensification.intensification import Intensifier from smac.optimizer import pSMAC @@ -24,13 +24,11 @@ from smac.optimizer.ei_optimization import AcquisitionFunctionMaximizer, \ RandomSearch from smac.runhistory.runhistory import RunHistory -from smac.runhistory.runhistory2epm import AbstractRunHistory2EPM, RunHistory2EPM4LogCost +from smac.runhistory.runhistory2epm import AbstractRunHistory2EPM from smac.scenario.scenario import Scenario from smac.stats.stats import Stats -from smac.tae.execute_ta_run import FirstRunCrashedException from smac.utils.io.traj_logging import TrajLogger from smac.utils.validate import Validator -from smac.utils.util_funcs import get_types from smac.utils.constants import MAXINT @@ -146,7 +144,7 @@ def __init__(self, self._random_search = RandomSearch( acquisition_func, self.config_space, rng ) - + self.predict_incumbent = predict_incumbent def start(self): @@ -233,7 +231,7 @@ def run(self): def choose_next(self, X: np.ndarray, Y: np.ndarray, incumbent_value: float=None): - """Choose next candidate solution with Bayesian optimization. The + """Choose next candidate solution with Bayesian optimization. The suggested configurations depend on the argument ``acq_optimizer`` to the ``SMBO`` class. @@ -273,18 +271,18 @@ def choose_next(self, X: np.ndarray, Y: np.ndarray, self.acquisition_func.update(model=self.model, eta=incumbent_value, num_data=len(self.runhistory.data)) challengers = self.acq_optimizer.maximize( - runhistory=self.runhistory, - stats=self.stats, - num_points=self.scenario.acq_opt_challengers, + runhistory=self.runhistory, + stats=self.stats, + num_points=self.scenario.acq_opt_challengers, random_configuration_chooser=self.random_configuration_chooser ) return challengers - + def _get_incumbent_value(self): ''' get incumbent value either from runhistory or from best predicted performance on configs in runhistory (depends on self.predict_incumbent)" - + Return ------ float @@ -298,15 +296,18 @@ def _get_incumbent_value(self): )) incumbent_value = np.min(costs) # won't need log(y) if EPM was already trained on log(y) - + else: if self.runhistory.empty(): raise ValueError("Runhistory is empty and the cost value of " "the incumbent is unknown.") incumbent_value = self.runhistory.get_cost(self.incumbent) - if isinstance(self.rh2EPM,RunHistory2EPM4LogCost): - incumbent_value = np.log(incumbent_value) - + # It's unclear how to do this for inv scaling and potential future scaling. This line should be changed if + # necessary + incumbent_value_as_array = np.array(incumbent_value).reshape((1, 1)) + incumbent_value = self.rh2EPM.transform_response_values(incumbent_value_as_array) + incumbent_value = incumbent_value[0][0] + return incumbent_value def validate(self, config_mode='inc', instance_mode='train+test', @@ -387,67 +388,102 @@ def _get_timebound_for_intensification(self, time_spent:float): "intensification: %.4f (%.2f)" % (total_time, time_spent, (1 - frac_intensify), time_left, frac_intensify)) return time_left - + def _component_builder(self, conf:typing.Union[Configuration, dict]) \ -> typing.Tuple[AbstractAcquisitionFunction, AbstractEPM]: """ builds new Acquisition function object and EPM object and returns these - + Parameters ---------- conf: typing.Union[Configuration, dict] configuration specificing "model" and "acq_func" - + Returns ------- typing.Tuple[AbstractAcquisitionFunction, AbstractEPM] - + """ types, bounds = get_types(self.config_space, instance_features=self.scenario.feature_array) - + if conf["model"] == "RF": model = RandomForestWithInstances( - types=types, - bounds=bounds, - instance_features=self.scenario.feature_array, - seed=self.rng.randint(MAXINT), - pca_components=conf.get("pca_dim", self.scenario.PCA_DIM), - log_y=conf.get("log_y", self.scenario.transform_y in ["LOG", "LOGS"]), - num_trees=conf.get("num_trees", self.scenario.rf_num_trees), - do_bootstrapping=conf.get("do_bootstrapping", self.scenario.rf_do_bootstrapping), - ratio_features=conf.get("ratio_features", self.scenario.rf_ratio_features), - min_samples_split=conf.get("min_samples_split", self.scenario.rf_min_samples_split), - min_samples_leaf=conf.get("min_samples_leaf", self.scenario.rf_min_samples_leaf), - max_depth=conf.get("max_depth", self.scenario.rf_max_depth)) - + configspace=self.config_space, + types=types, + bounds=bounds, + instance_features=self.scenario.feature_array, + seed=self.rng.randint(MAXINT), + pca_components=conf.get("pca_dim", self.scenario.PCA_DIM), + log_y=conf.get("log_y", self.scenario.transform_y in ["LOG", "LOGS"]), + num_trees=conf.get("num_trees", self.scenario.rf_num_trees), + do_bootstrapping=conf.get("do_bootstrapping", self.scenario.rf_do_bootstrapping), + ratio_features=conf.get("ratio_features", self.scenario.rf_ratio_features), + min_samples_split=conf.get("min_samples_split", self.scenario.rf_min_samples_split), + min_samples_leaf=conf.get("min_samples_leaf", self.scenario.rf_min_samples_leaf), + max_depth=conf.get("max_depth", self.scenario.rf_max_depth), + ) + elif conf["model"] == "GP": - cov_amp = 2 - n_dims = len(types) + from smac.epm.gp_kernels import ConstantKernel, HammingKernel, WhiteKernel, Matern - initial_ls = np.ones([n_dims]) - exp_kernel = george.kernels.Matern52Kernel(initial_ls, ndim=n_dims) - kernel = cov_amp * exp_kernel + cov_amp = ConstantKernel( + 2.0, + constant_value_bounds=(np.exp(-10), np.exp(2)), + prior=LognormalPrior(mean=0.0, sigma=1.0, rng=self.rng), + ) - prior = DefaultPrior(len(kernel) + 1, rng=self.rng) + cont_dims = np.nonzero(types == 0)[0] + cat_dims = np.nonzero(types != 0)[0] + + if len(cont_dims) > 0: + exp_kernel = Matern( + np.ones([len(cont_dims)]), + [(np.exp(-10), np.exp(2)) for _ in range(len(cont_dims))], + nu=2.5, + operate_on=cont_dims, + ) + + if len(cat_dims) > 0: + ham_kernel = HammingKernel( + np.ones([len(cat_dims)]), + [(np.exp(-10), np.exp(2)) for _ in range(len(cat_dims))], + operate_on=cat_dims, + ) + noise_kernel = WhiteKernel( + noise_level=1e-8, + noise_level_bounds=(np.exp(-25), np.exp(2)), + prior=HorseshoePrior(scale=0.1, rng=self.rng), + ) - n_hypers = 3 * len(kernel) - if n_hypers % 2 == 1: - n_hypers += 1 + if len(cont_dims) > 0 and len(cat_dims) > 0: + # both + kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel + elif len(cont_dims) > 0 and len(cat_dims) == 0: + # only cont + kernel = cov_amp * exp_kernel + noise_kernel + elif len(cont_dims) == 0 and len(cat_dims) > 0: + # only cont + kernel = cov_amp * ham_kernel + noise_kernel + else: + raise ValueError() + + n_mcmc_walkers = 3 * len(kernel.theta) + if n_mcmc_walkers % 2 == 1: + n_mcmc_walkers += 1 model = GaussianProcessMCMC( + self.config_space, types=types, bounds=bounds, kernel=kernel, - prior=prior, - n_hypers=n_hypers, - chain_length=200, - burnin_steps=100, - normalize_input=True, - normalize_output=True, - rng=self.rng, + n_mcmc_walkers=n_mcmc_walkers, + chain_length=250, + burnin_steps=250, + normalize_y=True, + seed=self.rng.randint(low=0, high=10000), ) - + if conf["acq_func"] == "EI": acq = EI(model=model, par=conf.get("par_ei", 0)) @@ -461,53 +497,56 @@ def _component_builder(self, conf:typing.Union[Configuration, dict]) \ # par value should be in log-space acq = LogEI(model=model, par=conf.get("par_logei", 0)) - + return acq, model - + def _get_acm_cs(self): """ - returns a configuration space + returns a configuration space designed for querying ~smac.optimizer.smbo._component_builder - + Returns - ------- + ------- ConfigurationSpace """ - + cs = ConfigurationSpace() cs.seed(self.rng.randint(0,2**20)) - - model = CategoricalHyperparameter("model", choices=("RF", "GP")) - + + if 'gp' in smac.extras_installed: + model = CategoricalHyperparameter("model", choices=("RF", "GP")) + else: + model = Constant("model", value="RF") + num_trees = Constant("num_trees", value=10) bootstrap = CategoricalHyperparameter("do_bootstrapping", choices=(True, False), default_value=True) ratio_features = CategoricalHyperparameter("ratio_features", choices=(3 / 6, 4 / 6, 5 / 6, 1), default_value=1) min_split = UniformIntegerHyperparameter("min_samples_to_split", lower=1, upper=10, default_value=2) min_leaves = UniformIntegerHyperparameter("min_samples_in_leaf", lower=1, upper=10, default_value=1) - + cs.add_hyperparameters([model, num_trees, bootstrap, ratio_features, min_split, min_leaves]) - + inc_num_trees = InCondition(num_trees, model, ["RF"]) inc_bootstrap = InCondition(bootstrap, model, ["RF"]) inc_ratio_features = InCondition(ratio_features, model, ["RF"]) inc_min_split = InCondition(min_split, model, ["RF"]) inc_min_leavs = InCondition(min_leaves, model, ["RF"]) - + cs.add_conditions([inc_num_trees, inc_bootstrap, inc_ratio_features, inc_min_split, inc_min_leavs]) - + acq = CategoricalHyperparameter("acq_func", choices=("EI", "LCB", "PI", "LogEI")) par_ei = UniformFloatHyperparameter("par_ei", lower=-10, upper=10) par_pi = UniformFloatHyperparameter("par_pi", lower=-10, upper=10) par_logei = UniformFloatHyperparameter("par_logei", lower=0.001, upper=100, log=True) par_lcb = UniformFloatHyperparameter("par_lcb", lower=0.0001, upper=0.9999) - + cs.add_hyperparameters([acq, par_ei, par_pi, par_logei, par_lcb]) - + inc_par_ei = InCondition(par_ei, acq, ["EI"]) inc_par_pi = InCondition(par_pi, acq, ["PI"]) inc_par_logei = InCondition(par_logei, acq, ["LogEI"]) inc_par_lcb = InCondition(par_lcb, acq, ["LCB"]) - + cs.add_conditions([inc_par_ei, inc_par_pi, inc_par_logei, inc_par_lcb]) - + return cs diff --git a/smac/requirements.txt b/smac/requirements.txt new file mode 120000 index 000000000..dc833dd4b --- /dev/null +++ b/smac/requirements.txt @@ -0,0 +1 @@ +../requirements.txt \ No newline at end of file diff --git a/smac/runhistory/runhistory2epm.py b/smac/runhistory/runhistory2epm.py index f5d36ff2b..6bc433588 100644 --- a/smac/runhistory/runhistory2epm.py +++ b/smac/runhistory/runhistory2epm.py @@ -46,11 +46,12 @@ def __init__( self, scenario: Scenario, num_params: int, - success_states: typing.Optional[typing.List[StatusType]]=None, - impute_censored_data: bool=False, - impute_state: typing.Optional[typing.List[StatusType]]=None, - imputor: typing.Optional[BaseImputor]=None, - rng: typing.Optional[np.random.RandomState]=None, + success_states: typing.Optional[typing.List[StatusType]] = None, + impute_censored_data: bool = False, + impute_state: typing.Optional[typing.List[StatusType]] = None, + imputor: typing.Optional[BaseImputor] = None, + scale_perc: int = 5, + rng: typing.Optional[np.random.RandomState] = None, ) -> None: """Constructor @@ -71,6 +72,9 @@ def __init__( List of states that mark censored data (such as StatusType.TIMEOUT) in combination with runtime < cutoff_time If None, set to [StatusType.CAPPED, ] + scale_perc: int + scaled y-transformation use a percentile to estimate distance to optimum; + only used by some subclasses of AbstractRunHistory2EPM rng : numpy.random.RandomState only used for reshuffling data after imputation """ @@ -82,6 +86,7 @@ def __init__( self.scenario = scenario self.rng = rng self.num_params = num_params + self.scale_perc = scale_perc # Configuration self.success_states = success_states @@ -106,7 +111,6 @@ def __init__( self.num_params = num_params # Sanity checks - # TODO: Decide whether we need this if impute_censored_data and scenario.run_obj != "runtime": # So far we don't know how to handle censored quality data self.logger.critical("Cannot impute censored data when not " @@ -125,11 +129,17 @@ def __init__( "smac.epm.base_imputor.BaseImputor, but %s" % type(self.imputor)) + # Learned statistics + self.min_y = None + self.max_y = None + self.perc = None + @abc.abstractmethod def _build_matrix(self, run_dict: typing.Mapping[RunKey, RunValue], runhistory: RunHistory, - instances: list=None, - par_factor: int=1): + instances: list = None, + return_time_as_y: bool = False, + store_statistics: bool = False): """Builds x,y matrixes from selected runs from runhistory Parameters @@ -140,8 +150,10 @@ def _build_matrix(self, run_dict: typing.Mapping[RunKey, RunValue], runhistory object instances: list list of instances - par_factor: int - penalization factor for censored runtime data + return_time_as_y: bool + Return the time instead of cost as y value. Necessary to access the raw y values for imputation. + store_statistics: bool + Whether to store statistics about the data (to be used at subsequent calls) Returns ------- @@ -175,7 +187,7 @@ def transform(self, runhistory: RunHistory): # Store a list of instance IDs s_instance_id_list = [k.instance_id for k in s_run_dict.keys()] X, Y = self._build_matrix(run_dict=s_run_dict, runhistory=runhistory, - instances=s_instance_id_list) + instances=s_instance_id_list, store_statistics=True) # Also get TIMEOUT runs t_run_dict = {run: runhistory.data[run] for run in runhistory.data.keys() @@ -184,9 +196,9 @@ def transform(self, runhistory: RunHistory): t_instance_id_list = [k.instance_id for k in s_run_dict.keys()] # use penalization (e.g. PAR10) for EPM training + store_statistics = True if self.min_y is None else False tX, tY = self._build_matrix(run_dict=t_run_dict, runhistory=runhistory, - instances=t_instance_id_list, - par_factor=self.scenario.par_factor) + instances=t_instance_id_list, store_statistics=store_statistics) # if we don't have successful runs, # we have to return all timeout runs @@ -212,13 +224,15 @@ def transform(self, runhistory: RunHistory): cen_X, cen_Y = self._build_matrix(run_dict=c_run_dict, runhistory=runhistory, instances=c_instance_id_list, - par_factor=1) + return_time_as_y=True, + store_statistics=False,) # Also impute TIMEOUTS tX, tY = self._build_matrix(run_dict=t_run_dict, runhistory=runhistory, instances=t_instance_id_list, - par_factor=1) + return_time_as_y=True, + store_statistics=False,) cen_X = np.vstack((cen_X, tX)) cen_Y = np.concatenate((cen_Y, tY)) self.logger.debug("%d TIMOUTS, %d censored, %d regular" % @@ -240,6 +254,21 @@ def transform(self, runhistory: RunHistory): self.logger.debug("Converted %d observations" % (X.shape[0])) return X, Y + @abc.abstractmethod + def transform_response_values(self, values: np.ndarray, ) -> np.ndarray: + """Transform function response values. + + Parameters + ---------- + values : np.ndarray + Response values to be transformed. + + Returns + ------- + np.ndarray + """ + raise NotImplementedError + def get_X_y(self, runhistory: RunHistory): """Simple interface to obtain all data in runhistory in X, y format @@ -278,8 +307,10 @@ class RunHistory2EPM4Cost(AbstractRunHistory2EPM): """TODO""" def _build_matrix(self, run_dict: typing.Mapping[RunKey, RunValue], - runhistory: RunHistory, instances: typing.List[str]=None, - par_factor: int=1): + runhistory: RunHistory, + instances: list = None, + return_time_as_y: bool = False, + store_statistics: bool = False): """"Builds X,y matrixes from selected runs from runhistory Parameters @@ -290,8 +321,10 @@ def _build_matrix(self, run_dict: typing.Mapping[RunKey, RunValue], runhistory object instances: list list of instances - par_factor: int - penalization factor for censored runtime data + return_time_as_y: bool + Return the time instead of cost as y value. Necessary to access the raw y values for imputation. + store_statistics: bool + Whether to store statistics about the data (to be used at subsequent calls) Returns ------- @@ -316,232 +349,209 @@ def _build_matrix(self, run_dict: typing.Mapping[RunKey, RunValue], else: X[row, :] = conf_vector # run_array[row, -1] = instances[row] - if self.scenario.run_obj == "runtime": - if run.status != StatusType.SUCCESS: - y[row, 0] = run.time * par_factor - else: - y[row, 0] = run.time + if return_time_as_y: + y[row, 0] = run.time else: y[row, 0] = run.cost + if y.size > 0: + if store_statistics: + self.perc = np.percentile(y, self.scale_perc) + self.min_y = np.min(y) + self.max_y = np.max(y) + y = self.transform_response_values(values=y) + return X, y + def transform_response_values(self, values: np.ndarray) -> np.ndarray: + """Transform function response values. + + Returns the input values. + + Parameters + ---------- + values : np.ndarray + Response values to be transformed. + + Returns + ------- + np.ndarray + """ + return values + class RunHistory2EPM4LogCost(RunHistory2EPM4Cost): """TODO""" - def _build_matrix(self, run_dict: typing.Mapping[RunKey, RunValue], - runhistory: RunHistory, instances: typing.List[str]=None, - par_factor: int=1): - """Builds X,y matrices from selected runs from runhistory; transforms - y by using log + def transform_response_values(self, values: np.ndarray) -> np.ndarray: + """Transform function response values. + + Transforms the response values by using a log transformation. Parameters ---------- - run_dict: dict(RunKey -> RunValue) - Dictionary from RunHistory.RunKey to RunHistory.RunValue - runhistory: RunHistory - Runhistory object - instances: list - List of instances - par_factor: int - Penalization factor for censored runtime data + values : np.ndarray + Response values to be transformed. Returns ------- - X: np.ndarray - Y: np.ndarray + np.ndarray """ - X, y = super()._build_matrix(run_dict=run_dict, runhistory=runhistory, - instances=instances, par_factor=par_factor) + # ensure that minimal value is larger than 0 - if np.any(y <= 0): + if np.any(values <= 0): self.logger.warning( "Got cost of smaller/equal to 0. Replace by %f since we use" - " log cost." % (constants.MINIMAL_COST_FOR_LOG)) - y[y < constants.MINIMAL_COST_FOR_LOG] =\ + " log cost." % constants.MINIMAL_COST_FOR_LOG) + values[values < constants.MINIMAL_COST_FOR_LOG] = \ constants.MINIMAL_COST_FOR_LOG - y = np.log(y) + values = np.log(values) + return values - return X, y class RunHistory2EPM4ScaledCost(RunHistory2EPM4Cost): """TODO""" - def _build_matrix(self, run_dict: typing.Mapping[RunKey, RunValue], - runhistory: RunHistory, instances: typing.List[str]=None, - par_factor: int=1): - """Builds X,y matrices from selected runs from runhistory; transforms - y by linearly scaling + def transform_response_values(self, values: np.ndarray) -> np.ndarray: + """Transform function response values. + + Transforms the response values by linearly scaling them between zero and one. Parameters ---------- - run_dict: dict(RunKey -> RunValue) - Dictionary from RunHistory.RunKey to RunHistory.RunValue - runhistory: RunHistory - Runhistory object - instances: list - List of instances - par_factor: int - Penalization factor for censored runtime data + values : np.ndarray + Response values to be transformed. Returns ------- - X: np.ndarray - Y: np.ndarray + np.ndarray """ - X, y = super()._build_matrix(run_dict=run_dict, runhistory=runhistory, - instances=instances, par_factor=par_factor) - if y.size > 0: - perc = np.percentile(y, 5) - min_y = 2 * np.min(y) - perc # ensure that scaled y cannot be 0 - max_y = np.max(y) - # linear scaling - if min_y == max_y: - # prevent diving by zero - min_y *= 1 - 10**-101 - y = (y - min_y) / (max_y - min_y) + min_y = self.min_y - (self.perc - self.min_y) # Subtract the difference between the percentile and the minimum + # linear scaling + if self.min_y == self.max_y: + # prevent diving by zero + min_y *= 1 - 10 ** -101 + values = (values - min_y) / (self.max_y - min_y) + return values - return X, y class RunHistory2EPM4InvScaledCost(RunHistory2EPM4Cost): """TODO""" + def __init__(self, **kwargs): + super().__init__(**kwargs) + if self.instance_features is not None: + if len(self.instance_features) > 1: + raise NotImplementedError('Handling more than one instance is not supported for inverse scaled cost.') - def _build_matrix(self, run_dict: typing.Mapping[RunKey, RunValue], - runhistory: RunHistory, instances: typing.List[str]=None, - par_factor: int=1): - """Builds X,y matrices from selected runs from runhistory; transforms - y by linearly scaling and using inverse + def transform_response_values(self, values: np.ndarray) -> np.ndarray: + """Transform function response values. + + Transform the response values by linearly scaling them between zero and one and then using inverse scaling. Parameters ---------- - run_dict: dict(RunKey -> RunValue) - Dictionary from RunHistory.RunKey to RunHistory.RunValue - runhistory: RunHistory - Runhistory object - instances: list - List of instances - par_factor: int - Penalization factor for censored runtime data + values : np.ndarray + Response values to be transformed. Returns ------- - X: np.ndarray - Y: np.ndarray + np.ndarray """ - X, y = super()._build_matrix(run_dict=run_dict, runhistory=runhistory, - instances=instances, par_factor=par_factor) - if y.size > 0: - perc = np.percentile(y, 5) - min_y = 2 * np.min(y) - perc # ensure that scaled y cannot be 0 - max_y = np.max(y) - # linear scaling - if min_y == max_y: - # prevent diving by zero - min_y *= 1 - 10**-10 - y = (y - min_y) / (max_y - min_y) - y = 1 - 1/y + min_y = self.min_y - (self.perc - self.min_y) # Subtract the difference between the percentile and the minimum + min_y -= constants.VERY_SMALL_NUMBER # Minimal value to avoid numerical issues in the log scaling below + # linear scaling + if min_y == self.max_y: + # prevent diving by zero + min_y *= 1 - 10 ** -10 + values = (values - min_y) / (self.max_y - min_y) + values = 1 - 1 / values + return values + - return X, y - class RunHistory2EPM4SqrtScaledCost(RunHistory2EPM4Cost): """TODO""" + def __init__(self, **kwargs): + super().__init__(**kwargs) + if self.instance_features is not None: + if len(self.instance_features) > 1: + raise NotImplementedError('Handling more than one instance is not supported for sqrt scaled cost.') - def _build_matrix(self, run_dict: typing.Mapping[RunKey, RunValue], - runhistory: RunHistory, instances: typing.List[str]=None, - par_factor: int=1): - """Builds X,y matrices from selected runs from runhistory; transforms - y by linearly scaling and using sqrt + def transform_response_values(self, values: np.ndarray) -> np.ndarray: + """Transform function response values. + + Transform the response values by linearly scaling them between zero and one and then using the square root. Parameters ---------- - run_dict: dict(RunKey -> RunValue) - Dictionary from RunHistory.RunKey to RunHistory.RunValue - runhistory: RunHistory - Runhistory object - instances: list - List of instances - par_factor: int - Penalization factor for censored runtime data + values : np.ndarray + Response values to be transformed. Returns ------- - X: np.ndarray - Y: np.ndarray + np.ndarray """ - X, y = super()._build_matrix(run_dict=run_dict, runhistory=runhistory, - instances=instances, par_factor=par_factor) - if y.size > 0: - perc = np.percentile(y, 5) - min_y = 2 * np.min(y) - perc # ensure that scaled y cannot be 0 - max_y = np.max(y) - # linear scaling - if min_y == max_y: - # prevent diving by zero - min_y *= 1 - 10**-10 - y = (y - min_y) / (max_y - min_y) - y = np.sqrt(y) + min_y = self.min_y - (self.perc - self.min_y) # Subtract the difference between the percentile and the minimum + # linear scaling + if min_y == self.max_y: + # prevent diving by zero + min_y *= 1 - 10 ** -10 + values = (values - min_y) / (self.max_y - min_y) + values = np.sqrt(values) + return values - return X, y class RunHistory2EPM4LogScaledCost(RunHistory2EPM4Cost): """TODO""" - def _build_matrix(self, run_dict: typing.Mapping[RunKey, RunValue], - runhistory: RunHistory, instances: typing.List[str]=None, - par_factor: int=1): - """Builds X,y matrices from selected runs from runhistory; transforms - y by linearly scaling and using log + def transform_response_values(self, values: np.ndarray) -> np.ndarray: + """Transform function response values. + + Transform the response values by linearly scaling them between zero and one and then using the log + transformation. Parameters ---------- - run_dict: dict(RunKey -> RunValue) - Dictionary from RunHistory.RunKey to RunHistory.RunValue - runhistory: RunHistory - Runhistory object - instances: list - List of instances - par_factor: int - Penalization factor for censored runtime data + values : np.ndarray + Response values to be transformed. Returns ------- - X: np.ndarray - Y: np.ndarray + np.ndarray """ - X, y = super()._build_matrix(run_dict=run_dict, runhistory=runhistory, - instances=instances, par_factor=par_factor) - if y.size > 0: - perc = np.percentile(y, 5) - min_y = 2 * np.min(y) - perc # ensure that scaled y cannot be 0 - max_y = np.max(y) - # linear scaling - if min_y == max_y: - # prevent diving by zero - min_y *= 1 - 10**-10 - y = (y - min_y) / (max_y - min_y) - y = np.log(y) - - return X, y + min_y = self.min_y - (self.perc - self.min_y) # Subtract the difference between the percentile and the minimum + min_y -= constants.VERY_SMALL_NUMBER # Minimal value to avoid numerical issues in the log scaling below + # linear scaling + if min_y == self.max_y: + # prevent diving by zero + min_y *= 1 - 10 ** -10 + values = (values - min_y) / (self.max_y - min_y) + values = np.log(values) + return values class RunHistory2EPM4EIPS(AbstractRunHistory2EPM): """TODO""" def _build_matrix(self, run_dict: typing.Mapping[RunKey, RunValue], - runhistory: RunHistory, instances: typing.List[str]=None, - par_factor: int=1): + runhistory: RunHistory, instances: typing.List[str] = None, + return_time_as_y: bool = False, + store_statistics: bool = False): """TODO""" + if return_time_as_y: + raise NotImplementedError() + if store_statistics: + raise NotImplementedError() + # First build nan-matrix of size #configs x #params+1 n_rows = len(run_dict) n_cols = self.num_params X = np.ones([n_rows, n_cols + self.n_feats]) * np.nan - Y = np.ones([n_rows, 2]) + y = np.ones([n_rows, 2]) # Then populate matrix for row, (key, run) in enumerate(run_dict.items()): @@ -553,8 +563,27 @@ def _build_matrix(self, run_dict: typing.Mapping[RunKey, RunValue], X[row, :] = np.hstack((conf_vector, feats)) else: X[row, :] = conf_vector - # run_array[row, -1] = instances[row] - Y[row, 0] = run.cost - Y[row, 1] = np.log(1 + run.time) + y[row, 0] = run.cost + y[row, 1] = 1 + run.time - return X, Y + y = self.transform_response_values(values=y) + + return X, y + + def transform_response_values(self, values: np.ndarray): + """Transform function response values. + + Transform the runtimes by a log transformation (log(1 + runtime). + + Parameters + ---------- + values : np.ndarray + Response values to be transformed. + + Returns + ------- + np.ndarray + """ + + values[:, 1] = np.log(1 + values[:, 1]) + return values diff --git a/smac/smac_cli.py b/smac/smac_cli.py index c1ddc6a1a..14053f0d7 100644 --- a/smac/smac_cli.py +++ b/smac/smac_cli.py @@ -6,13 +6,13 @@ from smac.utils.io.cmd_reader import CMDReader from smac.scenario.scenario import Scenario -from smac.facade.smac_facade import SMAC +from smac.facade.smac_ac_facade import SMAC4AC from smac.facade.roar_facade import ROAR -from smac.facade.epils_facade import EPILS -from smac.facade.hydra_facade import Hydra -from smac.facade.psmac_facade import PSMAC -from smac.facade.borf_facade import BORF -from smac.facade.bogp_facade import BOGP +from smac.facade.experimental.epils_facade import EPILS +from smac.facade.experimental.hydra_facade import Hydra +from smac.facade.experimental.psmac_facade import PSMAC +from smac.facade.smac_hpo_facade import SMAC4HPO +from smac.facade.smac_bo_facade import SMAC4BO from smac.runhistory.runhistory import RunHistory from smac.stats.stats import Stats from smac.optimizer.objective import average_cost @@ -108,8 +108,8 @@ def main_cli(self, commandline_arguments: typing.List[str]=None): fn=traj_fn, cs=scen.cs) initial_configs.append(trajectory[-1]["incumbent"]) - if main_args_.mode == "SMAC": - optimizer = SMAC( + if main_args_.mode == "SMAC4AC": + optimizer = SMAC4AC( scenario=scen, rng=np.random.RandomState(main_args_.seed), runhistory=rh, @@ -117,8 +117,8 @@ def main_cli(self, commandline_arguments: typing.List[str]=None): stats=stats, restore_incumbent=incumbent, run_id=main_args_.seed) - elif main_args_.mode == "BORF": - optimizer = BORF( + elif main_args_.mode == "SMAC4HPO": + optimizer = SMAC4HPO( scenario=scen, rng=np.random.RandomState(main_args_.seed), runhistory=rh, @@ -126,8 +126,8 @@ def main_cli(self, commandline_arguments: typing.List[str]=None): stats=stats, restore_incumbent=incumbent, run_id=main_args_.seed) - elif main_args_.mode == "BOGP": - optimizer = BOGP( + elif main_args_.mode == "SMAC4BO": + optimizer = SMAC4BO( scenario=scen, rng=np.random.RandomState(main_args_.seed), runhistory=rh, diff --git a/smac/utils/constants.py b/smac/utils/constants.py index bdee47fcf..a7f6887fc 100644 --- a/smac/utils/constants.py +++ b/smac/utils/constants.py @@ -1,5 +1,7 @@ MAXINT = 2 ** 31 - 1 -MINIMAL_COST_FOR_LOG = 0.00001 +MINIMAL_COST_FOR_LOG = 0.00001 MAX_CUTOFF = 65535 +# The very small number is used to avoid numerical instabilities or dividing by zero +VERY_SMALL_NUMBER = 1e-10 N_TREES = 10 diff --git a/smac/utils/dependencies.py b/smac/utils/dependencies.py index cb1ffd67b..68e0c971b 100644 --- a/smac/utils/dependencies.py +++ b/smac/utils/dependencies.py @@ -8,6 +8,15 @@ r'^(?P[\w\-]+)%s?(,%s)?$' % (SUBPATTERN % (1, 1), SUBPATTERN % (2, 2))) +def are_valid_packages(packages): + try: + verify_packages(packages) + except (MissingPackageError, IncorrectPackageVersionError): + return False + else: + return True + + def verify_packages(packages): if not packages: return diff --git a/smac/utils/io/cmd_reader.py b/smac/utils/io/cmd_reader.py index 74d6f2f7b..87c0b479e 100644 --- a/smac/utils/io/cmd_reader.py +++ b/smac/utils/io/cmd_reader.py @@ -6,13 +6,12 @@ import re import shlex import sys -from smac.configspace import pcs, pcs_new -from smac.configspace import json as pcs_json -from smac.utils.constants import MAXINT, N_TREES -from smac.utils.io.input_reader import InputReader import time import typing +from smac.utils.constants import MAXINT, N_TREES +from smac.utils.io.input_reader import InputReader + __author__ = "Marius Lindauer" __copyright__ = "Copyright 2018, ML4AAD" __license__ = "3-clause BSD" @@ -21,6 +20,7 @@ parsed_scen_args = {} logger = None + def truthy(x): """Convert x into its truth value""" if isinstance(x, bool): @@ -134,19 +134,8 @@ def __call__(self, parser: ArgumentParser, namespace: Namespace, values: list, o fn = values if fn: if os.path.isfile(fn): - # Three possible formats: json, pcs and pcs_new. We prefer json. - with open(fn) as fp: - if fn.endswith('.json'): - parsed_scen_args['cs'] = pcs_json.read(fp.read()) - logger.debug("Loading pcs as json from: %s", fn) - else: - pcs_str = fp.readlines() - try: - parsed_scen_args["cs"] = pcs.read(pcs_str) - except: - logger.debug("Could not parse pcs file with old format; trying new format ...") - parsed_scen_args["cs"] = pcs_new.read(pcs_str) - parsed_scen_args["cs"].seed(42) + parsed_scen_args['cs'] = in_reader.read_pcs_file(fn) + parsed_scen_args["cs"].seed(42) else: parser.exit(1, "Could not find pcs file: {}".format(fn)) setattr(namespace, self.dest, values) @@ -380,7 +369,8 @@ def _add_main_options(self): default=logging.INFO, choices=["INFO", "DEBUG"], help="Verbosity level.") opt_opts.add_argument("--mode", - default="SMAC", choices=["SMAC", "ROAR", "EPILS", "Hydra", "PSMAC", "BORF", "BOGP"], + default="SMAC4AC", choices=["SMAC4AC", "ROAR", "EPILS", "Hydra", "PSMAC", "SMAC4HPO", + "SMAC4BO"], help="Configuration mode.") opt_opts.add_argument("--restore-state", "--restore_state", dest="restore_state", default=None, @@ -573,8 +563,9 @@ def _add_scen_options(self): help="[dev] Specifies the path to the execution-directory.") scen_opts.add_argument("--deterministic", dest="deterministic", default=False, type=truthy, - help="[dev] If true, the optimization process will be " - "repeatable.") + help="[dev] If true, SMAC assumes that the target function or algorithm is deterministic" + " (the same static seed of 0 is always passed to the function/algorithm)." + " If false, different random seeds are passed to the target function/algorithm.") scen_opts.add_argument("--run-obj", "--run_obj", dest="run_obj", type=str, action=ProcessRunObjectiveAction, required=True, choices=['runtime', 'quality'], diff --git a/smac/utils/io/input_reader.py b/smac/utils/io/input_reader.py index 794aa825d..de7a723c3 100644 --- a/smac/utils/io/input_reader.py +++ b/smac/utils/io/input_reader.py @@ -1,5 +1,6 @@ import numpy as np -from smac.configspace import pcs +from smac.configspace import pcs, pcs_new +from smac.configspace import json as pcs_json __author__ = "Marius Lindauer" __copyright__ = "Copyright 2015, ML4AAD" @@ -172,8 +173,11 @@ def read_instance_features_file(self, fn: str): instances[tmp[0]] = np.array(tmp[1:], dtype=np.double) return [f.strip() for f in lines[0].rstrip("\n").split(",")[1:]], instances - def read_pcs_file(self, fn: str): - """Encapsulates generating configuration space object + @staticmethod + def read_pcs_file(fn: str, logger=None): + """Encapsulates generating configuration space object from file. + + Automatically detects whether the cs is saved in json, pcs or pcs_new. Parameters ---------- @@ -184,5 +188,18 @@ def read_pcs_file(self, fn: str): ------- ConfigSpace: ConfigSpace """ - space = pcs.read(fn) - return space + # Three possible formats: json, pcs and pcs_new. We prefer json. + with open(fn) as fp: + if fn.endswith('.json'): + cs = pcs_json.read(fp.read()) + if logger: + logger.debug("Loading pcs as json from: %s", fn) + else: + pcs_str = fp.readlines() + try: + cs = pcs.read(pcs_str) + except NotImplementedError: + if logger: + logger.debug("Could not parse pcs file with old format; trying new format ...") + cs = pcs_new.read(pcs_str) + return cs diff --git a/smac/utils/io/output_directory.py b/smac/utils/io/output_directory.py index 894cddb78..37130ceab 100644 --- a/smac/utils/io/output_directory.py +++ b/smac/utils/io/output_directory.py @@ -1,5 +1,4 @@ from logging import Logger -import typing import os import shutil @@ -8,9 +7,9 @@ def create_output_directory( - scenario: typing.Type[Scenario], + scenario: Scenario, run_id: int, - logger: Logger=None, + logger: Logger = None, ): """Create output directory for this run. diff --git a/smac/utils/validate.py b/smac/utils/validate.py index e6095857e..1f5f3a056 100644 --- a/smac/utils/validate.py +++ b/smac/utils/validate.py @@ -10,6 +10,7 @@ from smac.configspace import Configuration, convert_configurations_to_array from smac.epm.rf_with_instances import RandomForestWithInstances from smac.epm.rfr_imputator import RFRImputator +from smac.epm.util_funcs import get_types from smac.optimizer.objective import average_cost from smac.runhistory.runhistory import RunHistory, RunKey, StatusType from smac.runhistory.runhistory2epm import RunHistory2EPM4Cost @@ -18,7 +19,6 @@ from smac.tae.execute_ta_run import ExecuteTARun from smac.tae.execute_ta_run_old import ExecuteTARunOld from smac.utils.constants import MAXINT -from smac.utils.util_funcs import get_types __author__ = "Joshua Marben" __copyright__ = "Copyright 2017, ML4AAD" @@ -284,11 +284,14 @@ def validate_epm(self, elif reuse_epm is False or self.epm is None: # Create RandomForest types, bounds = get_types(self.scen.cs, self.scen.feature_array) - self.epm = RandomForestWithInstances(types=types, - bounds=bounds, - instance_features=self.scen.feature_array, - seed=self.rng.randint(MAXINT), - ratio_features=1.0) + self.epm = RandomForestWithInstances( + configspace=self.scen.cs, + types=types, + bounds=bounds, + instance_features=self.scen.feature_array, + seed=self.rng.randint(MAXINT), + ratio_features=1.0, + ) # Use imputor if objective is runtime imputor = None impute_state = None @@ -553,7 +556,7 @@ def _get_configs(self, mode:str) -> typing.List[str]: configs.append(entry["incumbent"]) counter *= 2 if not self.traj[0]["incumbent"] in configs: - configs.append(traj[0]["incumbent"]) # add first + configs.append(self.traj[0]["incumbent"]) # add first if mode == "all": for entry in self.traj: if not entry["incumbent"] in configs: diff --git a/test/__init__.py b/test/__init__.py index e69de29bb..af427d41d 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -0,0 +1,8 @@ +import unittest + +from smac import extras_require, extras_installed + + +def requires_extra(name): + return unittest.skipUnless(name in extras_installed, + 'requires {}'.format(extras_require[name])) diff --git a/test/test_cli/test_deterministic_smac.py b/test/test_cli/test_deterministic_smac.py index 50a522fe7..067ed70b5 100644 --- a/test/test_cli/test_deterministic_smac.py +++ b/test/test_cli/test_deterministic_smac.py @@ -1,6 +1,5 @@ import json import os -import sys import unittest import shutil from nose.plugins.attrib import attr @@ -8,7 +7,6 @@ from unittest import mock from smac.smac_cli import SMACCLI -from smac.optimizer import ei_optimization from ConfigSpace.util import get_one_exchange_neighbourhood @@ -88,27 +86,27 @@ def test_modes(self): "--seed", "2", "--random_configuration_chooser", "test/test_cli/random_configuration_chooser_impl.py", "--output_dir", self.output_dir_3, - "--mode", 'SMAC'] + "--mode", 'SMAC4AC'] cli = SMACCLI() - with mock.patch("smac.smac_cli.SMAC") as MSMAC: + with mock.patch("smac.smac_cli.SMAC4AC") as MSMAC: MSMAC.return_value.optimize.return_value = True cli.main_cli(testargs[1:]) MSMAC.assert_called_once_with( initial_configurations=None, restore_incumbent=None, run_id=2, runhistory=None, stats=None, scenario=mock.ANY, rng=mock.ANY) - testargs[-1] = 'BOGP' + testargs[-1] = 'SMAC4BO' cli = SMACCLI() - with mock.patch("smac.smac_cli.BOGP") as MSMAC: + with mock.patch("smac.smac_cli.SMAC4BO") as MSMAC: MSMAC.return_value.optimize.return_value = True cli.main_cli(testargs[1:]) MSMAC.assert_called_once_with( initial_configurations=None, restore_incumbent=None, run_id=2, runhistory=None, stats=None, scenario=mock.ANY, rng=mock.ANY) - testargs[-1] = 'BORF' + testargs[-1] = 'SMAC4HPO' cli = SMACCLI() - with mock.patch("smac.smac_cli.BORF") as MSMAC: + with mock.patch("smac.smac_cli.SMAC4HPO") as MSMAC: MSMAC.return_value.optimize.return_value = True cli.main_cli(testargs[1:]) MSMAC.assert_called_once_with( @@ -135,4 +133,4 @@ def test_modes(self): MSMAC.assert_called_once_with( run_id=2, scenario=mock.ANY, rng=mock.ANY, n_incs=1, n_optimizers=1, shared_model=False, validate=False - ) \ No newline at end of file + ) diff --git a/test/test_cli/test_restore_state.py b/test/test_cli/test_restore_state.py index cd54172cf..d1b0f688e 100644 --- a/test/test_cli/test_restore_state.py +++ b/test/test_cli/test_restore_state.py @@ -1,20 +1,16 @@ import os -import sys import unittest from nose.plugins.attrib import attr import shutil import numpy as np -from unittest import mock - from ConfigSpace.hyperparameters import UniformFloatHyperparameter from smac.configspace import ConfigurationSpace from smac.smac_cli import SMACCLI from smac.scenario.scenario import Scenario -from smac.facade.smac_facade import SMAC -from smac.optimizer.smbo import SMBO +from smac.facade.smac_ac_facade import SMAC4AC from smac.stats.stats import Stats @@ -77,13 +73,12 @@ def test_illegal_input(self): stats = Stats(scen) # Recorded runs but no incumbent. stats.ta_runs = 10 - smac = SMAC(scen, stats=stats, rng=np.random.RandomState(42)) + smac = SMAC4AC(scen, stats=stats, rng=np.random.RandomState(42)) self.output_dirs.append(scen.output_dir) self.assertRaises(ValueError, smac.optimize) # Incumbent but no recoreded runs. incumbent = cs.get_default_configuration() - smac = SMAC(scen, restore_incumbent=incumbent, - rng=np.random.RandomState(42)) + smac = SMAC4AC(scen, restore_incumbent=incumbent, rng=np.random.RandomState(42)) self.assertRaises(ValueError, smac.optimize) @attr('slow') diff --git a/test/test_configspace/test_configspace.py b/test/test_configspace/test_configspace.py index 444f60553..ba2530ef5 100644 --- a/test/test_configspace/test_configspace.py +++ b/test/test_configspace/test_configspace.py @@ -7,7 +7,8 @@ import unittest from ConfigSpace.read_and_write import pcs -from ConfigSpace.hyperparameters import CategoricalHyperparameter +from ConfigSpace.hyperparameters import CategoricalHyperparameter, \ + UniformFloatHyperparameter from ConfigSpace.conditions import EqualsCondition import smac.configspace @@ -27,24 +28,3 @@ def test_spear(self): for i in range(100): config = cs.sample_configuration() - print(config.get_dictionary()) - - - def test_impute_inactive_hyperparameters(self): - cs = smac.configspace.ConfigurationSpace() - a = cs.add_hyperparameter(CategoricalHyperparameter('a', [0, 1], - default_value=0)) - b = cs.add_hyperparameter(CategoricalHyperparameter('b', [0, 1], - default_value=1)) - cs.add_condition(EqualsCondition(b, a, 1)) - cs.seed(1) - configs = cs.sample_configuration(size=100) - config_array = smac.configspace.convert_configurations_to_array(configs) - for line in config_array: - if line[0] == 0: - self.assertEqual(line[1], 1) - - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_epm/test_gp.py b/test/test_epm/test_gp.py index 62bf02d6d..924a796ed 100644 --- a/test/test_epm/test_gp.py +++ b/test/test_epm/test_gp.py @@ -1,77 +1,260 @@ -import unittest import unittest.mock -import george +from ConfigSpace import EqualsCondition +import scipy.optimize import numpy as np import sklearn.datasets import sklearn.model_selection +from smac.configspace import ( + ConfigurationSpace, + UniformFloatHyperparameter, + CategoricalHyperparameter, + convert_configurations_to_array, +) from smac.epm.gaussian_process import GaussianProcess -from smac.epm.gp_default_priors import DefaultPrior +from smac.epm.gp_base_prior import HorseshoePrior, LognormalPrior +from test import requires_extra -def get_gp(n_dimensions, rs, noise=1e-3): - cov_amp = 2 - initial_ls = np.ones([n_dimensions]) - exp_kernel = george.kernels.Matern52Kernel(initial_ls, ndim=n_dimensions) - kernel = cov_amp * exp_kernel +def get_gp(n_dimensions, rs, noise=1e-3, normalize_y=True) -> GaussianProcess: + from smac.epm.gp_kernels import ConstantKernel, Matern, WhiteKernel - prior = DefaultPrior(len(kernel) + 1, rng=rs) - - n_hypers = 3 * len(kernel) - if n_hypers % 2 == 1: - n_hypers += 1 + cov_amp = ConstantKernel( + 2.0, + constant_value_bounds=(1e-10, 2), + prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rs), + ) + exp_kernel = Matern( + np.ones([n_dimensions]), + [(np.exp(-10), np.exp(2)) for _ in range(n_dimensions)], + nu=2.5, + ) + noise_kernel = WhiteKernel( + noise_level=noise, + noise_level_bounds=(1e-10, 2), + prior=HorseshoePrior(scale=0.1, rng=rs), + ) + kernel = cov_amp * exp_kernel + noise_kernel bounds = [(0., 1.) for _ in range(n_dimensions)] types = np.zeros(n_dimensions) + configspace = ConfigurationSpace() + for i in range(n_dimensions): + configspace.add_hyperparameter(UniformFloatHyperparameter('x%d' % i, 0, 1)) + model = GaussianProcess( - bounds=bounds, types=types, kernel=kernel, - prior=prior, rng=rs, noise=noise, - normalize_output=False, normalize_input=True, + configspace=configspace, + bounds=bounds, + types=types, + kernel=kernel, + seed=rs.randint(low=1, high=10000), + normalize_y=normalize_y, + n_opt_restarts=2, ) return model +def get_cont_data(rs): + X = rs.rand(20, 10) + Y = rs.rand(10, 1) + n_dims = 10 + return X, Y, n_dims + + +def get_cat_data(rs): + X_cont = rs.rand(20, 5) + X_cat = rs.randint(low=0, high=3, size=(20, 5)) + X = np.concatenate([X_cat, X_cont], axis=1) + Y = rs.rand(10, 1) + cat_dims = [0, 1, 2, 3, 4] + cont_dims = [5, 6, 7, 8, 9] + return X, Y, cat_dims, cont_dims + + +def get_mixed_gp(cat_dims, cont_dims, rs, noise=1e-3, normalize_y=True): + from smac.epm.gp_kernels import ConstantKernel, Matern, WhiteKernel, HammingKernel + + cat_dims = np.array(cat_dims, dtype=np.int) + cont_dims = np.array(cont_dims, dtype=np.int) + n_dimensions = len(cat_dims) + len(cont_dims) + cov_amp = ConstantKernel( + 2.0, + constant_value_bounds=(1e-10, 2), + prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rs), + ) + + exp_kernel = Matern( + np.ones([len(cont_dims)]), + [(np.exp(-10), np.exp(2)) for _ in range(len(cont_dims))], + nu=2.5, + operate_on=cont_dims, + ) + + ham_kernel = HammingKernel( + np.ones([len(cat_dims)]), + [(np.exp(-10), np.exp(2)) for _ in range(len(cat_dims))], + operate_on=cat_dims, + ) + noise_kernel = WhiteKernel( + noise_level=noise, + noise_level_bounds=(1e-10, 2), + prior=HorseshoePrior(scale=0.1, rng=rs), + ) + kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel + + bounds = [0] * n_dimensions + types = np.zeros(n_dimensions) + for c in cont_dims: + bounds[c] = (0., 1.) + for c in cat_dims: + types[c] = 3 + bounds[c] = (3, np.nan) + + cs = ConfigurationSpace() + for c in cont_dims: + cs.add_hyperparameter(UniformFloatHyperparameter('X%d' % c, 0, 1)) + for c in cat_dims: + cs.add_hyperparameter(CategoricalHyperparameter('X%d' % c, [0, 1, 2, 3])) + + model = GaussianProcess( + configspace=cs, + bounds=bounds, + types=types, + kernel=kernel, + seed=rs.randint(low=1, high=10000), + normalize_y=normalize_y, + ) + return model + + +@requires_extra('gp') class TestGP(unittest.TestCase): + def test_predict_wrong_X_dimensions(self): rs = np.random.RandomState(1) - model = get_gp(10, rs) - X = rs.rand(10) - self.assertRaisesRegexp(ValueError, "Expected 2d array, got 1d array!", - model.predict, X) - X = rs.rand(10, 10, 10) - self.assertRaisesRegexp(ValueError, "Expected 2d array, got 3d array!", - model.predict, X) + # cont + X, Y, n_dims = get_cont_data(rs) + # cat + X, Y, cat_dims, cont_dims = get_cat_data(rs) - X = rs.rand(10, 5) - self.assertRaisesRegexp(ValueError, "Rows in X should have 10 entries " - "but have 5!", - model.predict, X) + for model in (get_gp(n_dims, rs), get_mixed_gp(cat_dims, cont_dims, rs)): + X = rs.rand(10) + self.assertRaisesRegexp(ValueError, "Expected 2d array, got 1d array!", + model.predict, X) + X = rs.rand(10, 10, 10) + self.assertRaisesRegexp(ValueError, "Expected 2d array, got 3d array!", + model.predict, X) + + X = rs.rand(10, 5) + self.assertRaisesRegexp(ValueError, "Rows in X should have 10 entries " + "but have 5!", + model.predict, X) def test_predict(self): rs = np.random.RandomState(1) - X = rs.rand(20, 10) - Y = rs.rand(10, 1) - model = get_gp(10, rs) - model.train(X[:10], Y[:10]) - m_hat, v_hat = model.predict(X[10:]) - self.assertEqual(m_hat.shape, (10, 1)) - self.assertEqual(v_hat.shape, (10, 1)) + + # cont + X, Y, n_dims = get_cont_data(rs) + # cat + X, Y, cat_dims, cont_dims = get_cat_data(rs) + + for model in (get_gp(n_dims, rs), get_mixed_gp(cat_dims, cont_dims, rs)): + model.train(X[:10], Y[:10]) + m_hat, v_hat = model.predict(X[10:]) + self.assertEqual(m_hat.shape, (10, 1)) + self.assertEqual(v_hat.shape, (10, 1)) + + def test_train_do_optimize(self): + # Check that do_optimize does not mess with the kernel hyperparameters given to the Gaussian process! + + rs = np.random.RandomState(1) + X, Y, n_dims = get_cont_data(rs) + + model = get_gp(n_dims, rs) + model._train(X[:10], Y[:10], do_optimize=False) + theta = model.gp.kernel.theta + theta_ = model.gp.kernel_.theta + fixture = np.array([0.693147, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., -6.907755]) + np.testing.assert_array_almost_equal(theta, fixture) + np.testing.assert_array_almost_equal(theta_, fixture) + np.testing.assert_array_almost_equal(theta, theta_) + + model._train(X[:10], Y[:10], do_optimize=True) + theta = model.gp.kernel.theta + theta_ = model.gp.kernel_.theta + fixture = np.array([0.693147, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., -6.907755]) + self.assertFalse(np.any(theta == fixture)) + self.assertFalse(np.any(theta_ == fixture)) + np.testing.assert_array_almost_equal(theta, theta_) + + @unittest.mock.patch('skopt.learning.gaussian_process.GaussianProcessRegressor.fit') + def test_train_continue_on_linalg_error(self, fit_mock): + # Check that training does not stop on a linalg error, but that uncertainty is increased! + + class Dummy: + counter = 0 + def __call__(self, X, y): + if self.counter >= 10: + return None + else: + self.counter += 1 + raise np.linalg.LinAlgError + + fit_mock.side_effect = Dummy() + + rs = np.random.RandomState(1) + X, Y, n_dims = get_cont_data(rs) + + model = get_gp(n_dims, rs) + fixture = np.exp(model.kernel.theta[-1]) + model._train(X[:10], Y[:10], do_optimize=False) + self.assertAlmostEqual(np.exp(model.gp.kernel.theta[-1]), fixture + 10) + + @unittest.mock.patch('skopt.learning.gaussian_process.GaussianProcessRegressor.log_marginal_likelihood') + def test_train_continue_on_linalg_error_2(self, fit_mock): + # Check that training does not stop on a linalg error during hyperparameter optimization + + class Dummy: + counter = 0 + def __call__(self, X, eval_gradient=True): + # If this is not aligned with the GP an error will be raised that None is not iterable + if self.counter == 13: + return None + else: + self.counter += 1 + raise np.linalg.LinAlgError + + fit_mock.side_effect = Dummy() + + rs = np.random.RandomState(1) + X, Y, n_dims = get_cont_data(rs) + + model = get_gp(n_dims, rs) + fixture = model.kernel.theta + model._train(X[:10], Y[:10], do_optimize=True) + np.testing.assert_array_almost_equal( + model.gp.kernel.theta, + [0.69314718, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.69314718] + ) @unittest.mock.patch.object(GaussianProcess, 'predict') def test_predict_marginalized_over_instances_no_features(self, rf_mock): """The GP should fall back to the regular predict() method.""" - rs = np.random.RandomState(1) - X = rs.rand(20, 10) - Y = rs.rand(10, 1) - model = get_gp(10, rs) - model.train(X[:10], Y[:10]) - model.predict(X[10:]) - self.assertEqual(rf_mock.call_count, 1) + + # cont + X, Y, n_dims = get_cont_data(rs) + # cat + X, Y, cat_dims, cont_dims = get_cat_data(rs) + + for ct, model in enumerate((get_gp(n_dims, rs), get_mixed_gp(cat_dims, cont_dims, rs))): + model.train(X[:10], Y[:10]) + model.predict(X[10:]) + self.assertEqual(rf_mock.call_count, ct+1) def test_predict_with_actual_values(self): X = np.array([ @@ -93,20 +276,21 @@ def test_predict_with_actual_values(self): [109.], [109.2]], dtype=np.float64) rs = np.random.RandomState(1) - model = get_gp(3, rs, noise=1e-9) + model = get_gp(3, rs) model.train(np.vstack((X, X, X, X, X, X, X, X)), np.vstack((y, y, y, y, y, y, y, y))) - y_hat, mu_hat = model.predict(X) + mu_hat, var_hat = model.predict(X) for y_i, y_hat_i, mu_hat_i in zip( - y.reshape((1, -1)).flatten(), y_hat.reshape((1, -1)).flatten(), mu_hat.reshape((1, -1)).flatten(), + y.reshape((1, -1)).flatten(), mu_hat.reshape((1, -1)).flatten(), var_hat.reshape((1, -1)).flatten(), ): - self.assertAlmostEqual(y_i, y_hat_i, delta=0.001) - self.assertAlmostEqual(mu_hat_i, 0) + self.assertAlmostEqual(y_hat_i, y_i, delta=2) + self.assertAlmostEqual(mu_hat_i, 0, delta=2) # Regression test that performance does not drastically decrease in the near future - y_hat, mu_hat = model.predict(np.array([[10, 10, 10]])) - self.assertAlmostEqual(y_hat[0][0], 54.61250000002053) - self.assertAlmostEqual(mu_hat[0][0], 2.0) + mu_hat, var_hat = model.predict(np.array([[10, 10, 10]])) + self.assertAlmostEqual(mu_hat[0][0], 54.612500000000004) + # There's a slight difference between my local installation and travis + self.assertLess(abs(var_hat[0][0] - 1121.8409184001594), 2) def test_gp_on_sklearn_data(self): X, y = sklearn.datasets.load_boston(return_X_y=True) @@ -116,7 +300,7 @@ def test_gp_on_sklearn_data(self): model = get_gp(X.shape[1], rs) cv = sklearn.model_selection.KFold(shuffle=True, random_state=rs, n_splits=2) - maes = [10.109955737245306468, 9.553761121008572789] + maes = [8.712875586609810299, 8.7608419489812271634] for i, (train_split, test_split) in enumerate(cv.split(X, y)): X_train = X[train_split] @@ -127,3 +311,75 @@ def test_gp_on_sklearn_data(self): y_hat, mu_hat = model.predict(X_test) mae = np.mean(np.abs(y_hat - y_test), dtype=np.float128) self.assertAlmostEqual(mae, maes[i]) + + def test_nll(self): + rs = np.random.RandomState(1) + gp = get_gp(1, rs) + gp.train(np.array([[0], [1]]), np.array([0, 1])) + n_above_1 = 0 + for i in range(1000): + theta = np.array( + [rs.uniform(1e-10, 10), rs.uniform(-10, 2), rs.uniform(-10, 1)] + ) # Values from the default prior + error = scipy.optimize.check_grad(lambda x: gp._nll(x)[0], lambda x: gp._nll(x)[1], theta, epsilon=1e-5) + if error > 0.1: + n_above_1 += 1 + self.assertLessEqual(n_above_1, 10) + + def test_sampling_shape(self): + X = np.arange(-5, 5, 0.1).reshape((-1, 1)) + X_test = np.arange(-5.05, 5.05, 0.1).reshape((-1, 1)) + y = np.sin(X) + rng = np.random.RandomState(1) + for gp in ( + get_gp(n_dimensions=1, rs=rng, noise=1e-10, normalize_y=False), + get_gp(n_dimensions=1, rs=rng, noise=1e-10, normalize_y=True), + ): + gp._train(X, y) + func = gp.sample_functions(X_test=X_test, n_funcs=1) + self.assertEqual(func.shape, (101, 1)) + func = gp.sample_functions(X_test=X_test, n_funcs=2) + self.assertEqual(func.shape, (101, 2)) + + def test_normalization(self): + X = np.arange(-5, 5, 0.1).reshape((-1, 1)) + X_test = np.arange(-5.05, 5.05, 0.1).reshape((-1, 1)) + y = np.sin(X) + rng = np.random.RandomState(1) + gp = get_gp(n_dimensions=1, rs=rng, noise=1e-10, normalize_y=False) + gp._train(X, y, do_optimize=False) + mu_hat, var_hat = gp.predict(X_test) + gp_norm = get_gp(n_dimensions=1, rs=rng, noise=1e-10, normalize_y=True) + gp_norm._train(X, y, do_optimize=False) + mu_hat_prime, var_hat_prime = gp_norm.predict(X_test) + np.testing.assert_array_almost_equal(mu_hat, mu_hat_prime, decimal=4) + np.testing.assert_array_almost_equal(var_hat, var_hat_prime, decimal=4) + + func = gp.sample_functions(X_test=X_test, n_funcs=2) + func_prime = gp_norm.sample_functions(X_test=X_test, n_funcs=2) + np.testing.assert_array_almost_equal(func, func_prime, decimal=1) + + def test_impute_inactive_hyperparameters(self): + cs = ConfigurationSpace() + a = cs.add_hyperparameter(CategoricalHyperparameter('a', [0, 1])) + b = cs.add_hyperparameter(CategoricalHyperparameter('b', [0, 1])) + c = cs.add_hyperparameter(UniformFloatHyperparameter('c', 0, 1)) + cs.add_condition(EqualsCondition(b, a, 1)) + cs.add_condition(EqualsCondition(c, a, 0)) + cs.seed(1) + + configs = cs.sample_configuration(size=100) + config_array = convert_configurations_to_array(configs) + for line in config_array: + if line[0] == 0: + self.assertTrue(np.isnan(line[1])) + elif line[0] == 1: + self.assertTrue(np.isnan(line[2])) + + gp = get_gp(3, np.random.RandomState(1)) + config_array = gp._impute_inactive(config_array) + for line in config_array: + if line[0] == 0: + self.assertEqual(line[1], -1) + elif line[0] == 1: + self.assertEqual(line[2], -1) diff --git a/test/test_epm/test_gp_mcmc.py b/test/test_epm/test_gp_mcmc.py index 343db7c9c..77b4c73b7 100644 --- a/test/test_epm/test_gp_mcmc.py +++ b/test/test_epm/test_gp_mcmc.py @@ -1,46 +1,65 @@ -import unittest import unittest.mock -import george import numpy as np import sklearn.datasets import sklearn.model_selection +from smac.configspace import ConfigurationSpace, UniformFloatHyperparameter from smac.epm.gaussian_process_mcmc import GaussianProcessMCMC -from smac.epm.gp_default_priors import DefaultPrior +from smac.epm.gp_base_prior import LognormalPrior, HorseshoePrior +from test import requires_extra -def get_gp(n_dimensions, rs, noise=-8): - cov_amp = 2 - initial_ls = np.ones([n_dimensions]) - exp_kernel = george.kernels.Matern52Kernel(initial_ls, ndim=n_dimensions) - kernel = cov_amp * exp_kernel +def get_gp(n_dimensions, rs, noise=1e-3, normalize_y=True, average_samples=False, n_iter=50): + from smac.epm.gp_kernels import ConstantKernel, Matern, WhiteKernel - prior = DefaultPrior(len(kernel) + 1, rng=rs) + cov_amp = ConstantKernel( + 2.0, + constant_value_bounds=(1e-10, 2), + prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rs), + ) + exp_kernel = Matern( + np.ones([n_dimensions]), + [(np.exp(-10), np.exp(2)) for _ in range(n_dimensions)], + nu=2.5, + prior=None, + ) + noise_kernel = WhiteKernel( + noise_level=noise, + noise_level_bounds=(1e-10, 2), + prior=HorseshoePrior(scale=0.1, rng=rs), + ) + kernel = cov_amp * exp_kernel + noise_kernel - n_hypers = 3 * len(kernel) - if n_hypers % 2 == 1: - n_hypers += 1 + n_mcmc_walkers = 3 * len(kernel.theta) + if n_mcmc_walkers % 2 == 1: + n_mcmc_walkers += 1 bounds = [(0., 1.) for _ in range(n_dimensions)] types = np.zeros(n_dimensions) + configspace = ConfigurationSpace() + for i in range(n_dimensions): + configspace.add_hyperparameter(UniformFloatHyperparameter('x%d' % i, 0, 1)) + model = GaussianProcessMCMC( + configspace=configspace, types=types, bounds=bounds, kernel=kernel, - prior=prior, - n_hypers=n_hypers, - chain_length=20, - burnin_steps=100, - normalize_input=True, - normalize_output=True, - rng=rs, noise=noise, + n_mcmc_walkers=n_mcmc_walkers, + chain_length=n_iter, + burnin_steps=n_iter, + normalize_y=normalize_y, + seed=rs.randint(low=1, high=10000), + mcmc_sampler='emcee', + average_samples=average_samples, ) return model +@requires_extra('gp') class TestGPMCMC(unittest.TestCase): def test_predict_wrong_X_dimensions(self): rs = np.random.RandomState(1) @@ -58,6 +77,46 @@ def test_predict_wrong_X_dimensions(self): "but have 5!", model.predict, X) + def test_gp_train(self): + rs = np.random.RandomState(1) + X = rs.rand(20, 10) + Y = rs.rand(10, 1) + + fixture = np.array([0.693147, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., -6.907755]) + + model = get_gp(10, rs) + np.testing.assert_array_almost_equal(model.kernel.theta, fixture) + model.train(X[:10], Y[:10]) + + for base_model in model.models: + theta = base_model.gp.kernel.theta + theta_ = base_model.gp.kernel_.theta + # Test that the kernels of the base GP are actually changed! + np.testing.assert_array_almost_equal(theta, theta_) + self.assertFalse(np.any(theta == fixture)) + self.assertFalse(np.any(theta_ == fixture)) + + def test_gp_train_posterior_mean(self): + rs = np.random.RandomState(1) + X = rs.rand(20, 10) + Y = rs.rand(10, 1) + + fixture = np.array([0.693147, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., -6.907755]) + + model = get_gp(10, rs, average_samples=True) + np.testing.assert_array_almost_equal(model.kernel.theta, fixture) + model.train(X[:10], Y[:10]) + + for base_model in model.models: + theta = base_model.gp.kernel.theta + theta_ = base_model.gp.kernel_.theta + # Test that the kernels of the base GP are actually changed! + np.testing.assert_array_almost_equal(theta, theta_) + self.assertFalse(np.any(theta == fixture)) + self.assertFalse(np.any(theta_ == fixture)) + + self.assertEqual(len(model.models), 1) + def test_predict(self): rs = np.random.RandomState(1) X = rs.rand(20, 10) @@ -100,32 +159,33 @@ def test_predict_with_actual_values(self): [109.], [109.2]], dtype=np.float64) rs = np.random.RandomState(1) - model = get_gp(3, rs, noise=-8) + model = get_gp(3, rs, noise=1e-10, n_iter=200) model.train(np.vstack((X, X, X, X, X, X, X, X)), np.vstack((y, y, y, y, y, y, y, y))) y_hat, var_hat = model.predict(X) for y_i, y_hat_i, var_hat_i in zip( y.reshape((1, -1)).flatten(), y_hat.reshape((1, -1)).flatten(), var_hat.reshape((1, -1)).flatten(), ): - # Chain length too short to get excellent predictions - self.assertAlmostEqual(y_i, y_hat_i, delta=0.1) - self.assertAlmostEqual(var_hat_i, 0, delta=0.1) + # Chain length too short to get excellent predictions, apparently there's a lot of predictive variance + self.assertAlmostEqual(y_i, y_hat_i, delta=1) + self.assertAlmostEqual(var_hat_i, 0, delta=500) # Regression test that performance does not drastically decrease in the near future y_hat, var_hat = model.predict(np.array([[10, 10, 10]])) - self.assertAlmostEqual(y_hat[0][0], 54.61249999999999) - # Massive variance due to internally used law of total variances - self.assertAlmostEqual(var_hat[0][0], 27691.09369406573) + self.assertAlmostEqual(y_hat[0][0], 54.613410745846785, delta=0.1) + # Massive variance due to internally used law of total variances, also a massive difference locally and on + # travis-ci + self.assertLessEqual(abs(var_hat[0][0]) - 860, 200, msg=str(var_hat)) def test_gp_on_sklearn_data(self): X, y = sklearn.datasets.load_boston(return_X_y=True) # Normalize such that the bounds in get_gp hold X = X / X.max(axis=0) rs = np.random.RandomState(1) - model = get_gp(X.shape[1], rs) + model = get_gp(X.shape[1], rs, noise=1e-10, normalize_y=True) cv = sklearn.model_selection.KFold(shuffle=True, random_state=rs, n_splits=2) - maes = [9.1921780939101723415, 8.929528339491875724] + maes = [6.829207681798763524, 7.5125099471964293836] for i, (train_split, test_split) in enumerate(cv.split(X, y)): X_train = X[train_split] @@ -136,3 +196,21 @@ def test_gp_on_sklearn_data(self): y_hat, mu_hat = model.predict(X_test) mae = np.mean(np.abs(y_hat - y_test), dtype=np.float128) self.assertAlmostEqual(mae, maes[i]) + + def test_normalization(self): + X = np.arange(-5, 5, 0.1).reshape((-1, 1)) + X_test = np.arange(-5.05, 5.05, 0.1).reshape((-1, 1)) + y = np.sin(X) + rng = np.random.RandomState(1) + gp = get_gp(n_dimensions=1, rs=rng, noise=1e-10, normalize_y=False) + gp._train(X, y, do_optimize=False) + self.assertFalse(gp.models[0].normalize_y) + self.assertFalse(hasattr(gp.models[0], 'mean_y_')) + mu_hat, var_hat = gp.predict(X_test) + gp_norm = get_gp(n_dimensions=1, rs=rng, noise=1e-10, normalize_y=True) + gp_norm._train(X, y, do_optimize=False) + self.assertTrue(gp_norm.models[0].normalize_y) + self.assertTrue(hasattr(gp_norm.models[0], 'mean_y_')) + mu_hat_prime, var_hat_prime = gp_norm.predict(X_test) + np.testing.assert_array_almost_equal(mu_hat, mu_hat_prime, decimal=4) + np.testing.assert_array_almost_equal(var_hat, var_hat_prime, decimal=4) diff --git a/test/test_epm/test_gp_priors.py b/test/test_epm/test_gp_priors.py new file mode 100644 index 000000000..f4dea7441 --- /dev/null +++ b/test/test_epm/test_gp_priors.py @@ -0,0 +1,197 @@ +import unittest + +import numpy as np +import scipy.optimize +import scipy.stats as scst +import scipy.integrate as scin + +from smac.epm.gp_base_prior import TophatPrior, HorseshoePrior, LognormalPrior, GammaPrior, SoftTopHatPrior +from smac.utils.constants import VERY_SMALL_NUMBER + + +def wrap_ln(theta, prior): + return np.exp(prior.lnprob(np.log(theta))) + + +class TestTophatPrior(unittest.TestCase): + + def test_lnprob_and_grad_scalar(self): + prior = TophatPrior(lower_bound=np.exp(-10), upper_bound=np.exp(2)) + + # Legal scalar + for val in (-1, 0, 1): + self.assertEqual(prior.lnprob(val), 0, msg=str(val)) + self.assertEqual(prior.gradient(val), 0, msg=str(val)) + + # Boundary + for val in (-10, 2): + self.assertEqual(prior.lnprob(val), 0) + self.assertEqual(prior.gradient(val), 0) + + # Values outside the boundary + for val in (-10 - VERY_SMALL_NUMBER, 2 + VERY_SMALL_NUMBER, -50, 50): + self.assertTrue(np.isinf(prior.lnprob(val))) + self.assertEqual(prior.gradient(val), 0) + + def test_sample_from_prior(self): + prior = TophatPrior(lower_bound=np.exp(-10), upper_bound=np.exp(2), rng=np.random.RandomState(1)) + samples = prior.sample_from_prior(10) + np.testing.assert_array_equal(samples >= -10, True) + np.testing.assert_array_equal(samples <= 2, True) + # Test that the rng is set + self.assertAlmostEqual(samples[0], -4.995735943569112) + + def test_sample_from_prior_shapes(self): + rng = np.random.RandomState(1) + lower_bound = 2 + rng.random_sample() * 50 + upper_bound = lower_bound + rng.random_sample() * 50 + prior = TophatPrior(lower_bound=lower_bound, upper_bound=upper_bound) + sample = prior.sample_from_prior(1) + self.assertEqual(sample.shape, (1,)) + sample = prior.sample_from_prior(2) + self.assertEqual(sample.shape, (2,)) + sample = prior.sample_from_prior(10) + self.assertEqual(sample.shape, (10,)) + with self.assertRaises(ValueError): + prior.sample_from_prior(0) + with self.assertRaises(ValueError): + prior.sample_from_prior((2,)) + + +class TestHorseshoePrior(unittest.TestCase): + + def test_lnprob_and_grad_scalar(self): + prior = HorseshoePrior(scale=1) + + # Legal scalar + self.assertEqual(prior.lnprob(-1), 1.1450937952919953) + self.assertEqual(prior.gradient(-1), -0.6089187456211098) + + # Boundary + self.assertTrue(np.isinf(prior._lnprob(0))) + self.assertTrue(np.isinf(prior._gradient(0))) + + def test_sample_from_prior(self): + prior = HorseshoePrior(scale=1, rng=np.random.RandomState(1)) + samples = prior.sample_from_prior(10) + # Test that the rng is set + self.assertAlmostEqual(samples[0], 1.0723988839129437) + + def test_sample_from_prior_shapes(self): + rng = np.random.RandomState(1) + lower_bound = 2 + rng.random_sample() * 50 + upper_bound = lower_bound + rng.random_sample() * 50 + prior = TophatPrior(lower_bound=lower_bound, upper_bound=upper_bound) + sample = prior.sample_from_prior(1) + self.assertEqual(sample.shape, (1,)) + sample = prior.sample_from_prior(2) + self.assertEqual(sample.shape, (2,)) + sample = prior.sample_from_prior(10) + self.assertEqual(sample.shape, (10,)) + with self.assertRaises(ValueError): + prior.sample_from_prior(0) + with self.assertRaises(ValueError): + prior.sample_from_prior((2,)) + + def test_gradient(self): + for scale in (0.1, 0.5, 1., 2.): + prior = HorseshoePrior(scale=scale) + # The function appears to be unstable above 15 + for theta in range(-20, 15): + if theta == 0: + continue + error = scipy.optimize.check_grad( + lambda x: prior.lnprob(x[0]), + lambda x: prior.gradient(x[0]), + np.array([theta]), + epsilon=1e-5, + ) + self.assertAlmostEqual(error, 0, delta=5) + + +class TestGammaPrior(unittest.TestCase): + + def test_lnprob_and_grad_scalar(self): + prior = GammaPrior(a=0.5, scale=1/2, loc=0) + + # Legal scalar + x = -1 + self.assertEqual(prior.lnprob(x), -0.46155023498761205) + self.assertEqual(prior.gradient(x), -1.2357588823428847) + + def test_lnprob_and_grad_array(self): + prior = GammaPrior(a=0.5, scale=1/2, loc=0) + val = np.array([-1, -1]) + with self.assertRaises(NotImplementedError): + prior.lnprob(val) + with self.assertRaises(NotImplementedError): + prior.gradient(val) + + def test_gradient(self): + for scale in (0.5, 1., 2.): + prior = GammaPrior(a=2, scale=scale, loc=0) + # The function appears to be unstable above 10 + for theta in np.arange(1e-15, 10, 0.01): + if theta == 0: + continue + error = scipy.optimize.check_grad( + lambda x: prior.lnprob(x[0]), + lambda x: prior.gradient(x[0]), + np.array([theta]), + epsilon=1e-5, + ) + self.assertAlmostEqual(error, 0, delta=5, msg=str(theta)) + + +class TestLogNormalPrior(unittest.TestCase): + + def test_gradient(self): + for sigma in (0.5, 1., 2.): + prior = LognormalPrior(mean=0, sigma=sigma) + # The function appears to be unstable above 15 + for theta in range(0, 15): + # Gradient approximation becomes unstable when going closer to zero + theta += 1e-2 + error = scipy.optimize.check_grad( + lambda x: prior.lnprob(x[0]), + lambda x: prior.gradient(x[0]), + np.array([theta]), + epsilon=1e-5, + ) + self.assertAlmostEqual(error, 0, delta=5, msg=theta) + + +class TestSoftTopHatPrior(unittest.TestCase): + + def test_lnprob(self): + prior = SoftTopHatPrior(lower_bound=np.exp(-5), upper_bound=np.exp(5)) + + # Legal values + self.assertEqual(prior.lnprob(-5), 0) + self.assertEqual(prior.lnprob(0), 0) + self.assertEqual(prior.lnprob(5), 0) + + # Illegal values + self.assertAlmostEqual(prior.lnprob(-5.1), -0.01) + self.assertAlmostEqual(prior.lnprob(-6), -1) + self.assertAlmostEqual(prior.lnprob(-7), -4) + self.assertAlmostEqual(prior.lnprob(5.1), -0.01) + self.assertAlmostEqual(prior.lnprob(6), -1) + self.assertAlmostEqual(prior.lnprob(7), -4) + + def test_grad(self): + prior = SoftTopHatPrior(lower_bound=np.exp(-5), upper_bound=np.exp(5)) + + # Legal values + self.assertEqual(prior.gradient(-5), 0) + self.assertEqual(prior.gradient(0), 0) + self.assertEqual(prior.gradient(5), 0) + + for theta in [-10, -7, -6, -5.1, 5.1, 6, 7, 10]: + # Gradient approximation becomes unstable when going closer to zero + theta += 1e-2 + grad = prior.gradient(theta) + grad_vector = prior.gradient(theta) + self.assertEqual(grad, grad_vector) + error = scipy.optimize.check_grad(prior.lnprob, prior.gradient, np.array([theta]), epsilon=1e-5) + self.assertAlmostEqual(error, 0, delta=5, msg=theta) diff --git a/test/test_epm/test_rf_with_instances.py b/test/test_epm/test_rf_with_instances.py index 280a6f809..b21ca84f0 100644 --- a/test/test_epm/test_rf_with_instances.py +++ b/test/test_epm/test_rf_with_instances.py @@ -3,23 +3,32 @@ import numpy as np -from ConfigSpace.hyperparameters import CategoricalHyperparameter, \ +from ConfigSpace import CategoricalHyperparameter, \ UniformFloatHyperparameter, UniformIntegerHyperparameter, \ - OrdinalHyperparameter + OrdinalHyperparameter, ConfigurationSpace, EqualsCondition from smac.epm.rf_with_instances import RandomForestWithInstances +from smac.epm.util_funcs import get_types import smac import smac.configspace -from smac.utils.util_funcs import get_types class TestRFWithInstances(unittest.TestCase): + + def _get_cs(self, n_dimensions): + configspace = smac.configspace.ConfigurationSpace() + for i in range(n_dimensions): + configspace.add_hyperparameter(UniformFloatHyperparameter('x%d' % i, 0, 1)) + return configspace + def test_predict_wrong_X_dimensions(self): rs = np.random.RandomState(1) model = RandomForestWithInstances( + configspace=self._get_cs(10), types=np.zeros((10,), dtype=np.uint), bounds=list(map(lambda x: (0, 10), range(10))), + seed=1, ) X = rs.rand(10) self.assertRaisesRegexp(ValueError, "Expected 2d array, got 1d array!", @@ -38,8 +47,10 @@ def test_predict(self): X = rs.rand(20, 10) Y = rs.rand(10, 1) model = RandomForestWithInstances( + configspace=self._get_cs(10), types=np.zeros((10,), dtype=np.uint), bounds=list(map(lambda x: (0, 10), range(10))), + seed=1, ) model.train(X[:10], Y[:10]) m_hat, v_hat = model.predict(X[10:]) @@ -52,8 +63,10 @@ def test_train_with_pca(self): F = rs.rand(10, 10) Y = rs.rand(20, 1) model = RandomForestWithInstances( + configspace=self._get_cs(10), types=np.zeros((20,), dtype=np.uint), bounds=list(map(lambda x: (0, 10), range(10))), + seed=1, pca_components=2, instance_features=F, ) @@ -67,9 +80,13 @@ def test_train_with_pca(self): def test_predict_marginalized_over_instances_wrong_X_dimensions(self): rs = np.random.RandomState(1) - model = RandomForestWithInstances(np.zeros((10,), dtype=np.uint), - instance_features=rs.rand(10, 2), - bounds=np.array(list(map(lambda x: (0, 10), range(10))), dtype=object)) + model = RandomForestWithInstances( + configspace=self._get_cs(10), + types=np.zeros((10,), dtype=np.uint), + instance_features=rs.rand(10, 2), + seed=1, + bounds=list(map(lambda x: (0, 10), range(10))), + ) X = rs.rand(10) self.assertRaisesRegexp(ValueError, "Expected 2d array, got 1d array!", model.predict_marginalized_over_instances, X) @@ -84,8 +101,12 @@ def test_predict_marginalized_over_instances_no_features(self, rf_mock): rs = np.random.RandomState(1) X = rs.rand(20, 10) Y = rs.rand(10, 1) - model = RandomForestWithInstances(np.zeros((10,), dtype=np.uint), bounds=np.array( - list(map(lambda x: (0, 10), range(10))), dtype=object)) + model = RandomForestWithInstances( + configspace=self._get_cs(10), + types=np.zeros((10,), dtype=np.uint), + bounds=list(map(lambda x: (0, 10), range(10))), + seed=1, + ) model.train(X[:10], Y[:10]) model.predict(X[10:]) self.assertEqual(rf_mock.call_count, 1) @@ -97,9 +118,13 @@ def test_predict_marginalized_over_instances(self): Y = rs.rand(len(X) * len(F), 1) X_ = rs.rand(200, 15) - model = RandomForestWithInstances(np.zeros((15,), dtype=np.uint), - instance_features=F, - bounds=np.array(list(map(lambda x: (0, 10), range(10))), dtype=object)) + model = RandomForestWithInstances( + configspace=self._get_cs(10), + types=np.zeros((15,), dtype=np.uint), + instance_features=F, + bounds=list(map(lambda x: (0, 10), range(10))), + seed=1, + ) model.train(X_, Y) means, vars = model.predict_marginalized_over_instances(X) self.assertEqual(means.shape, (20, 1)) @@ -121,9 +146,13 @@ def __call__(self, X): rs = np.random.RandomState(1) F = rs.rand(10, 5) - model = RandomForestWithInstances(np.zeros((15,), dtype=np.uint), - instance_features=F, - bounds=np.array(list(map(lambda x: (0, 10), range(10))), dtype=object)) + model = RandomForestWithInstances( + configspace=self._get_cs(10), + types=np.zeros((15,), dtype=np.uint), + instance_features=F, + bounds=list(map(lambda x: (0, 10), range(10))), + seed=1, + ) X = rs.rand(20, 10) F = rs.rand(10, 5) Y = rs.randint(1, size=(len(X) * len(F), 1)) * 1. @@ -135,7 +164,7 @@ def __call__(self, X): self.assertEqual(vars.shape, (11, 1)) for i in range(11): self.assertEqual(means[i], 0.) - self.assertEqual(vars[i], 1.e-05) + self.assertEqual(vars[i], 1.e-10) def test_predict_with_actual_values(self): X = np.array([ @@ -156,20 +185,19 @@ def test_predict_with_actual_values(self): [100.2], [109.], [109.2]], dtype=np.float64) - model = RandomForestWithInstances(types=np.array([0, 0, 0], dtype=np.uint), - bounds=np.array([(0, np.nan), (0, np.nan), (0, np.nan)], dtype=object), - instance_features=None, seed=12345, - ratio_features=1.0) + model = RandomForestWithInstances( + configspace=self._get_cs(3), + types=np.array([0, 0, 0], dtype=np.uint), + bounds=[(0, np.nan), (0, np.nan), (0, np.nan)], + instance_features=None, + seed=12345, + ratio_features=1.0, + ) model.train(np.vstack((X, X, X, X, X, X, X, X)), np.vstack((y, y, y, y, y, y, y, y))) - # for idx, x in enumerate(X): - # print(model.rf.all_leaf_values(x)) - # print(x, model.predict(np.array([x]))[0], y[idx]) y_hat, _ = model.predict(X) for y_i, y_hat_i in zip(y.reshape((1, -1)).flatten(), y_hat.reshape((1, -1)).flatten()): - # print(y_i, y_hat_i) self.assertAlmostEqual(y_i, y_hat_i, delta=0.1) - # print() def test_with_ordinal(self): cs = smac.configspace.ConfigurationSpace() @@ -181,10 +209,15 @@ def test_with_ordinal(self): feat_array = np.array([0, 0, 0]).reshape(1, -1) types, bounds = get_types(cs, feat_array) - model = RandomForestWithInstances(types=types, bounds=bounds, - instance_features=feat_array, - seed=1, ratio_features=1.0, - pca_components=9) + model = RandomForestWithInstances( + configspace=cs, + types=types, + bounds=bounds, + instance_features=feat_array, + seed=1, + ratio_features=1.0, + pca_components=9, + ) self.assertEqual(bounds[0][0], 2) self.assertTrue(bounds[0][1] is np.nan) self.assertEqual(bounds[1][0], 0) @@ -207,3 +240,78 @@ def test_with_ordinal(self): mean, _ = model.predict(X) for idx, m in enumerate(mean): self.assertAlmostEqual(y[idx], m, 0.05) + + def test_rf_on_sklearn_data(self): + import sklearn.datasets + X, y = sklearn.datasets.load_boston(return_X_y=True) + rs = np.random.RandomState(1) + + types = np.zeros(X.shape[1]) + bounds = [(np.min(X[:, i]), np.max(X[:, i])) for i in range(X.shape[1])] + + cv = sklearn.model_selection.KFold(shuffle=True, random_state=rs, n_splits=2) + + for do_log in [False, True]: + if do_log: + targets = np.log(y) + model = RandomForestWithInstances( + configspace=self._get_cs(X.shape[1]), + types=types, + bounds=bounds, + seed=1, + ratio_features=1.0, + pca_components=100, + log_y=True, + ) + maes = [0.43186902865718386507, 0.4267519520332511912] + else: + targets = y + model = RandomForestWithInstances( + configspace=self._get_cs(X.shape[1]), + types=types, + bounds=bounds, + seed=1, + ratio_features=1.0, + pca_components=100, + ) + maes = [9.3298376833224042496, 9.348010654109179346] + + for i, (train_split, test_split) in enumerate(cv.split(X, targets)): + X_train = X[train_split] + y_train = targets[train_split] + X_test = X[test_split] + y_test = targets[test_split] + model.train(X_train, y_train) + y_hat, mu_hat = model.predict(X_test) + mae = np.mean(np.abs(y_hat - y_test), dtype=np.float128) + self.assertAlmostEqual(mae, maes[i], msg=('Do log: %s, iteration %i' % (str(do_log), i))) + + def test_impute_inactive_hyperparameters(self): + cs = smac.configspace.ConfigurationSpace() + a = cs.add_hyperparameter(CategoricalHyperparameter('a', [0, 1])) + b = cs.add_hyperparameter(CategoricalHyperparameter('b', [0, 1])) + c = cs.add_hyperparameter(UniformFloatHyperparameter('c', 0, 1)) + cs.add_condition(EqualsCondition(b, a, 1)) + cs.add_condition(EqualsCondition(c, a, 0)) + cs.seed(1) + + configs = cs.sample_configuration(size=100) + config_array = smac.configspace.convert_configurations_to_array(configs) + for line in config_array: + if line[0] == 0: + self.assertTrue(np.isnan(line[1])) + elif line[0] == 1: + self.assertTrue(np.isnan(line[2])) + + model = RandomForestWithInstances( + configspace=cs, + types=np.zeros((3,), dtype=np.uint), + bounds=list(map(lambda x: (0, 1), range(10))), + seed=1, + ) + config_array = model._impute_inactive(config_array) + for line in config_array: + if line[0] == 0: + self.assertEqual(line[1], 2) + elif line[0] == 1: + self.assertEqual(line[2], -1) diff --git a/test/test_epm/test_rf_with_instances_hpo.py b/test/test_epm/test_rf_with_instances_hpo.py index 9936fcc82..c1ef81b47 100644 --- a/test/test_epm/test_rf_with_instances_hpo.py +++ b/test/test_epm/test_rf_with_instances_hpo.py @@ -1,11 +1,11 @@ import unittest import unittest.mock -import george import numpy as np import sklearn.datasets import sklearn.model_selection +import smac.configspace from smac.epm.rf_with_instances_hpo import RandomForestWithInstancesHPO @@ -13,13 +13,18 @@ def get_rf(n_dimensions, rs): bounds = [(0., 1.) for _ in range(n_dimensions)] types = np.zeros(n_dimensions) + configspace = smac.configspace.ConfigurationSpace() + for i in range(n_dimensions): + configspace.add_hyperparameter(smac.configspace.UniformFloatHyperparameter('x%d' % i, 0, 1)) + model = RandomForestWithInstancesHPO( - types=types, bounds=bounds, log_y=False, bootstrap=False, n_iters=5, n_splits=5, + configspace=configspace, types=types, bounds=bounds, log_y=False, bootstrap=False, n_iters=5, n_splits=5, seed=1, ) return model class TestRandomForestWithInstancesHPO(unittest.TestCase): + def test_predict_wrong_X_dimensions(self): rs = np.random.RandomState(1) model = get_rf(10, rs) @@ -93,7 +98,6 @@ def test_predict_with_actual_values(self): y_hat, var_hat = model.predict(np.array([[10., 10., 10.]])) self.assertAlmostEqual(y_hat[0][0], 109.2) # No variance because the data point is outside of the observed data - print(var_hat) self.assertAlmostEqual(var_hat[0][0], 0) def test_rf_on_sklearn_data(self): @@ -104,7 +108,7 @@ def test_rf_on_sklearn_data(self): model = get_rf(X.shape[1], rs) cv = sklearn.model_selection.KFold(shuffle=True, random_state=rs, n_splits=2) - maes = [9.129075955836567871, 9.367454539503001523] + maes = [9.212142058553677762, 9.002105421756146103] for i, (train_split, test_split) in enumerate(cv.split(X, y)): X_train = X[train_split] diff --git a/test/test_epm/test_uncorrelated_mo_rf_with_instances.py b/test/test_epm/test_uncorrelated_mo_rf_with_instances.py index 042c098a9..7c1cb1fe9 100644 --- a/test/test_epm/test_uncorrelated_mo_rf_with_instances.py +++ b/test/test_epm/test_uncorrelated_mo_rf_with_instances.py @@ -1,32 +1,38 @@ import sys import unittest +from unittest import mock import numpy as np +import smac.configspace from smac.epm.uncorrelated_mo_rf_with_instances import \ UncorrelatedMultiObjectiveRandomForestWithInstances from smac.epm.rf_with_instances import RandomForestWithInstances -if sys.version_info[0] == 2: - import mock -else: - from unittest import mock - class TestUncorrelatedMultiObjectiveWrapper(unittest.TestCase): + + def _get_cs(self, n_dimensions): + configspace = smac.configspace.ConfigurationSpace() + for i in range(n_dimensions): + configspace.add_hyperparameter(smac.configspace.UniformFloatHyperparameter('x%d' % i, 0, 1)) + return configspace + def test_train_and_predict_with_rf(self): rs = np.random.RandomState(1) X = rs.rand(20, 10) Y = rs.rand(10, 2) model = UncorrelatedMultiObjectiveRandomForestWithInstances( - ['cost', 'ln(runtime)'], + configspace=self._get_cs(10), + target_names=['cost', 'ln(runtime)'], types=np.zeros((10, ), dtype=np.uint), - bounds=np.array([ + bounds=[ (0, np.nan), (0, np.nan), (0, np.nan), (0, np.nan), (0, np.nan), (0, np.nan), (0, np.nan), (0, np.nan), (0, np.nan), (0, np.nan) - ], dtype=object), + ], + seed=1, rf_kwargs={'seed': 1}, - pca_components=5 + pca_components=5, ) self.assertEqual(model.estimators[0].seed, 1) self.assertEqual(model.estimators[1].seed, 1) @@ -55,12 +61,15 @@ def __call__(self, X): X = rs.rand(20, 10) Y = rs.rand(10, 3) model = UncorrelatedMultiObjectiveRandomForestWithInstances( - ['cost', 'ln(runtime)', 'foo'], + target_names=['cost', 'ln(runtime)', 'foo'], + configspace=self._get_cs(10), types=np.zeros((10,), dtype=np.uint), - bounds=np.array([ + bounds=[ (0, np.nan), (0, np.nan), (0, np.nan), (0, np.nan), (0, np.nan), (0, np.nan), (0, np.nan), (0, np.nan), (0, np.nan), (0, np.nan) - ], dtype=object), + ], + seed=1, + rf_kwargs={'seed': 1}, ) model.train(X[:10], Y[:10]) diff --git a/test/test_epm/test_util_funcs.py b/test/test_epm/test_util_funcs.py new file mode 100644 index 000000000..441c442ec --- /dev/null +++ b/test/test_epm/test_util_funcs.py @@ -0,0 +1,56 @@ +import unittest + +from ConfigSpace.hyperparameters import CategoricalHyperparameter, \ + UniformFloatHyperparameter, UniformIntegerHyperparameter, Constant, \ + OrdinalHyperparameter +from ConfigSpace import ConfigurationSpace, EqualsCondition +import numpy as np + +from smac.epm.util_funcs import get_types + + +class TestUtilFuncs(unittest.TestCase): + def test_get_types(self): + cs = ConfigurationSpace() + cs.add_hyperparameter(CategoricalHyperparameter('a', ['a', 'b'])) + cs.add_hyperparameter(UniformFloatHyperparameter('b', 1, 5)) + cs.add_hyperparameter(UniformIntegerHyperparameter('c', 3, 7)) + cs.add_hyperparameter(Constant('d', -5)) + cs.add_hyperparameter(OrdinalHyperparameter('e', ['cold', 'hot'])) + cs.add_hyperparameter(CategoricalHyperparameter('f', ['x', 'y'])) + types, bounds = get_types(cs, None) + np.testing.assert_array_equal(types, [2, 0, 0, 0, 0, 2]) + self.assertEqual(bounds[0][0], 2) + self.assertFalse(np.isfinite(bounds[0][1])) + np.testing.assert_array_equal(bounds[1], [0, 1]) + np.testing.assert_array_equal(bounds[2], [0, 1]) + self.assertEqual(bounds[3][0], 0) + self.assertFalse(np.isfinite(bounds[3][1])) + np.testing.assert_array_equal(bounds[4], [0, 1]) + self.assertEqual(bounds[5][0], 2) + self.assertFalse(np.isfinite(bounds[5][1])) + + def test_get_types_with_inactive(self): + cs = ConfigurationSpace() + a = cs.add_hyperparameter(CategoricalHyperparameter('a', ['a', 'b'])) + b = cs.add_hyperparameter(UniformFloatHyperparameter('b', 1, 5)) + c = cs.add_hyperparameter(UniformIntegerHyperparameter('c', 3, 7)) + d = cs.add_hyperparameter(Constant('d', -5)) + e = cs.add_hyperparameter(OrdinalHyperparameter('e', ['cold', 'hot'])) + f = cs.add_hyperparameter(CategoricalHyperparameter('f', ['x', 'y'])) + cs.add_condition(EqualsCondition(b, a, 'a')) + cs.add_condition(EqualsCondition(c, a, 'a')) + cs.add_condition(EqualsCondition(d, a, 'a')) + cs.add_condition(EqualsCondition(e, a, 'a')) + cs.add_condition(EqualsCondition(f, a, 'a')) + types, bounds = get_types(cs, None) + np.testing.assert_array_equal(types, [2, 0, 0, 2, 0, 3]) + self.assertEqual(bounds[0][0], 2) + self.assertFalse(np.isfinite(bounds[0][1])) + np.testing.assert_array_equal(bounds[1], [-1, 1]) + np.testing.assert_array_equal(bounds[2], [-1, 1]) + self.assertEqual(bounds[3][0], 2) + self.assertFalse(np.isfinite(bounds[3][1])) + np.testing.assert_array_equal(bounds[4], [0, 2]) + self.assertEqual(bounds[5][0], 3) + self.assertFalse(np.isfinite(bounds[5][1])) \ No newline at end of file diff --git a/test/test_facade/test_hydra_facade.py b/test/test_facade/test_hydra_facade.py index 5b090971d..dacddbc2d 100644 --- a/test/test_facade/test_hydra_facade.py +++ b/test/test_facade/test_hydra_facade.py @@ -7,7 +7,7 @@ import numpy as np -from smac.facade.hydra_facade import Hydra, PSMAC +from smac.facade.experimental.hydra_facade import Hydra, PSMAC from smac.utils.io.output_writer import OutputWriter from smac.scenario.scenario import Scenario @@ -32,8 +32,8 @@ def get_best_incumbents_ids(self, incs): global MOCKCALLS for inc in incs: # in successive runs will always be smaller -> hydra doesn't terminate early - cost_per_conf_v[inc] = cost_per_conf_e[inc] = {inst: max(100 - MOCKCALLS, - 0) for inst in self.scenario.train_insts} + cost_per_conf_v[inc] = cost_per_conf_e[inc] = {inst: max(100 - MOCKCALLS, 0) + for inst in self.scenario.train_insts} if not self.validate: cost_per_conf_v = val_ids = None return cost_per_conf_v, val_ids, cost_per_conf_e, est_ids @@ -46,13 +46,13 @@ def setUp(self): fn = os.path.join(os.path.dirname(__file__), '../test_files/spear_hydra_test_scenario.txt') self.scenario = Scenario(fn) - @patch('smac.facade.hydra_facade.PSMAC', new=MockPSMAC) + @patch('smac.facade.experimental.hydra_facade.PSMAC', new=MockPSMAC) def test_hydra(self): optimizer = Hydra(self.scenario, n_iterations=3) portfolio = optimizer.optimize() self.assertEqual(len(portfolio), 3) - @patch('smac.facade.hydra_facade.PSMAC', new=MockPSMAC) + @patch('smac.facade.experimental.hydra_facade.PSMAC', new=MockPSMAC) def test_hydra_mip(self): optimizer = Hydra(self.scenario, n_iterations=3, incs_per_round=2) portfolio = optimizer.optimize() diff --git a/test/test_facade/test_psmac_facade.py b/test/test_facade/test_psmac_facade.py index fdb622959..5a60280b8 100644 --- a/test/test_facade/test_psmac_facade.py +++ b/test/test_facade/test_psmac_facade.py @@ -2,10 +2,11 @@ import shutil import os import glob +import joblib import unittest from unittest.mock import patch -from smac.facade.psmac_facade import PSMAC +from smac.facade.experimental.psmac_facade import PSMAC from smac.optimizer.smbo import SMBO from smac.scenario.scenario import Scenario @@ -26,17 +27,18 @@ def setUp(self): fn = os.path.join(os.path.dirname(__file__), '../test_files/spear_hydra_test_scenario.txt') self.scenario = Scenario(fn) - @patch('smac.facade.smac_facade.SMBO', new=MockSMBO) + @patch('smac.facade.smac_ac_facade.SMBO', new=MockSMBO) def test_psmac(self): - optimizer = PSMAC(self.scenario, n_optimizers=3, n_incs=2, validate=False) - incs = optimizer.optimize() - self.assertEquals(len(incs), 2) - optimizer = PSMAC(self.scenario, n_optimizers=1, n_incs=4, validate=False) - incs = optimizer.optimize() - self.assertEquals(len(incs), 2) - optimizer = PSMAC(self.scenario, n_optimizers=5, n_incs=4, validate=False) - incs = optimizer.optimize() - self.assertEquals(len(incs), 4) + with joblib.parallel_backend('multiprocessing', n_jobs=1): + optimizer = PSMAC(self.scenario, n_optimizers=3, n_incs=2, validate=False) + incs = optimizer.optimize() + self.assertEqual(len(incs), 2) + optimizer = PSMAC(self.scenario, n_optimizers=1, n_incs=4, validate=False) + incs = optimizer.optimize() + self.assertEqual(len(incs), 2) + optimizer = PSMAC(self.scenario, n_optimizers=5, n_incs=4, validate=False) + incs = optimizer.optimize() + self.assertEqual(len(incs), 4) def tearDown(self): hydras = glob.glob1('.', 'psmac*') @@ -48,4 +50,4 @@ def tearDown(self): shutil.rmtree(dirname) for output_dir in self.output_dirs: if output_dir: - shutil.rmtree(output_dir, ignore_errors=True) \ No newline at end of file + shutil.rmtree(output_dir, ignore_errors=True) diff --git a/test/test_facade/test_roar_facade.py b/test/test_facade/test_roar_facade.py index 069cac828..6b47330b6 100644 --- a/test/test_facade/test_roar_facade.py +++ b/test/test_facade/test_roar_facade.py @@ -23,7 +23,7 @@ def setUp(self): self.scenario = Scenario({'cs': self.cs, 'run_obj': 'quality', 'output_dir': ''}) self.output_dirs = [] - + def tearDown(self): shutil.rmtree('run_1', ignore_errors=True) for i in range(20): @@ -34,14 +34,6 @@ def tearDown(self): if output_dir: shutil.rmtree(output_dir, ignore_errors=True) - def test_inject_stats_and_runhistory_object_to_TAE(self): - ta = ExecuteTAFuncArray(lambda x: x**2) - self.assertIsNone(ta.stats) - self.assertIsNone(ta.runhistory) - ROAR(tae_runner=ta, scenario=self.scenario) - self.assertIsInstance(ta.stats, Stats) - self.assertIsInstance(ta.runhistory, RunHistory) - @attr('slow') def test_check_deterministic_rosenbrock(self): def rosenbrock_2d(x): @@ -77,4 +69,4 @@ def opt_rosenbrock(): x1_2 = i2.get('x1') x2_2 = i2.get('x2') self.assertAlmostEqual(x1_1, x1_2) - self.assertAlmostEqual(x2_1, x2_2) \ No newline at end of file + self.assertAlmostEqual(x2_1, x2_2) diff --git a/test/test_facade/test_smac_facade.py b/test/test_facade/test_smac_facade.py index 337986a04..81f616759 100644 --- a/test/test_facade/test_smac_facade.py +++ b/test/test_facade/test_smac_facade.py @@ -11,19 +11,28 @@ from smac.configspace import ConfigurationSpace -from smac.epm.base_epm import AbstractEPM -from smac.facade.smac_facade import SMAC +from smac.epm.random_epm import RandomEPM +from smac.epm.rf_with_instances import RandomForestWithInstances +from smac.epm.uncorrelated_mo_rf_with_instances import UncorrelatedMultiObjectiveRandomForestWithInstances +from smac.epm.util_funcs import get_rng +from smac.facade.smac_ac_facade import SMAC4AC from smac.initial_design.default_configuration_design import DefaultConfiguration +from smac.initial_design.initial_design import InitialDesign +from smac.initial_design.random_configuration_design import RandomConfigurations +from smac.initial_design.latin_hypercube_design import LHDesign +from smac.initial_design.factorial_design import FactorialInitialDesign +from smac.initial_design.sobol_design import SobolDesign from smac.intensification.intensification import Intensifier from smac.runhistory.runhistory import RunHistory -from smac.runhistory.runhistory2epm import RunHistory2EPM4Cost +from smac.runhistory.runhistory2epm import RunHistory2EPM4EIPS from smac.scenario.scenario import Scenario -from smac.optimizer.acquisition import EI, AbstractAcquisitionFunction -from smac.stats.stats import Stats +from smac.optimizer.acquisition import EI, EIPS, LCB +from smac.optimizer.random_configuration_chooser import ( + ChooserCosineAnnealing, + ChooserProb, +) from smac.tae.execute_func import ExecuteTAFuncDict -from smac.tae.execute_ta_run import ExecuteTARun -from smac.utils.io.traj_logging import TrajLogger -from smac.utils.util_funcs import get_rng + class TestSMACFacade(unittest.TestCase): @@ -42,20 +51,15 @@ def tearDown(self): if output_dir: shutil.rmtree(output_dir, ignore_errors=True) - def test_inject_stats_and_runhistory_object_to_TAE(self): - ta = ExecuteTAFuncDict(lambda x: x**2) - self.assertIsNone(ta.stats) - self.assertIsNone(ta.runhistory) - SMAC(tae_runner=ta, scenario=self.scenario) - self.assertIsInstance(ta.stats, Stats) - self.assertIsInstance(ta.runhistory, RunHistory) + #################################################################################################################### + # Test that the objects are constructed correctly def test_pass_callable(self): # Check that SMAC accepts a callable as target algorithm and that it is # correctly wrapped with ExecuteTaFunc def target_algorithm(conf, inst): return 5 - smac = SMAC(tae_runner=target_algorithm, scenario=self.scenario) + smac = SMAC4AC(tae_runner=target_algorithm, scenario=self.scenario) self.assertIsInstance(smac.solver.intensifier.tae_runner, ExecuteTAFuncDict) self.assertIs(smac.solver.intensifier.tae_runner.ta, target_algorithm) @@ -63,27 +67,176 @@ def target_algorithm(conf, inst): def test_pass_invalid_tae_runner(self): self.assertRaisesRegex( TypeError, - "Argument 'tae_runner' is , but must be either a callable or an instance of ExecuteTaRun.", - SMAC, + "Argument 'tae_runner' is , but must be either None, a callable or an " + "object implementing ExecuteTaRun.", + SMAC4AC, tae_runner=1, scenario=self.scenario, ) def test_pass_tae_runner_objective(self): - tae = ExecuteTAFuncDict(lambda: 1, run_obj='runtime') self.assertRaisesRegex( ValueError, "Objective for the target algorithm runner and the scenario must be the same, but are 'runtime' and " "'quality'", - SMAC, - tae_runner=tae, + SMAC4AC, + tae_runner=lambda: 1, + tae_runner_kwargs={'run_obj': 'runtime'}, scenario=self.scenario, ) - @unittest.mock.patch.object(SMAC, '__init__') + def test_construct_runhistory(self): + fixture = 'dummy' + + smbo = SMAC4AC(self.scenario) + self.assertIsInstance(smbo.solver.runhistory, RunHistory) + smbo = SMAC4AC(self.scenario, runhistory_kwargs={'aggregate_func': fixture}) + self.assertEqual(smbo.solver.runhistory.aggregate_func, fixture) + smbo = SMAC4AC(self.scenario, runhistory=RunHistory) + self.assertIsInstance(smbo.solver.runhistory, RunHistory) + rh = RunHistory(aggregate_func=None) + smbo = SMAC4AC(self.scenario, runhistory=rh) + self.assertIsNotNone(smbo.solver.runhistory.aggregate_func) + + def test_construct_random_configuration_chooser(self): + rng = np.random.RandomState(42) + smbo = SMAC4AC(self.scenario) + self.assertIsInstance(smbo.solver.random_configuration_chooser, ChooserProb) + self.assertIsNot(smbo.solver.random_configuration_chooser, rng) + smbo = SMAC4AC(self.scenario, rng=rng) + self.assertIsInstance(smbo.solver.random_configuration_chooser, ChooserProb) + self.assertIs(smbo.solver.random_configuration_chooser.rng, rng) + smbo = SMAC4AC(self.scenario, random_configuration_chooser_kwargs={'rng': rng}) + self.assertIsInstance(smbo.solver.random_configuration_chooser, ChooserProb) + self.assertIs(smbo.solver.random_configuration_chooser.rng, rng) + smbo = SMAC4AC(self.scenario, random_configuration_chooser_kwargs={'prob': 0.1}) + self.assertIsInstance(smbo.solver.random_configuration_chooser, ChooserProb) + self.assertEqual(smbo.solver.random_configuration_chooser.prob, 0.1) + smbo = SMAC4AC( + self.scenario, + random_configuration_chooser=ChooserCosineAnnealing, + random_configuration_chooser_kwargs={'prob_max': 1, 'prob_min': 0.1, 'restart_iteration': 10}, + ) + self.assertIsInstance(smbo.solver.random_configuration_chooser, ChooserCosineAnnealing) + # Check for construction failure on wrong argument + with self.assertRaisesRegex(Exception, 'got an unexpected keyword argument'): + SMAC4AC(self.scenario, random_configuration_chooser_kwargs={'dummy': 0.1}) + + def test_construct_epm(self): + rng = np.random.RandomState(42) + smbo = SMAC4AC(self.scenario) + self.assertIsInstance(smbo.solver.model, RandomForestWithInstances) + smbo = SMAC4AC(self.scenario, rng=rng) + self.assertIsInstance(smbo.solver.model, RandomForestWithInstances) + self.assertEqual(smbo.solver.model.seed, 1935803228) + smbo = SMAC4AC(self.scenario, model_kwargs={'seed': 2}) + self.assertIsInstance(smbo.solver.model, RandomForestWithInstances) + self.assertEqual(smbo.solver.model.seed, 2) + smbo = SMAC4AC(self.scenario, model_kwargs={'num_trees': 20}) + self.assertIsInstance(smbo.solver.model, RandomForestWithInstances) + self.assertEqual(smbo.solver.model.rf_opts.num_trees, 20) + smbo = SMAC4AC(self.scenario, model=RandomEPM, model_kwargs={'seed': 2}) + self.assertIsInstance(smbo.solver.model, RandomEPM) + self.assertEqual(smbo.solver.model.seed, 2) + # Check for construction failure on wrong argument + with self.assertRaisesRegex(Exception, 'got an unexpected keyword argument'): + SMAC4AC(self.scenario, model_kwargs={'dummy': 0.1}) + + def test_construct_acquisition_function(self): + rng = np.random.RandomState(42) + smbo = SMAC4AC(self.scenario) + self.assertIsInstance(smbo.solver.acquisition_func, EI) + smbo = SMAC4AC(self.scenario, rng=rng) + self.assertIsInstance(smbo.solver.acquisition_func.model, RandomForestWithInstances) + self.assertEqual(smbo.solver.acquisition_func.model.seed, 1935803228) + smbo = SMAC4AC(self.scenario, acquisition_function_kwargs={'par': 17}) + self.assertIsInstance(smbo.solver.acquisition_func, EI) + self.assertEqual(smbo.solver.acquisition_func.par, 17) + smbo = SMAC4AC(self.scenario, acquisition_function=LCB, acquisition_function_kwargs={'par': 19}) + self.assertIsInstance(smbo.solver.acquisition_func, LCB) + self.assertEqual(smbo.solver.acquisition_func.par, 19) + # Check for construction failure on wrong argument + with self.assertRaisesRegex(Exception, 'got an unexpected keyword argument'): + SMAC4AC(self.scenario, acquisition_function_kwargs={'dummy': 0.1}) + + def test_construct_intensifier(self): + + class DummyIntensifier(Intensifier): + pass + + rng = np.random.RandomState(42) + smbo = SMAC4AC(self.scenario) + self.assertIsInstance(smbo.solver.intensifier, Intensifier) + self.assertIsNot(smbo.solver.intensifier.rs, rng) + smbo = SMAC4AC(self.scenario, rng=rng) + self.assertIsInstance(smbo.solver.intensifier, Intensifier) + self.assertIs(smbo.solver.intensifier.rs, rng) + smbo = SMAC4AC(self.scenario, intensifier_kwargs={'maxR': 987}) + self.assertEqual(smbo.solver.intensifier.maxR, 987) + smbo = SMAC4AC( + self.scenario, intensifier=DummyIntensifier, intensifier_kwargs={'maxR': 987}, + ) + self.assertIsInstance(smbo.solver.intensifier, DummyIntensifier) + self.assertEqual(smbo.solver.intensifier.maxR, 987) + # Check for construction failure on wrong argument + with self.assertRaisesRegex(Exception, 'got an unexpected keyword argument'): + SMAC4AC(self.scenario, intensifier_kwargs={'dummy': 0.1}) + + def test_construct_initial_design(self): + + rng = np.random.RandomState(42) + smbo = SMAC4AC(self.scenario) + self.assertIsInstance(smbo.solver.initial_design, DefaultConfiguration) + self.assertIsNot(smbo.solver.intensifier.rs, rng) + smbo = SMAC4AC(self.scenario, rng=rng) + self.assertIsInstance(smbo.solver.intensifier, Intensifier) + self.assertIs(smbo.solver.intensifier.rs, rng) + smbo = SMAC4AC(self.scenario, intensifier_kwargs={'maxR': 987}) + self.assertEqual(smbo.solver.intensifier.maxR, 987) + smbo = SMAC4AC( + self.scenario, + initial_design=InitialDesign, + initial_design_kwargs={'configs': 'dummy'}, + ) + self.assertIsInstance(smbo.solver.initial_design, InitialDesign) + self.assertEqual(smbo.solver.initial_design.configs, 'dummy') + # Check for construction failure on wrong argument + with self.assertRaisesRegex(Exception, 'got an unexpected keyword argument'): + SMAC4AC(self.scenario, intensifier_kwargs={'dummy': 0.1}) + + for initial_incumbent_string, expected_instance in ( + ("DEFAULT", DefaultConfiguration), + ("RANDOM", RandomConfigurations), + ("LHD", LHDesign), + ("FACTORIAL", FactorialInitialDesign), + ("SOBOL", SobolDesign), + ): + self.scenario.initial_incumbent = initial_incumbent_string + smbo = SMAC4AC(self.scenario) + self.assertIsInstance(smbo.solver.initial_design, expected_instance) + + def test_init_EIPS_as_arguments(self): + for objective in ['runtime', 'quality']: + self.scenario.run_obj = objective + smbo = SMAC4AC( + self.scenario, + model=UncorrelatedMultiObjectiveRandomForestWithInstances, + model_kwargs={'target_names': ['a', 'b'], 'rf_kwargs': {'seed': 1}}, + acquisition_function=EIPS, + runhistory2epm=RunHistory2EPM4EIPS, + ).solver + self.assertIsInstance(smbo.model, UncorrelatedMultiObjectiveRandomForestWithInstances) + self.assertIsInstance(smbo.acquisition_func, EIPS) + self.assertIsInstance(smbo.acquisition_func.model, UncorrelatedMultiObjectiveRandomForestWithInstances) + self.assertIsInstance(smbo.rh2EPM, RunHistory2EPM4EIPS) + + #################################################################################################################### + # Other tests... + + @unittest.mock.patch.object(SMAC4AC, '__init__') def test_check_random_states(self, patch): patch.return_value = None - smac = SMAC() + smac = SMAC4AC() smac.logger = unittest.mock.MagicMock() # Check some properties @@ -178,8 +331,8 @@ def opt_rosenbrock(): "intensification_percentage": 0.000000001 }) - smac = SMAC(scenario=scenario, rng=np.random.RandomState(42), - tae_runner=rosenbrock_2d) + smac = SMAC4AC(scenario=scenario, rng=np.random.RandomState(42), + tae_runner=rosenbrock_2d) incumbent = smac.optimize() return incumbent, smac.scenario.output_dir @@ -195,66 +348,16 @@ def opt_rosenbrock(): self.assertAlmostEqual(x2_1, x2_2) def test_get_runhistory_and_trajectory_and_tae_runner(self): - ta = ExecuteTAFuncDict(lambda x: x ** 2) - smac = SMAC(tae_runner=ta, scenario=self.scenario) + def func(x): + return x ** 2 + smac = SMAC4AC(tae_runner=func, scenario=self.scenario) self.assertRaises(ValueError, smac.get_runhistory) self.assertRaises(ValueError, smac.get_trajectory) smac.trajectory = 'dummy' self.assertEqual(smac.get_trajectory(), 'dummy') smac.runhistory = 'dummy' self.assertEqual(smac.get_runhistory(), 'dummy') - self.assertEqual(smac.get_tae_runner(), ta) - - def test_inject_dependencies(self): - # initialize objects with missing dependencies - ta = ExecuteTAFuncDict(lambda x: x ** 2) - rh = RunHistory(aggregate_func=None) - acqu_func = EI(model=None) - intensifier = Intensifier(tae_runner=None, - stats=None, - traj_logger=None, - rng=np.random.RandomState(), - instances=None) - init_design = DefaultConfiguration(tae_runner=None, - scenario=None, - stats=None, - traj_logger=None, - rng=np.random.RandomState()) - rh2epm = RunHistory2EPM4Cost(scenario=self.scenario, num_params=0) - rh2epm.scenario = None - - # assert missing dependencies - self.assertIsNone(rh.aggregate_func) - self.assertIsNone(acqu_func.model) - self.assertIsNone(intensifier.tae_runner) - self.assertIsNone(intensifier.stats) - self.assertIsNone(intensifier.traj_logger) - self.assertIsNone(init_design.tae_runner) - self.assertIsNone(init_design.scenario) - self.assertIsNone(init_design.stats) - self.assertIsNone(init_design.traj_logger) - self.assertIsNone(rh2epm.scenario) - - # initialize smac-object - SMAC(scenario=self.scenario, - tae_runner=ta, - runhistory=rh, - intensifier=intensifier, - acquisition_function=acqu_func, - runhistory2epm=rh2epm, - initial_design=init_design) - - # assert that missing dependencies are injected - self.assertIsNotNone(rh.aggregate_func, AbstractAcquisitionFunction) - self.assertIsInstance(acqu_func.model, AbstractEPM) - self.assertIsInstance(intensifier.tae_runner, ExecuteTARun) - self.assertIsInstance(intensifier.stats, Stats) - self.assertIsInstance(intensifier.traj_logger, TrajLogger) - self.assertIsInstance(init_design.tae_runner, ExecuteTARun) - self.assertIsInstance(init_design.scenario, Scenario) - self.assertIsInstance(init_design.stats, Stats) - self.assertIsInstance(init_design.traj_logger, TrajLogger) - self.assertIsInstance(rh2epm.scenario, Scenario) + self.assertEqual(smac.get_tae_runner().ta, func) def test_output_structure(self): """Test whether output-dir is moved correctly.""" @@ -265,19 +368,19 @@ def test_output_structure(self): } scen1 = Scenario(test_scenario_dict) self.output_dirs.append(scen1.output_dir) - smac = SMAC(scenario=scen1, run_id=1) + smac = SMAC4AC(scenario=scen1, run_id=1) self.assertEqual(smac.output_dir, os.path.join( test_scenario_dict['output_dir'], 'run_1')) self.assertTrue(os.path.isdir(smac.output_dir)) - smac2 = SMAC(scenario=scen1, run_id=1) + smac2 = SMAC4AC(scenario=scen1, run_id=1) self.assertTrue(os.path.isdir(smac2.output_dir + '.OLD')) - smac3 = SMAC(scenario=scen1, run_id=1) + smac3 = SMAC4AC(scenario=scen1, run_id=1) self.assertTrue(os.path.isdir(smac3.output_dir + '.OLD.OLD')) - smac4 = SMAC(scenario=scen1, run_id=2) + smac4 = SMAC4AC(scenario=scen1, run_id=2) self.assertEqual(smac4.output_dir, os.path.join( test_scenario_dict['output_dir'], 'run_2')) self.assertTrue(os.path.isdir(smac4.output_dir)) @@ -299,5 +402,5 @@ def test_no_output(self): 'cs': ConfigurationSpace() } scen1 = Scenario(test_scenario_dict) - smac = SMAC(scenario=scen1, run_id=1) + smac = SMAC4AC(scenario=scen1, run_id=1) self.assertFalse(os.path.isdir(smac.output_dir)) diff --git a/test/test_files/spear_qcp b/test/test_files/spear_qcp index ea9ceb280..a5bdc5aff 120000 --- a/test/test_files/spear_qcp +++ b/test/test_files/spear_qcp @@ -1 +1 @@ -../../examples/spear_qcp/ \ No newline at end of file +../../examples/spear_qcp \ No newline at end of file diff --git a/test/test_initial_design/test_multiple_config_initial_design.py b/test/test_initial_design/test_multiple_config_initial_design.py deleted file mode 100644 index f3348f243..000000000 --- a/test/test_initial_design/test_multiple_config_initial_design.py +++ /dev/null @@ -1,52 +0,0 @@ -import unittest - -import numpy as np - -from ConfigSpace.hyperparameters import UniformFloatHyperparameter - -from smac.configspace import ConfigurationSpace, Configuration - -from smac.runhistory.runhistory import RunHistory -from smac.facade.smac_facade import SMAC -from smac.scenario.scenario import Scenario -from smac.stats.stats import Stats -from smac.tae.execute_func import ExecuteTAFuncDict -from smac.initial_design.multi_config_initial_design import MultiConfigInitialDesign -from smac.utils.io.traj_logging import TrajLogger -from smac.optimizer.objective import average_cost -from smac.intensification.intensification import Intensifier - -class TestMultiInitialDesign(unittest.TestCase): - - def setUp(self): - self.cs = ConfigurationSpace() - self.cs.add_hyperparameter(UniformFloatHyperparameter( - name="x1", lower=1, upper=10, default_value=2) - ) - self.scenario = Scenario({'cs': self.cs, 'run_obj': 'quality', - 'output_dir': ''}) - self.ta = ExecuteTAFuncDict(lambda x: x["x1"]**2) - - def test_multi_config_design(self): - stats = Stats(scenario=self.scenario) - stats.start_timing() - self.ta.stats = stats - tj = TrajLogger(output_dir=None, stats=stats) - rh = RunHistory(aggregate_func=average_cost) - self.ta.runhistory = rh - rng = np.random.RandomState(seed=12345) - - intensifier = Intensifier(tae_runner=self.ta, stats=stats, traj_logger=tj, rng=rng, instances=[None], - run_obj_time=False) - - configs = [Configuration(configuration_space=self.cs, values={"x1":4}), - Configuration(configuration_space=self.cs, values={"x1":2})] - dc = MultiConfigInitialDesign(tae_runner=self.ta, scenario=self.scenario, stats=stats, - traj_logger=tj, runhistory=rh, rng=rng, configs=configs, - intensifier=intensifier, aggregate_func=average_cost) - - inc = dc.run() - self.assertTrue(stats.ta_runs==4) # two runs per config - self.assertTrue(len(rh.data)==4) # two runs per config - self.assertTrue(rh.get_cost(inc) == 4) - diff --git a/test/test_initial_design/test_single_config_initial_design.py b/test/test_initial_design/test_single_config_initial_design.py index 5afdc5537..9f2feb563 100644 --- a/test/test_initial_design/test_single_config_initial_design.py +++ b/test/test_initial_design/test_single_config_initial_design.py @@ -1,19 +1,18 @@ import unittest import numpy as np - -from ConfigSpace.hyperparameters import UniformFloatHyperparameter +from ConfigSpace import Configuration, UniformFloatHyperparameter from smac.configspace import ConfigurationSpace - +from smac.initial_design.default_configuration_design import DefaultConfiguration +from smac.initial_design.initial_design import InitialDesign +from smac.intensification.intensification import Intensifier +from smac.optimizer.objective import average_cost from smac.runhistory.runhistory import RunHistory -from smac.facade.smac_facade import SMAC from smac.scenario.scenario import Scenario from smac.stats.stats import Stats -from smac.tae.execute_func import ExecuteTAFuncDict -from smac.initial_design.default_configuration_design import DefaultConfiguration from smac.utils.io.traj_logging import TrajLogger -from smac.optimizer.objective import average_cost +from smac.tae.execute_func import ExecuteTAFuncDict class TestSingleInitialDesign(unittest.TestCase): @@ -34,10 +33,54 @@ def test_single_default_config_design(self): tj = TrajLogger(output_dir=None, stats=stats) rh = RunHistory(aggregate_func=average_cost) - dc = DefaultConfiguration(tae_runner=self.ta, scenario=self.scenario, - stats=stats, traj_logger=tj, - rng=np.random.RandomState(seed=12345)) + dc = DefaultConfiguration( + tae_runner=self.ta, + scenario=self.scenario, + stats=stats, + traj_logger=tj, + rng=np.random.RandomState(seed=12345), + runhistory=rh, + intensifier=None, + aggregate_func=average_cost, + ) inc = dc.run() self.assertTrue(stats.ta_runs==1) - self.assertTrue(len(rh.data)==0) \ No newline at end of file + self.assertTrue(len(rh.data)==0) + + def test_multi_config_design(self): + stats = Stats(scenario=self.scenario) + stats.start_timing() + self.ta.stats = stats + tj = TrajLogger(output_dir=None, stats=stats) + rh = RunHistory(aggregate_func=average_cost) + self.ta.runhistory = rh + rng = np.random.RandomState(seed=12345) + + intensifier = Intensifier( + tae_runner=self.ta, + stats=stats, + traj_logger=tj, + rng=rng, + instances=[None], + run_obj_time=False, + ) + + configs = [Configuration(configuration_space=self.cs, values={"x1":4}), + Configuration(configuration_space=self.cs, values={"x1":2})] + dc = InitialDesign( + tae_runner=self.ta, + scenario=self.scenario, + stats=stats, + traj_logger=tj, + runhistory=rh, + rng=rng, + configs=configs, + intensifier=intensifier, + aggregate_func=average_cost, + ) + + inc = dc.run() + self.assertTrue(stats.ta_runs==4) # two runs per config + self.assertTrue(len(rh.data)==4) # two runs per config + self.assertTrue(rh.get_cost(inc) == 4) diff --git a/test/test_integration/eips.py b/test/test_integration/eips.py deleted file mode 100644 index 355fc8cc3..000000000 --- a/test/test_integration/eips.py +++ /dev/null @@ -1,40 +0,0 @@ -import time -import unittest - -from smac.scenario.scenario import Scenario -from smac.utils import test_helpers -from smac.optimizer.smbo import SMBO, get_types -from smac.optimizer.ei_optimization import ChooserNoCoolDown -from smac.runhistory.runhistory2epm import RunHistory2EPM4EIPS -from smac.epm.uncorrelated_mo_rf_with_instances import \ - UncorrelatedMultiObjectiveRandomForestWithInstances -from smac.optimizer.acquisition import EIPS -from smac.tae.execute_func import ExecuteTAFunc - - -def test_function(conf): - x = conf['x'] - y = conf['y'] - runtime = y / 150. - time.sleep(runtime) - return x - y - - -class TestEIPS(unittest.TestCase): - def test_eips(self): - scenario = Scenario({'cs': test_helpers.get_branin_config_space(), - 'run_obj': 'quality', - 'deterministic': True, - 'output_dir': ''}) - types = get_types(scenario.cs, None) - umrfwi = UncorrelatedMultiObjectiveRandomForestWithInstances( - ['cost', 'runtime'], types) - eips = EIPS(umrfwi) - rh2EPM = RunHistory2EPM4EIPS(scenario, 2) - taf = ExecuteTAFunc(test_function) - smbo = SMBO(scenario, model=umrfwi, acquisition_function=eips, - runhistory2epm=rh2EPM, tae_runner=taf, - random_configuration_chooser=ChooserNoCoolDown(2.0)) - smbo.run(5) - print(smbo.incumbent) - raise ValueError() diff --git a/test/test_runhistory/test_rfr_imputor.py b/test/test_runhistory/test_rfr_imputor.py index e6f4a2525..ebb655781 100644 --- a/test/test_runhistory/test_rfr_imputor.py +++ b/test/test_runhistory/test_rfr_imputor.py @@ -16,8 +16,8 @@ from smac.scenario import scenario from smac.epm import rfr_imputator from smac.epm.rf_with_instances import RandomForestWithInstances +from smac.epm.util_funcs import get_types from smac.optimizer.objective import average_cost -from smac.utils.util_funcs import get_types def generate_config(cs, rs): @@ -97,8 +97,12 @@ def setUp(self): types, bounds = get_types(self.cs, None) self.model = RandomForestWithInstances( - types=types, bounds=bounds, - instance_features=None, seed=1234567980) + configspace=self.cs, + types=types, + bounds=bounds, + instance_features=None, + seed=1234567980, + ) @attr('slow') def testRandomImputation(self): @@ -136,9 +140,13 @@ def testRandomImputation(self): print(cen_X) print(uncen_X) print('~'*120) - self.model = RandomForestWithInstances(types=types, bounds=bounds, - instance_features=None, - seed=1234567980) + self.model = RandomForestWithInstances( + configspace=cs, + types=types, + bounds=bounds, + instance_features=None, + seed=1234567980, + ) imputor = rfr_imputator.RFRImputator(rng=rs, cutoff=cutoff, threshold=cutoff*10, diff --git a/test/test_runhistory/test_runhistory2epm.py b/test/test_runhistory/test_runhistory2epm.py index 5d589034a..3280be077 100644 --- a/test/test_runhistory/test_runhistory2epm.py +++ b/test/test_runhistory/test_runhistory2epm.py @@ -19,7 +19,7 @@ from smac.optimizer.objective import average_cost from smac.epm.rfr_imputator import RFRImputator from smac.epm.rf_with_instances import RandomForestWithInstances -from smac.utils.util_funcs import get_types +from smac.epm.util_funcs import get_types def get_config_space(): @@ -57,15 +57,19 @@ def test_log_runtime_with_imputation(self): ''' adding some rundata to RunHistory2EPM4LogCost and impute censored data ''' - self.imputor = RFRImputator(rng=np.random.RandomState(seed=12345), - cutoff=np.log(self.scen.cutoff), - threshold=np.log( - self.scen.cutoff * self.scen.par_factor), - model=RandomForestWithInstances(types=self.types, bounds=self.bounds, - instance_features=None, - seed=12345, - ratio_features=1.0) - ) + self.imputor = RFRImputator( + rng=np.random.RandomState(seed=12345), + cutoff=np.log(self.scen.cutoff), + threshold=np.log(self.scen.cutoff * self.scen.par_factor), + model=RandomForestWithInstances( + configspace=self.cs, + types=self.types, + bounds=self.bounds, + instance_features=None, + seed=12345, + ratio_features=1.0, + ) + ) rh2epm = runhistory2epm.RunHistory2EPM4LogCost(num_params=2, scenario=self.scen, @@ -105,7 +109,7 @@ def test_log_runtime_with_imputation(self): [0.995, 0.005], [0.995, 0.995]]), decimal=3) - + np.testing.assert_array_almost_equal(y, np.array([[0.], [2.727], [5.2983]]), decimal=3) @@ -155,15 +159,20 @@ def test_cost_with_imputation(self): adding some rundata to RunHistory2EPM4Cost and impute censored data ''' - self.imputor = RFRImputator(rng=np.random.RandomState(seed=12345), - - cutoff=self.scen.cutoff, - threshold=self.scen.cutoff * self.scen.par_factor, - model=RandomForestWithInstances(types=self.types, bounds=self.bounds, - instance_features=None, - seed=12345, n_points_per_tree=90, - ratio_features=1.0) - ) + self.imputor = RFRImputator( + rng=np.random.RandomState(seed=12345), + cutoff=self.scen.cutoff, + threshold=self.scen.cutoff * self.scen.par_factor, + model=RandomForestWithInstances( + configspace=self.cs, + types=self.types, + bounds=self.bounds, + instance_features=None, + seed=12345, + n_points_per_tree=90, + ratio_features=1.0, + ) + ) rh2epm = runhistory2epm.RunHistory2EPM4Cost(num_params=2, scenario=self.scen, @@ -210,8 +219,7 @@ def test_cost_without_imputation(self): adding some rundata to RunHistory2EPM4Cost without imputation ''' - rh2epm = runhistory2epm.RunHistory2EPM4Cost(num_params=2, - scenario=self.scen) + rh2epm = runhistory2epm.RunHistory2EPM4Cost(num_params=2, scenario=self.scen) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=23, @@ -222,17 +230,16 @@ def test_cost_without_imputation(self): self.assertTrue(np.allclose(X, np.array([[0.005, 0.995]]), atol=0.001)) self.assertTrue(np.allclose(y, np.array([[1.]]))) - # rh2epm should use time and not cost field later - self.rh.add(config=self.config3, cost=2, time=20, + # rh2epm should use cost and not time + self.rh.add(config=self.config3, cost=200, time=20, status=StatusType.TIMEOUT, instance_id=1, seed=45, additional_info={"start_time": 20}) X, y = rh2epm.transform(self.rh) - self.assertTrue( - np.allclose(X, np.array([[0.005, 0.995], [0.995, 0.995]]), atol=0.001)) + np.testing.assert_allclose(X, np.array([[0.005, 0.995], [0.995, 0.995]]), atol=0.001) # log_10(20 * 10) - self.assertTrue(np.allclose(y, np.array([[1.], [200.]]), atol=0.001)) + np.testing.assert_allclose(y, np.array([[1.], [200.]]), atol=0.001) self.rh.add(config=self.config2, cost=100, time=10, status=StatusType.TIMEOUT, instance_id=1, @@ -279,14 +286,14 @@ def test_cost_quality(self): self.assertTrue(np.allclose(y, np.array([[1.], [200.]]), atol=0.001)) #TODO: unit test for censored data in quality scenario - + def test_get_X_y(self): ''' add some data to RH and check returned values in X,y format ''' self.scen = Scenario({'cutoff_time': 20, 'cs': self.cs, - 'run_obj': 'runtime', + 'run_obj': 'runtime', 'instances': [['1'],['2']], 'features': { '1': [1,1], @@ -301,38 +308,38 @@ def test_get_X_y(self): status=StatusType.SUCCESS, instance_id='1', seed=None, additional_info=None) - + self.rh.add(config=self.config1, cost=2, time=10, status=StatusType.SUCCESS, instance_id='2', seed=None, additional_info=None) - + self.rh.add(config=self.config2, cost=1, time=10, status=StatusType.TIMEOUT, instance_id='1', seed=None, additional_info=None) - + self.rh.add(config=self.config2, cost=0.1, time=10, status=StatusType.CAPPED, instance_id='2', seed=None, additional_info=None) - + X,y,c = rh2epm.get_X_y(self.rh) - + print(X,y,c) - + X_sol = np.array([[0,100,1,1], [0,100,2,2], [100,0,1,1], [100,0,2,2]]) self.assertTrue(np.all(X==X_sol)) - + y_sol = np.array([1,2,1,0.1]) self.assertTrue(np.all(y==y_sol)) - + c_sol = np.array([False, False, True, True]) self.assertTrue(np.all(c==c_sol)) - + if __name__ == "__main__": unittest.main() diff --git a/test/test_smbo/test_acquisition.py b/test/test_smbo/test_acquisition.py index 0e89ae352..a28c62830 100644 --- a/test/test_smbo/test_acquisition.py +++ b/test/test_smbo/test_acquisition.py @@ -3,7 +3,7 @@ import numpy as np -from smac.optimizer.acquisition import EI, LogEI, EIPS, PI, LCB +from smac.optimizer.acquisition import EI, LogEI, EIPS, PI, LCB, IntegratedAcquisitionFunction class ConfigurationMock(object): @@ -39,6 +39,78 @@ def predict_marginalized_over_instances(self, X): self.num_targets).reshape((-1, 2)) +class TestIntegratedAcquisitionFunction(unittest.TestCase): + def setUp(self): + self.model = unittest.mock.Mock() + self.model.models = [MockModel(), MockModel(), MockModel()] + self.ei = EI(self.model) + + def test_update(self): + iaf = IntegratedAcquisitionFunction(model=self.model, acquisition_function=self.ei) + iaf.update(model=self.model, eta=2) + for func in iaf._functions: + self.assertEqual(func.eta, 2) + + with self.assertRaisesRegex( + ValueError, + 'IntegratedAcquisitionFunction requires at least one model to integrate!', + ): + iaf.update(model=MockModel()) + + with self.assertRaisesRegex( + ValueError, + 'IntegratedAcquisitionFunction requires at least one model to integrate!', + ): + self.model.models = [] + iaf.update(model=self.model) + + def test_compute(self): + class CountingMock: + counter = 0 + long_name = 'CountingMock' + + def _compute(self, *args, **kwargs): + self.counter += 1 + return self.counter + + def update(self, **kwargs): + pass + + iaf = IntegratedAcquisitionFunction(model=self.model, acquisition_function=CountingMock()) + iaf.update(model=self.model) + configurations = [ConfigurationMock([1.0, 1.0, 1.0])] + rval = iaf(configurations) + self.assertEqual(rval, 1) + + # Test that every counting mock is called only once! + for counting_mock in iaf._functions: + self.assertEqual(counting_mock.counter, 1) + + def test_compute_with_different_numbers_of_models(self): + class CountingMock: + counter = 0 + long_name = 'CountingMock' + + def _compute(self, *args, **kwargs): + self.counter += 1 + return self.counter + + def update(self, **kwargs): + pass + + for i in range(1, 3): + self.model.models = [MockModel()] * i + iaf = IntegratedAcquisitionFunction(model=self.model, acquisition_function=self.ei) + iaf.update(model=self.model, eta=1) + configurations = [ConfigurationMock([1.0, 1.0, 1.0])] + rval = iaf(configurations) + self.assertEqual(rval.shape, (1, 1)) + + configurations = [ConfigurationMock([1.0, 1.0, 1.0]), ConfigurationMock([1.0, 2.0, 3.0])] + rval = iaf(configurations) + self.assertEqual(rval.shape, (2, 1)) + + class TestEI(unittest.TestCase): def setUp(self): self.model = MockModel() @@ -108,14 +180,14 @@ class TestLogEI(unittest.TestCase): def setUp(self): self.model = MockModel() self.ei = LogEI(self.model) - + def test_1xD(self): self.ei.update(model=self.model, eta=1.0) configurations = [ConfigurationMock([1.0, 1.0, 1.0])] acq = self.ei(configurations) self.assertEqual(acq.shape, (1, 1)) self.assertAlmostEqual(acq[0][0], 0.6480973967332011) - + def test_NxD(self): self.ei.update(model=self.model, eta=1.0) configurations = [ConfigurationMock([0.1, 0.0, 0.0]), diff --git a/test/test_smbo/test_ei_optimization.py b/test/test_smbo/test_ei_optimization.py index 6f0bfd347..b958ca2fd 100644 --- a/test/test_smbo/test_ei_optimization.py +++ b/test/test_smbo/test_ei_optimization.py @@ -10,10 +10,12 @@ import numpy as np from scipy.spatial.distance import euclidean -from smac.facade.smac_facade import SMAC from smac.configspace import pcs +from smac.optimizer.objective import average_cost from smac.optimizer.acquisition import EI from smac.optimizer.ei_optimization import LocalSearch, RandomSearch +from smac.runhistory.runhistory import RunHistory +from smac.tae.execute_ta_run import StatusType from smac.configspace import ConfigurationSpace from ConfigSpace.hyperparameters import CategoricalHyperparameter, \ UniformFloatHyperparameter, UniformIntegerHyperparameter @@ -40,6 +42,7 @@ def rosenbrock_4d(cfg): return(val) + class TestLocalSearch(unittest.TestCase): def setUp(self): current_dir = os.path.dirname(__file__) @@ -58,18 +61,19 @@ def setUp(self): def test_local_search(self): - def acquisition_function(point): - point = [p.get_array() for p in point] - opt = np.array([1, 1, 1, 1]) - dist = [euclidean(point, opt)] - return np.array([-np.min(dist)]) + def acquisition_function(points): + rval = [] + for point in points: + opt = np.array([1, 1, 1, 1]) + rval.append([-euclidean(point.get_array(), opt)]) + return np.array(rval) - l = LocalSearch(acquisition_function, self.cs, max_steps=100000) + l = LocalSearch(acquisition_function, self.cs, max_steps=100) start_point = self.cs.sample_configuration() acq_val_start_point = acquisition_function([start_point]) - acq_val_incumbent, _ = l._one_iter(start_point) + acq_val_incumbent, _ = l._do_search(start_point)[0] # Local search needs to find something that is as least as good as the # start point @@ -90,8 +94,8 @@ def test_local_search_2( config_space = pcs.read(fh.readlines()) config_space.seed(seed) - def acquisition_function(point): - return np.array([np.count_nonzero(point[0].get_array())]) + def acquisition_function(points): + return np.array([[np.count_nonzero(point.get_array())] for point in points]) start_point = config_space.get_default_configuration() _get_initial_points_patch.return_value = [start_point] @@ -106,7 +110,7 @@ def acquisition_function(point): np.ones(len(config_space.get_hyperparameters())) ) - @unittest.mock.patch.object(LocalSearch, '_one_iter') + @unittest.mock.patch.object(LocalSearch, '_do_search') @unittest.mock.patch.object(LocalSearch, '_get_initial_points') def test_get_next_by_local_search( self, @@ -115,13 +119,12 @@ def test_get_next_by_local_search( ): # Without known incumbent class SideEffect(object): - def __init__(self): - self.call_number = 0 def __call__(self, *args, **kwargs): - rval = 9 - self.call_number - self.call_number += 1 - return (rval, ConfigurationMock(rval)) + rval = [] + for i in range(len(args[0])): + rval.append((i, ConfigurationMock(i))) + return rval patch.side_effect = SideEffect() cs = test_helpers.get_branin_config_space() @@ -137,29 +140,58 @@ def __call__(self, *args, **kwargs): rval = ls._maximize(runhistory, None, 9) self.assertEqual(len(rval), 9) - self.assertEqual(patch.call_count, 9) + self.assertEqual(patch.call_count, 1) for i in range(9): self.assertIsInstance(rval[i][1], ConfigurationMock) - self.assertEqual(rval[i][1].value, 9 - i) - self.assertEqual(rval[i][0], 9 - i) + self.assertEqual(rval[i][1].value, 8 - i) + self.assertEqual(rval[i][0], 8 - i) self.assertEqual(rval[i][1].origin, 'Local Search') - # With known incumbent + # Check that the known 'incumbent' is transparently passed through patch.side_effect = SideEffect() _get_initial_points_patch.return_value = ['Incumbent'] + rand_confs rval = ls._maximize(runhistory, None, 10) self.assertEqual(len(rval), 10) - self.assertEqual(patch.call_count, 19) + self.assertEqual(patch.call_count, 2) # Only the first local search in each iteration starts from the # incumbent - self.assertEqual(patch.call_args_list[9][0][0], 'Incumbent') + self.assertEqual(patch.call_args_list[1][0][0][0], 'Incumbent') for i in range(10): self.assertEqual(rval[i][1].origin, 'Local Search') + def test_local_search_finds_minimum(self): + + class AcquisitionFunction: + + model = None + + def __call__(self, arrays): + rval = [] + for array in arrays: + rval.append([-rosenbrock_4d(array)]) + return np.array(rval) + + ls = LocalSearch( + acquisition_function=AcquisitionFunction(), + config_space=self.cs, + n_steps_plateau_walk=10, + max_steps=np.inf, + ) + + runhistory = RunHistory(aggregate_func=average_cost) + self.cs.seed(1) + random_configs = self.cs.sample_configuration(size=100) + costs = [rosenbrock_4d(random_config) for random_config in random_configs] + self.assertGreater(np.min(costs), 100) + for random_config, cost in zip(random_configs, costs): + runhistory.add(config=random_config, cost=cost, time=0, status=StatusType.SUCCESS) + minimizer = ls.maximize(runhistory, None, 10) + minima = [-rosenbrock_4d(m) for m in minimizer] + self.assertGreater(minima[0], -0.05) class TestRandomSearch(unittest.TestCase): - @unittest.mock.patch('smac.optimizer.ei_optimization.convert_configurations_to_array') + @unittest.mock.patch('smac.optimizer.acquisition.convert_configurations_to_array') @unittest.mock.patch.object(EI, '__call__') @unittest.mock.patch.object(ConfigurationSpace, 'sample_configuration') def test_get_next_by_random_search_sorted(self, @@ -206,5 +238,6 @@ def side_effect(size): self.assertEqual(rval[i][1].origin, 'Random Search') self.assertEqual(rval[i][0], 0) + if __name__ == "__main__": unittest.main() diff --git a/test/test_smbo/test_epils.py b/test/test_smbo/test_epils.py index 7cbfc532a..71c71cff0 100644 --- a/test/test_smbo/test_epils.py +++ b/test/test_smbo/test_epils.py @@ -15,9 +15,9 @@ from smac.epm.rf_with_instances import RandomForestWithInstances from smac.epm.uncorrelated_mo_rf_with_instances import \ UncorrelatedMultiObjectiveRandomForestWithInstances -from smac.utils.util_funcs import get_types -from smac.facade.epils_facade import EPILS -from smac.initial_design.single_config_initial_design import SingleConfigInitialDesign +from smac.epm.util_funcs import get_types +from smac.facade.experimental.epils_facade import EPILS +from smac.initial_design.initial_design import InitialDesign if sys.version_info[0] == 2: import mock @@ -49,12 +49,11 @@ def tearDown(self): shutil.rmtree(dirname) def branin(self, config): - print(config) y = (config[1] - (5.1 / (4 * np.pi ** 2)) * config[0] ** 2 + 5 * config[0] / np.pi - 6) ** 2 y += 10 * (1 - 1 / (8 * np.pi)) * np.cos(config[0]) + 10 return y - + def test_epils(self): taf = ExecuteTAFuncArray(ta=self.branin) epils = EPILS(self.scenario, tae_runner=taf) @@ -83,11 +82,11 @@ def test_init_EIPS_as_arguments(self): self.scenario.run_obj = objective types, bounds = get_types(self.scenario.cs, None) umrfwi = UncorrelatedMultiObjectiveRandomForestWithInstances( - ['cost', 'runtime'], types, bounds) + ['cost', 'runtime'], self.scenario.cs, types, bounds, seed=1, rf_kwargs={'seed': 1},) eips = EIPS(umrfwi) rh2EPM = RunHistory2EPM4EIPS(self.scenario, 2) epils = EPILS(self.scenario, model=umrfwi, acquisition_function=eips, - runhistory2epm=rh2EPM).solver + runhistory2epm=rh2EPM).solver self.assertIs(umrfwi, epils.model) self.assertIs(eips, epils.acquisition_func) self.assertIs(rh2EPM, epils.rh2EPM) @@ -109,7 +108,7 @@ def test_rng(self): 'np.random.RandomState', EPILS, self.scenario, rng='BLA') - @mock.patch.object(SingleConfigInitialDesign, 'run') + @mock.patch.object(InitialDesign, 'run') def test_abort_on_initial_design(self, patch): def target(x): return 5 diff --git a/test/test_smbo/test_random_configuration_chooser.py b/test/test_smbo/test_random_configuration_chooser.py index 65be906d2..f129208af 100644 --- a/test/test_smbo/test_random_configuration_chooser.py +++ b/test/test_smbo/test_random_configuration_chooser.py @@ -7,7 +7,7 @@ class TestRandomConfigurationChooser(unittest.TestCase): def test_no_cool_down(self): - c = ChooserNoCoolDown(modulus=3.0) + c = ChooserNoCoolDown(rng=None, modulus=3.0) self.assertFalse(c.check(1)) self.assertFalse(c.check(2)) self.assertTrue(c.check(3)) @@ -23,7 +23,7 @@ def test_no_cool_down(self): self.assertFalse(c.check(5)) self.assertTrue(c.check(6)) self.assertTrue(c.check(30)) - c = ChooserNoCoolDown(modulus=1.0) + c = ChooserNoCoolDown(rng=None, modulus=1.0) self.assertTrue(c.check(1)) self.assertTrue(c.check(2)) c.next_smbo_iteration() @@ -31,7 +31,7 @@ def test_no_cool_down(self): self.assertTrue(c.check(2)) def test_linear_cool_down(self): - c = ChooserLinearCoolDown(2.0, 1.0, 4.0) + c = ChooserLinearCoolDown(None, 2.0, 1.0, 4.0) self.assertFalse(c.check(1)) self.assertTrue(c.check(2)) self.assertFalse(c.check(3)) diff --git a/test/test_smbo/test_smbo.py b/test/test_smbo/test_smbo.py index 83c91b61b..3cd515185 100644 --- a/test/test_smbo/test_smbo.py +++ b/test/test_smbo/test_smbo.py @@ -1,7 +1,6 @@ -from contextlib import suppress import unittest from unittest import mock -import os +import sys import shutil from nose.plugins.attrib import attr @@ -10,22 +9,19 @@ from smac.epm.rf_with_instances import RandomForestWithInstances from smac.epm.gaussian_process_mcmc import GaussianProcessMCMC -from smac.epm.uncorrelated_mo_rf_with_instances import \ - UncorrelatedMultiObjectiveRandomForestWithInstances -from smac.facade.smac_facade import SMAC -from smac.initial_design.single_config_initial_design import SingleConfigInitialDesign -from smac.optimizer.acquisition import EI, EIPS, LogEI +from smac.facade.smac_ac_facade import SMAC4AC +from smac.initial_design.initial_design import InitialDesign +from smac.optimizer.acquisition import EI, LogEI from smac.optimizer.objective import average_cost from smac.runhistory.runhistory import RunHistory -from smac.runhistory.runhistory2epm import RunHistory2EPM4Cost, \ - RunHistory2EPM4LogCost, RunHistory2EPM4EIPS +from smac.runhistory.runhistory2epm import RunHistory2EPM4Cost, RunHistory2EPM4LogCost from smac.scenario.scenario import Scenario from smac.tae.execute_ta_run import FirstRunCrashedException from smac.utils import test_helpers -from smac.utils.util_funcs import get_types from smac.utils.io.traj_logging import TrajLogger from smac.utils.validate import Validator +from test import requires_extra class ConfigurationMock(object): @@ -59,40 +55,26 @@ def branin(self, x): def test_init_only_scenario_runtime(self): self.scenario.run_obj = 'runtime' self.scenario.cutoff = 300 - smbo = SMAC(self.scenario).solver + smbo = SMAC4AC(self.scenario).solver self.assertIsInstance(smbo.model, RandomForestWithInstances) self.assertIsInstance(smbo.rh2EPM, RunHistory2EPM4LogCost) self.assertIsInstance(smbo.acquisition_func, LogEI) def test_init_only_scenario_quality(self): - smbo = SMAC(self.scenario).solver + smbo = SMAC4AC(self.scenario).solver self.assertIsInstance(smbo.model, RandomForestWithInstances) self.assertIsInstance(smbo.rh2EPM, RunHistory2EPM4Cost) self.assertIsInstance(smbo.acquisition_func, EI) - def test_init_EIPS_as_arguments(self): - for objective in ['runtime', 'quality']: - self.scenario.run_obj = objective - types, bounds = get_types(self.scenario.cs, None) - umrfwi = UncorrelatedMultiObjectiveRandomForestWithInstances( - ['cost', 'runtime'], types, bounds) - eips = EIPS(umrfwi) - rh2EPM = RunHistory2EPM4EIPS(self.scenario, 2) - smbo = SMAC(self.scenario, model=umrfwi, acquisition_function=eips, - runhistory2epm=rh2EPM).solver - self.assertIs(umrfwi, smbo.model) - self.assertIs(eips, smbo.acquisition_func) - self.assertIs(rh2EPM, smbo.rh2EPM) - def test_rng(self): - smbo = SMAC(self.scenario, rng=None).solver + smbo = SMAC4AC(self.scenario, rng=None).solver self.assertIsInstance(smbo.rng, np.random.RandomState) self.assertIsInstance(smbo.num_run, int) - smbo = SMAC(self.scenario, rng=1).solver + smbo = SMAC4AC(self.scenario, rng=1).solver rng = np.random.RandomState(1) self.assertEqual(smbo.num_run, 1) self.assertIsInstance(smbo.rng, np.random.RandomState) - smbo = SMAC(self.scenario, rng=rng).solver + smbo = SMAC4AC(self.scenario, rng=rng).solver self.assertIsInstance(smbo.num_run, int) self.assertIs(smbo.rng, rng) # ML: I don't understand the following line and it throws an error @@ -100,14 +82,14 @@ def test_rng(self): TypeError, "Argument rng accepts only arguments of type None, int or np.random.RandomState, you provided " ".", - SMAC, + SMAC4AC, self.scenario, rng='BLA', ) def test_choose_next(self): seed = 42 - smbo = SMAC(self.scenario, rng=seed).solver + smbo = SMAC4AC(self.scenario, rng=seed).solver smbo.runhistory = RunHistory(aggregate_func=average_cost) X = self.scenario.cs.sample_configuration().get_array()[None, :] smbo.incumbent = self.scenario.cs.sample_configuration() @@ -119,7 +101,7 @@ def test_choose_next(self): def test_choose_next_w_empty_rh(self): seed = 42 - smbo = SMAC(self.scenario, rng=seed).solver + smbo = SMAC4AC(self.scenario, rng=seed).solver smbo.runhistory = RunHistory(aggregate_func=average_cost) X = self.scenario.cs.sample_configuration().get_array()[None, :] @@ -129,14 +111,14 @@ def test_choose_next_w_empty_rh(self): 'Runhistory is empty and the cost value of the incumbent is ' 'unknown.', smbo.choose_next, - **{"X":X, "Y":Y} + **{"X": X, "Y": Y} ) x = next(smbo.choose_next(X, Y, incumbent_value=0.0)).get_array() assert x.shape == (2,) def test_choose_next_empty_X(self): - smbo = SMAC(self.scenario, rng=1).solver + smbo = SMAC4AC(self.scenario, rng=1).solver smbo.acquisition_func._compute = mock.Mock( spec=RandomForestWithInstances ) @@ -154,7 +136,7 @@ def test_choose_next_empty_X(self): self.assertEqual(smbo.acquisition_func._compute.call_count, 0) def test_choose_next_empty_X_2(self): - smbo = SMAC(self.scenario, rng=1).solver + smbo = SMAC4AC(self.scenario, rng=1).solver X = np.zeros((0, 2)) Y = np.zeros((0, 1)) @@ -173,7 +155,7 @@ def side_effect_predict(X): m, v = np.ones((X.shape[0], 1)), None return m, v - smbo = SMAC(self.scenario, rng=1).solver + smbo = SMAC4AC(self.scenario, rng=1).solver smbo.incumbent = self.scenario.cs.sample_configuration() smbo.runhistory = RunHistory(aggregate_func=average_cost) smbo.runhistory.add(smbo.incumbent, 10, 10, 1) @@ -194,7 +176,7 @@ def side_effect_predict(X): # For each configuration it is randomly sampled whether to take it from the list of challengers or to sample it # completely at random. Therefore, it is not guaranteed to obtain twice the number of configurations selected # by EI. - self.assertEqual(len(challengers), 9913) + self.assertEqual(len(challengers), 9940) num_random_search_sorted = 0 num_random_search = 0 num_local_search = 0 @@ -207,12 +189,13 @@ def side_effect_predict(X): elif 'Local Search' == c.origin: num_local_search += 1 else: - raise ValueError(c.origin) + raise ValueError((c.origin, 'Local Search' == c.origin, type('Local Search'), type(c.origin))) - self.assertEqual(num_local_search, 1) - self.assertEqual(num_random_search_sorted, 4999) - self.assertEqual(num_random_search, 4913) + self.assertEqual(num_local_search, 11) + self.assertEqual(num_random_search_sorted, 5000) + self.assertEqual(num_random_search, 4929) + @unittest.skipIf(sys.version_info < (3, 6), 'Test not deterministic for Python 3.5 and earlier') def test_choose_next_3(self): # Test with ten configurations in the runhistory def side_effect(X): @@ -222,7 +205,7 @@ def side_effect_predict(X): m, v = np.ones((X.shape[0], 1)), None return m, v - smbo = SMAC(self.scenario, rng=1).solver + smbo = SMAC4AC(self.scenario, rng=1).solver smbo.incumbent = self.scenario.cs.sample_configuration() previous_configs = [smbo.incumbent] + [self.scenario.cs.sample_configuration() for i in range(0, 20)] smbo.runhistory = RunHistory(aggregate_func=average_cost) @@ -244,8 +227,8 @@ def side_effect_predict(X): # For each configuration it is randomly sampled whether to take it from the list of challengers or to sample it # completely at random. Therefore, it is not guaranteed to obtain twice the number of configurations selected - # by EI. - self.assertEqual(len(challengers), 9913) + # by EI + self.assertEqual(len(challengers), 9977) num_random_search_sorted = 0 num_random_search = 0 num_local_search = 0 @@ -260,11 +243,11 @@ def side_effect_predict(X): else: raise ValueError(c.origin) - self.assertEqual(num_local_search, 10) - self.assertEqual(num_random_search_sorted, 4990) - self.assertEqual(num_random_search, 4913) + self.assertEqual(num_local_search, 26) + self.assertEqual(num_random_search_sorted, 5000) + self.assertEqual(num_random_search, 4951) - @mock.patch.object(SingleConfigInitialDesign, 'run') + @mock.patch.object(InitialDesign, 'run') def test_abort_on_initial_design(self, patch): def target(x): return 5 @@ -273,20 +256,21 @@ def target(x): 'run_obj': 'quality', 'output_dir': 'data-test_smbo-abort', 'abort_on_first_run_crash': 1}) self.output_dirs.append(scen.output_dir) - smbo = SMAC(scen, tae_runner=target, rng=1).solver + smbo = SMAC4AC(scen, tae_runner=target, rng=1).solver self.assertRaises(FirstRunCrashedException, smbo.run) @attr('slow') def test_intensification_percentage(self): def target(x): return 5 + def get_smbo(intensification_perc): """ Return SMBO with intensification_percentage. """ scen = Scenario({'cs': test_helpers.get_branin_config_space(), 'run_obj': 'quality', 'output_dir': 'data-test_smbo-intensification', 'intensification_percentage': intensification_perc}) self.output_dirs.append(scen.output_dir) - return SMAC(scen, tae_runner=target, rng=1).solver + return SMAC4AC(scen, tae_runner=target, rng=1).solver # Test for valid values smbo = get_smbo(0.3) self.assertAlmostEqual(3.0, smbo._get_timebound_for_intensification(7.0)) @@ -307,58 +291,61 @@ def get_smbo(intensification_perc): def test_validation(self): with mock.patch.object(TrajLogger, "read_traj_aclib_format", - return_value=None) as traj_mock: + return_value=None) as traj_mock: self.scenario.output_dir = "test" - smac = SMAC(self.scenario) + smac = SMAC4AC(self.scenario) self.output_dirs.append(smac.output_dir) smbo = smac.solver - with mock.patch.object(Validator, "validate", - return_value=None) as validation_mock: + with mock.patch.object(Validator, "validate", return_value=None) as validation_mock: smbo.validate(config_mode='inc', instance_mode='train+test', repetitions=1, use_epm=False, n_jobs=-1, backend='threading') self.assertTrue(validation_mock.called) - with mock.patch.object(Validator, "validate_epm", - return_value=None) as epm_validation_mock: + with mock.patch.object(Validator, "validate_epm", return_value=None) as epm_validation_mock: smbo.validate(config_mode='inc', instance_mode='train+test', repetitions=1, use_epm=True, n_jobs=-1, backend='threading') self.assertTrue(epm_validation_mock.called) def test_no_initial_design(self): self.scenario.output_dir = "test" - smac = SMAC(self.scenario) + smac = SMAC4AC(self.scenario) self.output_dirs.append(smac.output_dir) smbo = smac.solver - with mock.patch.object(SingleConfigInitialDesign, "run", return_value=None) as initial_mock: + with mock.patch.object(InitialDesign, "run", return_value=None) as initial_mock: smbo.start() self.assertEqual(smbo.incumbent, smbo.scenario.cs.get_default_configuration()) - def test_comp_builder(self): + def test_rf_comp_builder(self): seed = 42 - smbo = SMAC(self.scenario, rng=seed).solver - conf = {"model":"RF", "acq_func":"EI"} + smbo = SMAC4AC(self.scenario, rng=seed).solver + conf = {"model": "RF", "acq_func": "EI"} acqf, model = smbo._component_builder(conf) - + self.assertTrue(isinstance(acqf, EI)) self.assertTrue(isinstance(model, RandomForestWithInstances)) - + + @requires_extra('gp') + def test_gp_comp_builder(self): + seed = 42 + smbo = SMAC4AC(self.scenario, rng=seed).solver conf = {"model":"GP", "acq_func":"EI"} acqf, model = smbo._component_builder(conf) - + self.assertTrue(isinstance(acqf, EI)) self.assertTrue(isinstance(model, GaussianProcessMCMC)) - + def test_smbo_cs(self): seed = 42 - smbo = SMAC(self.scenario, rng=seed).solver + smbo = SMAC4AC(self.scenario, rng=seed).solver cs = smbo._get_acm_cs() - + def test_cs_comp_builder(self): seed = 42 - smbo = SMAC(self.scenario, rng=seed).solver + smbo = SMAC4AC(self.scenario, rng=seed).solver cs = smbo._get_acm_cs() conf = cs.sample_configuration() acqf, model = smbo._component_builder(conf) + if __name__ == "__main__": unittest.main() diff --git a/test/test_utils/io/test_inputreader.py b/test/test_utils/io/test_inputreader.py index ee7d5fd11..bcde0ec98 100644 --- a/test/test_utils/io/test_inputreader.py +++ b/test/test_utils/io/test_inputreader.py @@ -9,7 +9,11 @@ import numpy as np +from smac.configspace import ConfigurationSpace +from ConfigSpace.hyperparameters import UniformFloatHyperparameter +from smac.configspace import pcs from smac.utils.io.input_reader import InputReader +from smac.utils.io.output_writer import OutputWriter class InputReaderTest(unittest.TestCase): @@ -24,7 +28,20 @@ def setUp(self): '..', '..')) os.chdir(base_directory) + # Files that will be created: + self.pcs_fn = "test/test_files/configspace.pcs" + self.json_fn = "test/test_files/configspace.json" + + self.output_files = [self.pcs_fn, self.json_fn] + def tearDown(self): + for output_file in self.output_files: + if output_file: + try: + os.remove(output_file) + except FileNotFoundError: + pass + os.chdir(self.current_dir) def test_feature_input(self): @@ -37,3 +54,36 @@ def test_feature_input(self): "inst3":[1.7, 1.8, 1.9]} for i in feats[1]: self.assertEqual(feats_original[i], list(feats[1][i])) + + def test_save_load_configspace(self): + """Check if inputreader can load different config-spaces""" + cs = ConfigurationSpace() + hyp = UniformFloatHyperparameter('A', 0.0, 1.0, default_value=0.5) + cs.add_hyperparameters([hyp]) + + output_writer = OutputWriter() + input_reader = InputReader() + + # pcs_new + output_writer.save_configspace(cs, self.pcs_fn, 'pcs_new') + restored_cs = input_reader.read_pcs_file(self.pcs_fn) + self.assertEqual(cs, restored_cs) + restored_cs = input_reader.read_pcs_file(self.pcs_fn, self.logger) + self.assertEqual(cs, restored_cs) + + # json + output_writer.save_configspace(cs, self.json_fn, 'json') + restored_cs = input_reader.read_pcs_file(self.json_fn) + self.assertEqual(cs, restored_cs) + restored_cs = input_reader.read_pcs_file(self.json_fn, self.logger) + self.assertEqual(cs, restored_cs) + + # pcs + with open(self.pcs_fn, 'w') as fh: + fh.write(pcs.write(cs)) + restored_cs = input_reader.read_pcs_file(self.pcs_fn) + self.assertEqual(cs, restored_cs) + restored_cs = input_reader.read_pcs_file(self.pcs_fn) + self.assertEqual(cs, restored_cs) + restored_cs = input_reader.read_pcs_file(self.pcs_fn, self.logger) + self.assertEqual(cs, restored_cs) diff --git a/tox.ini b/tox.ini index 87c0a63f9..f944460ef 100644 --- a/tox.ini +++ b/tox.ini @@ -7,5 +7,5 @@ deps = scipy > 0.9 nose commands= - python setup.py install + pip install .[all] python setup.py test