From 1c9f18711ad984581b42ef33df20a6d795c0d938 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Wed, 14 Aug 2019 16:11:37 +0200 Subject: [PATCH 1/4] Bump version number --- setup.py | 3 ++- smac/__init__.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 6cf4bd4f9..6f1020695 100644 --- a/setup.py +++ b/setup.py @@ -37,5 +37,6 @@ def get_author(): author=get_author(), version=get_version(), test_suite="nose.collector", - tests_require=["mock", "nose"] + tests_require=["mock", "nose"], + long_description_content_type='text/markdown', ) diff --git a/smac/__init__.py b/smac/__init__.py index 010b08ea4..687e76c86 100644 --- a/smac/__init__.py +++ b/smac/__init__.py @@ -5,7 +5,7 @@ import lazy_import from smac.utils import dependencies -__version__ = '0.11.0' +__version__ = '0.11.1dev' __author__ = 'Marius Lindauer, Matthias Feurer, Katharina Eggensperger, Joshua Marben, André Biedenkapp, Aaron Klein, Stefan Falkner and Frank Hutter' From afca7302f16cc6ad6a6bcc12081a79fbce466c2c Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 22 Aug 2019 14:51:22 +0200 Subject: [PATCH 2/4] Improve CI (#525) * run flake8 checks * re-order unit test execution order * change permissions * trigger travis * trigger travis * fix syntax error in yml file * Fix line length --- .travis.yml | 5 +- ci_scripts/run_flake8.sh | 141 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+), 1 deletion(-) create mode 100755 ci_scripts/run_flake8.sh diff --git a/.travis.yml b/.travis.yml index 62f17fba7..fc111a891 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,9 +20,12 @@ matrix: env: TESTSUITE=run_unittests.sh PYTHON_VERSION="3.7" COVERAGE="true" DOCPUSH="true" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" # Other tests (mypy, examples, flake8...) - os: linux - env: TESTSUITE=run_mypy.sh PYTHON_VERSION="3.6" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" + env: TESTSUITE=run_flake8.sh PYTHON_VERSION="3.6" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" - os: linux env: TESTSUITE=run_examples.sh PYTHON_VERSION="3.6" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" + - os: linux + env: TESTSUITE=run_mypy.sh PYTHON_VERSION="3.6" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" + # Disable OSX building because it takes too long and hinders progress # Set language to generic to not break travis-ci # https://github.com/travis-ci/travis-ci/issues/2312#issuecomment-195620855 diff --git a/ci_scripts/run_flake8.sh b/ci_scripts/run_flake8.sh new file mode 100755 index 000000000..7e5d2ce8c --- /dev/null +++ b/ci_scripts/run_flake8.sh @@ -0,0 +1,141 @@ +#!/bin/bash + +# This script is mostly taken from https://github.com/scikit-learn/scikit-learn/blob/master/build_tools/travis/flake8_diff.sh + +# This script is used in Travis to check that PRs do not add obvious +# flake8 violations. It relies on two things: +# - find common ancestor between branch and +# automl/auto-sklearn remote +# - run flake8 --diff on the diff between the branch and the common +# ancestor +# +# Additional features: +# - the line numbers in Travis match the local branch on the PR +# author machine. +# - ./build_tools/travis/flake8_diff.sh can be run locally for quick +# turn-around + +set -e +# pipefail is necessary to propagate exit codes +set -o pipefail + +PROJECT=automl/SMAC3 +PROJECT_URL=https://github.com/$PROJECT.git + +# Find the remote with the project name (upstream in most cases) +REMOTE=$(git remote -v | grep $PROJECT | cut -f1 | head -1 || echo '') + +# Add a temporary remote if needed. For example this is necessary when +# Travis is configured to run in a fork. In this case 'origin' is the +# fork and not the reference repo we want to diff against. +if [[ -z "$REMOTE" ]]; then + TMP_REMOTE=tmp_reference_upstream + REMOTE=$TMP_REMOTE + git remote add $REMOTE $PROJECT_URL +fi + +echo "Remotes:" +echo '--------------------------------------------------------------------------------' +git remote --verbose + +# Travis does the git clone with a limited depth. +# This may not be enough to find the common ancestor with +# $REMOTE/development so we unshallow the git checkout +if [[ -a .git/shallow ]]; then + echo -e '\nTrying to unshallow the repo:' + echo '--------------------------------------------------------------------------------' + git fetch --unshallow +fi + +if [[ "$TRAVIS" == "true" ]]; then + if [[ "$TRAVIS_PULL_REQUEST" == "false" ]] + then + # In main repo, using TRAVIS_COMMIT_RANGE to test the commits + # that were pushed into a branch + if [[ "$PROJECT" == "$TRAVIS_REPO_SLUG" ]]; then + if [[ -z "$TRAVIS_COMMIT_RANGE" ]]; then + echo "New branch, no commit range from Travis so passing this test by convention" + exit 0 + fi + COMMIT_RANGE=$TRAVIS_COMMIT_RANGE + fi + else + # We want to fetch the code as it is in the PR branch and not + # the result of the merge into development. This way line numbers + # reported by Travis will match with the local code. + LOCAL_BRANCH_REF=travis_pr_$TRAVIS_PULL_REQUEST + # In Travis the PR target is always origin + git fetch origin pull/$TRAVIS_PULL_REQUEST/head:refs/$LOCAL_BRANCH_REF + fi +fi + +# If not using the commit range from Travis we need to find the common +# ancestor between $LOCAL_BRANCH_REF and $REMOTE/development +if [[ -z "$COMMIT_RANGE" ]]; then + if [[ -z "$LOCAL_BRANCH_REF" ]]; then + LOCAL_BRANCH_REF=$(git rev-parse --abbrev-ref HEAD) + fi + echo -e "\nLast 2 commits in $LOCAL_BRANCH_REF:" + echo '--------------------------------------------------------------------------------' + git --no-pager log -2 $LOCAL_BRANCH_REF + + REMOTE_DEVELOPMENT_REF="$REMOTE/development" + # Make sure that $REMOTE_DEVELOPMENT_REF is a valid reference + echo -e "\nFetching $REMOTE_DEVELOPMENT_REF" + echo '--------------------------------------------------------------------------------' + git fetch $REMOTE development:refs/remotes/$REMOTE_DEVELOPMENT_REF + LOCAL_BRANCH_SHORT_HASH=$(git rev-parse --short $LOCAL_BRANCH_REF) + REMOTE_DEVELOPMENT_SHORT_HASH=$(git rev-parse --short $REMOTE_DEVELOPMENT_REF) + + COMMIT=$(git merge-base $LOCAL_BRANCH_REF $REMOTE_DEVELOPMENT_REF) || \ + echo "No common ancestor found for $(git show $LOCAL_BRANCH_REF -q) and $(git show $REMOTE_DEVELOPMENT_REF -q)" + + if [ -z "$COMMIT" ]; then + exit 1 + fi + + COMMIT_SHORT_HASH=$(git rev-parse --short $COMMIT) + + echo -e "\nCommon ancestor between $LOCAL_BRANCH_REF ($LOCAL_BRANCH_SHORT_HASH)"\ + "and $REMOTE_DEVELOPMENT_REF ($REMOTE_DEVELOPMENT_SHORT_HASH) is $COMMIT_SHORT_HASH:" + echo '--------------------------------------------------------------------------------' + git --no-pager show --no-patch $COMMIT_SHORT_HASH + + COMMIT_RANGE="$COMMIT_SHORT_HASH..$LOCAL_BRANCH_SHORT_HASH" + + if [[ -n "$TMP_REMOTE" ]]; then + git remote remove $TMP_REMOTE + fi + +else + echo "Got the commit range from Travis: $COMMIT_RANGE" +fi + +echo -e '\nRunning flake8 on the diff in the range' "$COMMIT_RANGE" \ + "($(git rev-list $COMMIT_RANGE | wc -l) commit(s)):" +echo '--------------------------------------------------------------------------------' + +# We need the following command to exit with 0 hence the echo in case +# there is no match +MODIFIED_FILES="$(git diff --name-only $COMMIT_RANGE || echo "no_match")" + +check_files() { + files="$1" + shift + options="$*" + if [ -n "$files" ]; then + # Conservative approach: diff without context (--unified=0) so that code + # that was not changed does not create failures + git diff --unified=0 $COMMIT_RANGE -- $files | flake8 --diff --max-line-length=120 --show-source $options + fi +} + +if [[ "$MODIFIED_FILES" == "no_match" ]]; then + echo "No file has been modified" +else + + check_files "$(echo "$MODIFIED_FILES" | grep -v ^examples)" + check_files "$(echo "$MODIFIED_FILES" | grep ^examples)" \ + --config ./examples/.flake8 +fi +echo -e "No problem detected by flake8\n" From 448316cdeb2da30fffafd74ffaf23edcac65154f Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Wed, 11 Sep 2019 11:31:42 +0200 Subject: [PATCH 3/4] update Gaussian process hyperparameters (#529) --- smac/epm/gaussian_process_mcmc.py | 3 --- smac/facade/smac_bo_facade.py | 16 ++++++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/smac/epm/gaussian_process_mcmc.py b/smac/epm/gaussian_process_mcmc.py index e64abecc7..8dd810fe4 100644 --- a/smac/epm/gaussian_process_mcmc.py +++ b/smac/epm/gaussian_process_mcmc.py @@ -176,7 +176,6 @@ def _train(self, X: np.ndarray, y: np.ndarray, do_optimize: bool=True): # Take the last samples from each walker self.hypers = sampler.chain[:, -1] - print('hypers', self.hypers.mean(axis=0), np.exp(self.hypers.mean(axis=0))) elif self.mcmc_sampler == 'nuts': # Originally published as: # http://www.stat.columbia.edu/~gelman/research/published/nuts.pdf @@ -210,7 +209,6 @@ def _train(self, X: np.ndarray, y: np.ndarray, do_optimize: bool=True): indices = [int(np.rint(ind)) for ind in np.linspace(start=0, stop=len(samples) - 1, num=10)] self.hypers = samples[indices] self.p0 = self.hypers.mean(axis=0) - print('hypers', 'log space', self.p0, 'regular space', np.exp(self.p0)) else: raise ValueError(self.mcmc_sampler) @@ -269,7 +267,6 @@ def _train(self, X: np.ndarray, y: np.ndarray, do_optimize: bool=True): model.std_y_ = self.std_y_ self.is_trained = True - print('#LL evaluations', self._n_ll_evals) def _ll(self, theta: np.ndarray) -> float: """ diff --git a/smac/facade/smac_bo_facade.py b/smac/facade/smac_bo_facade.py index 68583aabb..1eda02f73 100644 --- a/smac/facade/smac_bo_facade.py +++ b/smac/facade/smac_bo_facade.py @@ -5,7 +5,7 @@ from smac.epm.gp_base_prior import HorseshoePrior, LognormalPrior from smac.epm.util_funcs import get_types, get_rng from smac.initial_design.sobol_design import SobolDesign -from smac.runhistory.runhistory2epm import RunHistory2EPM4LogScaledCost +from smac.runhistory.runhistory2epm import RunHistory2EPM4Cost __author__ = "Marius Lindauer" @@ -20,6 +20,10 @@ class SMAC4BO(SMAC4AC): see smac.facade.smac_Facade for API This facade overwrites options available via the SMAC facade + Hyperparameters are chosen according to the best configuration for Gaussian process maximum likelihood found in + "Towards Assessing the Impact of Bayesian Optimization's Own Hyperparameters" by Lindauer et al., presented at the + DSO workshop 2019 (https://arxiv.org/abs/1908.06674). + Attributes ---------- logger @@ -40,10 +44,10 @@ def __init__(self, model_type='gp_mcmc', **kwargs): scenario = kwargs['scenario'] kwargs['initial_design'] = kwargs.get('initial_design', SobolDesign) - kwargs['runhistory2epm'] = kwargs.get('runhistory2epm', RunHistory2EPM4LogScaledCost) + kwargs['runhistory2epm'] = kwargs.get('runhistory2epm', RunHistory2EPM4Cost) init_kwargs = kwargs.get('initial_design_kwargs', dict()) - init_kwargs['n_configs_x_params'] = init_kwargs.get('n_configs_x_params', 10) + init_kwargs['n_configs_x_params'] = init_kwargs.get('n_configs_x_params', 8) init_kwargs['max_config_fracs'] = init_kwargs.get('max_config_fracs', 0.25) kwargs['initial_design_kwargs'] = init_kwargs @@ -68,7 +72,7 @@ def __init__(self, model_type='gp_mcmc', **kwargs): if len(cont_dims) > 0: exp_kernel = Matern( np.ones([len(cont_dims)]), - [(np.exp(-10), np.exp(2)) for _ in range(len(cont_dims))], + [(np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in range(len(cont_dims))], nu=2.5, operate_on=cont_dims, ) @@ -76,7 +80,7 @@ def __init__(self, model_type='gp_mcmc', **kwargs): if len(cat_dims) > 0: ham_kernel = HammingKernel( np.ones([len(cat_dims)]), - [(np.exp(-10), np.exp(2)) for _ in range(len(cat_dims))], + [(np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in range(len(cat_dims))], operate_on=cat_dims, ) @@ -125,7 +129,7 @@ def __init__(self, model_type='gp_mcmc', **kwargs): if kwargs.get('random_configuration_chooser') is None: random_config_chooser_kwargs = kwargs.get('random_configuration_chooser_kwargs', dict()) - random_config_chooser_kwargs['prob'] = random_config_chooser_kwargs.get('prob', 0.0) + random_config_chooser_kwargs['prob'] = random_config_chooser_kwargs.get('prob', 0.08447232371720552) kwargs['random_configuration_chooser_kwargs'] = random_config_chooser_kwargs if kwargs.get('acquisition_function_optimizer') is None: From 13f629e9b58728fa625f28e911c5449a5a7922bd Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Fri, 20 Sep 2019 14:11:10 +0200 Subject: [PATCH 4/4] Prepares the release of 0.11.1 (#530) * prepare new release * add HP changes of SMAC BO according to BOBO to constructor --- changelog.md | 7 +++++++ smac/__init__.py | 2 +- smac/facade/smac_bo_facade.py | 10 ++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 476c8f1b7..7ca3e33c7 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,10 @@ +# 0.11.1 + +## Changes + +* Updated the default hyperparameters of the Gaussian process facade to follow recent research (#529) +* Enabled `flake8` code style checks for newly merged code (#525) + # 0.11.0 ## Major changes diff --git a/smac/__init__.py b/smac/__init__.py index 687e76c86..83e85a4ea 100644 --- a/smac/__init__.py +++ b/smac/__init__.py @@ -5,7 +5,7 @@ import lazy_import from smac.utils import dependencies -__version__ = '0.11.1dev' +__version__ = '0.11.1' __author__ = 'Marius Lindauer, Matthias Feurer, Katharina Eggensperger, Joshua Marben, André Biedenkapp, Aaron Klein, Stefan Falkner and Frank Hutter' diff --git a/smac/facade/smac_bo_facade.py b/smac/facade/smac_bo_facade.py index 1eda02f73..60142e113 100644 --- a/smac/facade/smac_bo_facade.py +++ b/smac/facade/smac_bo_facade.py @@ -24,6 +24,16 @@ class SMAC4BO(SMAC4AC): "Towards Assessing the Impact of Bayesian Optimization's Own Hyperparameters" by Lindauer et al., presented at the DSO workshop 2019 (https://arxiv.org/abs/1908.06674). + Changes are: + * Instead of having an initial design of size 10*D as suggested by Jones et al. 1998 (actually, they suggested + 10*D+1), we use an initial design of 8*D. + * More restrictive lower and upper bounds on the length scale for the Matern and Hamming Kernel than the ones + suggested by Klein et al. 2017 in the RoBO package. In practice, they are ``np.exp(-6.754111155189306)`` + instead of ``np.exp(-10)`` for the lower bound and ``np.exp(0.0858637988771976)`` instead of + ``np.exp(2)`` for the upper bound. + * The initial design is set to be a Sobol grid + * The random fraction is set to ``0.08447232371720552``, it was ``0.0`` before. + Attributes ---------- logger