diff --git a/fedot/api/api_utils/api_params_repository.py b/fedot/api/api_utils/api_params_repository.py index 9a6e76f4ef..99c4f72d47 100644 --- a/fedot/api/api_utils/api_params_repository.py +++ b/fedot/api/api_utils/api_params_repository.py @@ -2,6 +2,8 @@ from functools import partial from typing import Sequence +from golem.core.optimisers.adaptive.context_agents import ContextAgentTypeEnum +from golem.core.optimisers.adaptive.operator_agent import MutationAgentTypeEnum from golem.core.optimisers.genetic.operators.inheritance import GeneticSchemeTypesEnum from golem.core.optimisers.genetic.operators.mutation import MutationTypesEnum @@ -68,7 +70,9 @@ def default_params_for_task(task_type: TaskTypesEnum) -> dict: cache_dir=default_fedot_data_dir(), keep_history=True, history_dir=default_fedot_data_dir(), - with_tuning=False + with_tuning=False, + adaptive_mutation_type=MutationAgentTypeEnum.default, + context_agent_type=ContextAgentTypeEnum.nodes_num ) return default_param_values_dict @@ -115,6 +119,9 @@ def get_params_for_gp_algorithm_params(self, params: dict) -> dict: """ Returns dict with parameters suitable for ``GPAlgorithmParameters``""" gp_algorithm_params = {'pop_size': params.get('pop_size'), 'genetic_scheme_type': GeneticSchemeTypesEnum.parameter_free} + + gp_algorithm_params = self._get_adaptive_optimization_params(params, gp_algorithm_params) + if params.get('genetic_scheme') == 'steady_state': gp_algorithm_params['genetic_scheme_type'] = GeneticSchemeTypesEnum.steady_state @@ -136,3 +143,22 @@ def _get_default_mutations(task_type: TaskTypesEnum, params) -> Sequence[Mutatio mutations.append(add_resample_mutation) return mutations + + @staticmethod + def _get_adaptive_optimization_params(params: dict, gp_algorithm_params: dict) -> dict: + """ Extracts params conserned with adaptive optimization including `adaptive_mutation_type` + and `context_agent_type` from params. """ + adaptive_mutation_type = params.get('adaptive_mutation_type') + context_agent_type = params.get('context_agent_type') + + if adaptive_mutation_type: + adaptive_mutation_type_class = MutationAgentTypeEnum[adaptive_mutation_type] \ + if isinstance(adaptive_mutation_type, str) else adaptive_mutation_type + gp_algorithm_params['adaptive_mutation_type'] = adaptive_mutation_type_class + + if context_agent_type: + context_agent_type_class = ContextAgentTypeEnum[context_agent_type] \ + if isinstance(context_agent_type, str) else context_agent_type + gp_algorithm_params['context_agent_type'] = context_agent_type_class + + return gp_algorithm_params diff --git a/fedot/api/main.py b/fedot/api/main.py index 5f5636e274..fc6636987e 100644 --- a/fedot/api/main.py +++ b/fedot/api/main.py @@ -8,7 +8,7 @@ from golem.core.log import Log, default_log from golem.core.optimisers.opt_history_objects.opt_history import OptHistory from golem.core.tuning.simultaneous import SimultaneousTuner -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence from golem.visualisation.opt_viz_extra import visualise_pareto from fedot.api.api_utils.api_composer import ApiComposer @@ -163,6 +163,12 @@ class Fedot: Default optimizer is :class:`~golem.core.optimisers.genetic.gp_optimizer.EvoGraphOptimizer`. See the `example \ `_ + + NB! 'adaptive_mutation_type' and 'context_agent_type' can be specified with 'composer_tuner_params'. + Both these params are a part of experimental functionality for adaptive evolution. + Adaptive evolution uses multi-armed bandits to choose most promising mutation at each iteration + of evolution cycle. + The full range of context agents can be obtained in GOLEM: golem.core.optimizers.adaptive.context_agents """ def __init__(self, diff --git a/fedot/core/caching/base_cache.py b/fedot/core/caching/base_cache.py index 47798a5ebc..c47440ba40 100644 --- a/fedot/core/caching/base_cache.py +++ b/fedot/core/caching/base_cache.py @@ -1,7 +1,7 @@ from typing import Union from golem.core.log import default_log -from golem.core.utilities.singleton_meta import SingletonMeta +from golem.utilities.singleton_meta import SingletonMeta from fedot.core.caching.pipelines_cache_db import OperationsCacheDB from fedot.core.caching.preprocessing_cache_db import PreprocessingCacheDB diff --git a/fedot/core/caching/pipelines_cache.py b/fedot/core/caching/pipelines_cache.py index e0ba7f4caf..23a9acc6e8 100644 --- a/fedot/core/caching/pipelines_cache.py +++ b/fedot/core/caching/pipelines_cache.py @@ -1,7 +1,7 @@ import sqlite3 from typing import List, Optional, TYPE_CHECKING, Union -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence from fedot.core.caching.base_cache import BaseCache from fedot.core.caching.pipelines_cache_db import OperationsCacheDB @@ -35,8 +35,12 @@ def save_nodes(self, nodes: Union[PipelineNode, List[PipelineNode]], fold_id: Op ] self._db.add_operations(mapped) except Exception as ex: - unexpected_exc = not (isinstance(ex, sqlite3.DatabaseError) and 'disk is full' in str(ex)) - self.log.warning(f'Nodes can not be saved: {ex}. Continue', exc=ex, raise_if_test=unexpected_exc) + exc_is_expected = (isinstance(ex, sqlite3.DatabaseError) and 'disk is full' in str(ex)) + msg = 'Nodes can not be saved' + if exc_is_expected: + self.log.warning(msg, exc_info=ex) + else: + self.log.log_or_raise('warning', ValueError(msg)) def save_pipeline(self, pipeline: 'Pipeline', fold_id: Optional[int] = None): """ @@ -61,8 +65,8 @@ def try_load_nodes(self, nodes: Union[PipelineNode, List[PipelineNode]], fold_id nodes_lst[idx].fitted_operation = cached_op else: nodes_lst[idx].fitted_operation = None - except Exception as ex: - self.log.warning(f'Cache can not be loaded: {ex}. Continue.', exc=ex, raise_if_test=True) + except Exception: + self.log.log_or_raise('info', ValueError('Cache can not be loaded.')) def try_load_into_pipeline(self, pipeline: 'Pipeline', fold_id: Optional[int] = None): """ diff --git a/fedot/core/caching/preprocessing_cache.py b/fedot/core/caching/preprocessing_cache.py index 4b0de47870..25299bae45 100644 --- a/fedot/core/caching/preprocessing_cache.py +++ b/fedot/core/caching/preprocessing_cache.py @@ -30,8 +30,8 @@ def try_load_preprocessor(self, pipeline: 'Pipeline', fold_id: Union[int, None]) processors = self._db.get_preprocessor(structural_id) if processors: pipeline.encoder, pipeline.imputer = processors - except Exception as ex: - self.log.warning(f'Preprocessor search error: {ex}', raise_if_test=True) + except Exception: + self.log.log_or_raise('warning', ValueError('Preprocessor search error')) def add_preprocessor(self, pipeline: 'Pipeline', fold_id: Optional[Union[int, None]] = None): """ diff --git a/fedot/core/composer/composer_builder.py b/fedot/core/composer/composer_builder.py index f6f725c9b4..a3eae7e1cf 100644 --- a/fedot/core/composer/composer_builder.py +++ b/fedot/core/composer/composer_builder.py @@ -8,7 +8,7 @@ from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters from golem.core.optimisers.initial_graphs_generator import InitialPopulationGenerator, GenerationFunction from golem.core.optimisers.optimizer import GraphOptimizer, AlgorithmParameters, GraphGenerationParams -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence from fedot.core.caching.pipelines_cache import OperationsCache from fedot.core.caching.preprocessing_cache import PreprocessingCache diff --git a/fedot/core/composer/metrics.py b/fedot/core/composer/metrics.py index 22f5196daa..d2cf813a4c 100644 --- a/fedot/core/composer/metrics.py +++ b/fedot/core/composer/metrics.py @@ -89,7 +89,7 @@ def get_value(cls, pipeline: 'Pipeline', reference_data: InputData, save_path=Path(save_path, 'forecast.png')) except Exception as ex: - pipeline.log.info(f'Metric can not be evaluated because of: {ex}', raise_if_test=True) + pipeline.log.log_or_raise('info', ValueError('Metric can not be evaluated')) return metric diff --git a/fedot/core/data/merge/data_merger.py b/fedot/core/data/merge/data_merger.py index 8d7763a52c..35146019f4 100644 --- a/fedot/core/data/merge/data_merger.py +++ b/fedot/core/data/merge/data_merger.py @@ -1,7 +1,7 @@ from typing import List, Iterable, Union from golem.core.log import default_log -from golem.core.utilities.data_structures import are_same_length +from golem.utilities.data_structures import are_same_length from fedot.core.data.array_utilities import * from fedot.core.data.data import OutputData, InputData diff --git a/fedot/core/optimisers/objective/data_objective_eval.py b/fedot/core/optimisers/objective/data_objective_eval.py index e4c4a08c85..6e0268ef58 100644 --- a/fedot/core/optimisers/objective/data_objective_eval.py +++ b/fedot/core/optimisers/objective/data_objective_eval.py @@ -80,8 +80,8 @@ def evaluate(self, graph: Pipeline) -> Fitness: if evaluated_fitness.valid: folds_metrics.append(evaluated_fitness.values) else: - self._log.warning(f'Invalid fitness after objective evaluation. ' - f'Skipping the graph: {graph_id}', raise_if_test=True) + self._log.log_or_raise('warning', ValueError(f'Invalid fitness after objective evaluation. ' + f'Skipping the graph: {graph_id}')) if self._do_unfit: graph.unfit() if folds_metrics: diff --git a/fedot/core/optimisers/objective/metrics_objective.py b/fedot/core/optimisers/objective/metrics_objective.py index a500792b88..5d893e2a7b 100644 --- a/fedot/core/optimisers/objective/metrics_objective.py +++ b/fedot/core/optimisers/objective/metrics_objective.py @@ -1,7 +1,7 @@ from typing import Union, Iterable, Callable from golem.core.optimisers.objective import Objective -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence from fedot.core.repository.quality_metrics_repository import MetricType, MetricsRepository, ComplexityMetricsEnum diff --git a/fedot/core/pipelines/pipeline.py b/fedot/core/pipelines/pipeline.py index 489cbc8430..c5a727108c 100644 --- a/fedot/core/pipelines/pipeline.py +++ b/fedot/core/pipelines/pipeline.py @@ -12,7 +12,7 @@ from golem.core.log import default_log from golem.core.optimisers.timer import Timer from golem.core.paths import copy_doc -from golem.core.utilities.serializable import Serializable +from golem.utilities.serializable import Serializable from golem.visualisation.graph_viz import NodeColorType from fedot.core.caching.pipelines_cache import OperationsCache diff --git a/fedot/core/pipelines/pipeline_node_factory.py b/fedot/core/pipelines/pipeline_node_factory.py index 0e99694ee1..254fac07c5 100644 --- a/fedot/core/pipelines/pipeline_node_factory.py +++ b/fedot/core/pipelines/pipeline_node_factory.py @@ -17,6 +17,15 @@ def __init__(self, requirements: PipelineComposerRequirements, self.advisor = advisor or PipelineChangeAdvisor() self.graph_model_repository = graph_model_repository or self._init_default_graph_model_repo() + def get_all_available_operations(self) -> List[str]: + """ Returns all available models and data operations. """ + available_operations = [] + if self.advisor.models: + available_operations += self.advisor.models + if self.advisor.data_operations: + available_operations += self.advisor.data_operations + return available_operations + def _init_default_graph_model_repo(self): """ Initialize default graph model repository with operations from composer requirements """ repo = PipelineOperationRepository(operations_by_keys={'primary': self.requirements.primary, diff --git a/fedot/core/pipelines/tuning/tuner_builder.py b/fedot/core/pipelines/tuning/tuner_builder.py index 698de90a95..c061768436 100644 --- a/fedot/core/pipelines/tuning/tuner_builder.py +++ b/fedot/core/pipelines/tuning/tuner_builder.py @@ -4,7 +4,7 @@ from golem.core.tuning.optuna_tuner import OptunaTuner from golem.core.tuning.simultaneous import SimultaneousTuner from golem.core.tuning.tuner_interface import BaseTuner -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence from fedot.core.constants import DEFAULT_TUNING_ITERATIONS_NUMBER from fedot.core.data.data import InputData diff --git a/fedot/core/repository/dataset_types.py b/fedot/core/repository/dataset_types.py index 09ed285759..3faca1ee06 100644 --- a/fedot/core/repository/dataset_types.py +++ b/fedot/core/repository/dataset_types.py @@ -1,4 +1,4 @@ -from golem.core.utilities.data_structures import ComparableEnum as Enum +from golem.utilities.data_structures import ComparableEnum as Enum class DataTypesEnum(Enum): diff --git a/fedot/core/repository/operation_types_repository.py b/fedot/core/repository/operation_types_repository.py index 741bd2a505..2228c32563 100644 --- a/fedot/core/repository/operation_types_repository.py +++ b/fedot/core/repository/operation_types_repository.py @@ -6,7 +6,7 @@ import numpy as np from golem.core.log import default_log -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence from fedot.core.constants import BEST_QUALITY_PRESET_NAME, AUTO_PRESET_NAME from fedot.core.repository.dataset_types import DataTypesEnum diff --git a/fedot/core/repository/quality_metrics_repository.py b/fedot/core/repository/quality_metrics_repository.py index 6dcdccf4d6..537d87bd20 100644 --- a/fedot/core/repository/quality_metrics_repository.py +++ b/fedot/core/repository/quality_metrics_repository.py @@ -2,7 +2,7 @@ from typing import Callable, Union, TypeVar from golem.core.dag.graph import Graph -from golem.core.utilities.data_structures import ComparableEnum as Enum +from golem.utilities.data_structures import ComparableEnum as Enum from fedot.core.composer.metrics import (ComputationTime, Accuracy, F1, Logloss, MAE, MAPE, SMAPE, MSE, MSLE, Metric, NodeNum, Precision, R2, diff --git a/fedot/core/repository/tasks.py b/fedot/core/repository/tasks.py index 0e3c01e84e..11fed93fb5 100644 --- a/fedot/core/repository/tasks.py +++ b/fedot/core/repository/tasks.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import Any, List, Optional -from golem.core.utilities.data_structures import ComparableEnum as Enum +from golem.utilities.data_structures import ComparableEnum as Enum @dataclass diff --git a/fedot/core/utils.py b/fedot/core/utils.py index 40ad8bb39a..044e5b2446 100644 --- a/fedot/core/utils.py +++ b/fedot/core/utils.py @@ -7,7 +7,7 @@ import numpy as np import pandas as pd -from golem.core.utilities.random import RandomStateHandler +from golem.utilities.random import RandomStateHandler from sklearn.model_selection import train_test_split DEFAULT_PARAMS_STUB = 'default_params' diff --git a/fedot/remote/infrastructure/clients/client.py b/fedot/remote/infrastructure/clients/client.py index 961b3cc766..ee4b0763d2 100644 --- a/fedot/remote/infrastructure/clients/client.py +++ b/fedot/remote/infrastructure/clients/client.py @@ -3,7 +3,7 @@ from typing import Optional, Type, TypeVar from golem.core.log import default_log -from golem.core.utilities.serializable import Serializable +from golem.utilities.serializable import Serializable from fedot.core.utils import default_fedot_data_dir diff --git a/fedot/remote/remote_evaluator.py b/fedot/remote/remote_evaluator.py index f27869879b..516aeb2154 100644 --- a/fedot/remote/remote_evaluator.py +++ b/fedot/remote/remote_evaluator.py @@ -5,7 +5,7 @@ import numpy as np from golem.core.log import default_log from golem.core.optimisers.genetic.evaluation import DelegateEvaluator -from golem.core.utilities.serializable import Serializable +from golem.utilities.serializable import Serializable from fedot.core.data.data import InputData from fedot.remote.infrastructure.clients.client import Client diff --git a/fedot/remote/run_pipeline.py b/fedot/remote/run_pipeline.py index 30a3e40158..6abb91d190 100644 --- a/fedot/remote/run_pipeline.py +++ b/fedot/remote/run_pipeline.py @@ -4,7 +4,7 @@ from typing import Union from golem.core.log import default_log -from golem.core.utilities.random import RandomStateHandler +from golem.utilities.random import RandomStateHandler from fedot.core.data.data import InputData from fedot.core.data.multi_modal import MultiModalData diff --git a/fedot/utilities/golem_imports_transition.py b/fedot/utilities/golem_imports_transition.py index d96828a0d4..059b869acf 100644 --- a/fedot/utilities/golem_imports_transition.py +++ b/fedot/utilities/golem_imports_transition.py @@ -21,7 +21,7 @@ 'fedot.core.adapter': 'golem.core.adapter', 'fedot.core.dag': 'golem.core.dag', 'fedot.core.optimisers': 'golem.core.optimisers', - 'fedot.core.utilities': 'golem.core.utilities', + 'fedot.core.utilities': 'golem.utilities', 'fedot.core.serializers': 'golem.serializers', 'fedot.core.visualisation': 'golem.visualisation', diff --git a/requirements.txt b/requirements.txt index 7dc30e2982..c23f18a05d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # Base framework -thegolem==0.3.3 +thegolem==0.4.0 # Data numpy>=1.16.0, !=1.24.0 diff --git a/test/integration/pipelines/tuning/test_pipeline_tuning.py b/test/integration/pipelines/tuning/test_pipeline_tuning.py index 34cdb062ab..b18a093708 100644 --- a/test/integration/pipelines/tuning/test_pipeline_tuning.py +++ b/test/integration/pipelines/tuning/test_pipeline_tuning.py @@ -7,7 +7,7 @@ from golem.core.tuning.optuna_tuner import OptunaTuner from golem.core.tuning.sequential import SequentialTuner from golem.core.tuning.simultaneous import SimultaneousTuner -from golem.core.utilities.data_structures import ensure_wrapped_in_sequence +from golem.utilities.data_structures import ensure_wrapped_in_sequence from hyperopt import hp from hyperopt.pyll.stochastic import sample as hp_sample diff --git a/test/unit/pipelines/prediction_intervals/data/pred_ints_model_test.pickle b/test/unit/pipelines/prediction_intervals/data/pred_ints_model_test.pickle index f1c88aae83..6ad6b5a8f4 100644 Binary files a/test/unit/pipelines/prediction_intervals/data/pred_ints_model_test.pickle and b/test/unit/pipelines/prediction_intervals/data/pred_ints_model_test.pickle differ diff --git a/test/unit/remote/test_remote_custom.py b/test/unit/remote/test_remote_custom.py index cb48a602f8..3378938ce6 100644 --- a/test/unit/remote/test_remote_custom.py +++ b/test/unit/remote/test_remote_custom.py @@ -4,7 +4,7 @@ from typing import Tuple, Union, Optional, Sequence import pytest -from golem.core.utilities.serializable import Serializable +from golem.utilities.serializable import Serializable from fedot.remote.infrastructure.clients.client import Client from fedot.remote.remote_evaluator import RemoteEvaluator, RemoteTaskParams