From 2f3508dea965f69d69e0d7d9c5db8731be968a64 Mon Sep 17 00:00:00 2001 From: Venkat Bala Date: Mon, 13 Mar 2023 15:43:11 +0000 Subject: [PATCH 01/46] add base cloud resource manager class --- covalent/cloud_resource_manager/__init__.py | 21 ++++++++ .../cloud_resource_manager.py | 48 +++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 covalent/cloud_resource_manager/__init__.py create mode 100644 covalent/cloud_resource_manager/cloud_resource_manager.py diff --git a/covalent/cloud_resource_manager/__init__.py b/covalent/cloud_resource_manager/__init__.py new file mode 100644 index 000000000..6c2fc494b --- /dev/null +++ b/covalent/cloud_resource_manager/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2023 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the GNU Affero General Public License 3.0 (the "License"). +# A copy of the License may be obtained with this software package or at +# +# https://www.gnu.org/licenses/agpl-3.0.en.html +# +# Use of this file is prohibited except in compliance with the License. Any +# modifications or derivative works of this file must retain this copyright +# notice, and modified files must contain a notice indicating that they have +# been altered from the originals. +# +# Covalent is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. +# +# Relief from the License may be granted by purchasing a commercial license. + +from .cloud_resource_manager import CloudResourceManager diff --git a/covalent/cloud_resource_manager/cloud_resource_manager.py b/covalent/cloud_resource_manager/cloud_resource_manager.py new file mode 100644 index 000000000..f2bc06703 --- /dev/null +++ b/covalent/cloud_resource_manager/cloud_resource_manager.py @@ -0,0 +1,48 @@ +# Copyright 2023 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the GNU Affero General Public License 3.0 (the "License"). +# A copy of the License may be obtained with this software package or at +# +# https://www.gnu.org/licenses/agpl-3.0.en.html +# +# Use of this file is prohibited except in compliance with the License. Any +# modifications or derivative works of this file must retain this copyright +# notice, and modified files must contain a notice indicating that they have +# been altered from the originals. +# +# Covalent is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. +# +# Relief from the License may be granted by purchasing a commercial license. + + +class CloudResourceManager: + """ + Base cloud resource manager class + """ + + def __init__(self, executor_name: str, *args, **kwargs): + self.executor_name = executor_name + self._args = args + self._kwargs = kwargs + + async def up(self): + """ + Setup executor resources + """ + pass + + async def down(self): + """ + Teardown executor resources + """ + pass + + async def status(self): + """ + Return executor resource deployment status + """ + pass From fd6b9e9b8b77e3084c885e4d30de43a336029aaa Mon Sep 17 00:00:00 2001 From: Venkat Bala Date: Mon, 13 Mar 2023 15:44:33 +0000 Subject: [PATCH 02/46] update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d030bb21c..536199fe0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +### Added + +- Adding the base `CloudResourceManager` class + ### Docs - Updated How-to documents. From 72c21a4a41894239e5c555e9b8354bad44a11964 Mon Sep 17 00:00:00 2001 From: Venkat Bala Date: Mon, 13 Mar 2023 18:11:29 +0000 Subject: [PATCH 03/46] implement initial skeleton fro the executor resource deployment --- covalent/_shared_files/exceptions.py | 4 + .../cloud_resource_manager.py | 73 ++++++++++++++++- covalent_dispatcher/_cli/cli.py | 3 +- covalent_dispatcher/_cli/groups/__init__.py | 1 + covalent_dispatcher/_cli/groups/deploy.py | 78 +++++++++++++++++++ 5 files changed, 154 insertions(+), 5 deletions(-) create mode 100644 covalent_dispatcher/_cli/groups/deploy.py diff --git a/covalent/_shared_files/exceptions.py b/covalent/_shared_files/exceptions.py index 3b57c52cb..c7a097975 100644 --- a/covalent/_shared_files/exceptions.py +++ b/covalent/_shared_files/exceptions.py @@ -29,3 +29,7 @@ class TaskRuntimeError(Exception): class TaskCancelledError(Exception): pass + + +class CommandNotFoundError(Exception): + pass diff --git a/covalent/cloud_resource_manager/cloud_resource_manager.py b/covalent/cloud_resource_manager/cloud_resource_manager.py index f2bc06703..f18619193 100644 --- a/covalent/cloud_resource_manager/cloud_resource_manager.py +++ b/covalent/cloud_resource_manager/cloud_resource_manager.py @@ -19,21 +19,86 @@ # Relief from the License may be granted by purchasing a commercial license. +import asyncio +import os +import shutil +from typing import Dict, Optional + +from covalent._shared_files.exceptions import CommandNotFoundError +from covalent._shared_files.logger import app_log + + +class ChangeDir(object): + """ + Change directory context manager + """ + + def __init__(self, dir_to_change: str): + self._pwd = os.getcwd() + self._dir = dir_to_change + + def __enter__(self): + os.chdir(self._dir) + + def __exit__(self, type, value, traceback): + os.chdir(self._pwd) + + class CloudResourceManager: """ Base cloud resource manager class """ - def __init__(self, executor_name: str, *args, **kwargs): + def __init__( + self, + executor_name: str, + executor_module_path: str, + options: Optional[Dict[str, str]] = None, + ): self.executor_name = executor_name - self._args = args - self._kwargs = kwargs + self.executor_module_path = executor_module_path + self.executor_options = options async def up(self): """ Setup executor resources """ - pass + terraform = shutil.which("terraform") + if not terraform: + raise CommandNotFoundError("Terraform not found on system") + + executor_infra_assets_path = os.path.join(self.executor_module_path, "assets/infra") + with ChangeDir(executor_infra_assets_path): + proc = await asyncio.create_subprocess_exec( + " ".join([terraform, "init"]), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + stdout, stderr = await proc.communicate() + if stdout: + app_log.debug(f"{self.executor_name} stdout: {stdout}") + if stderr: + app_log.debug(f"{self.executor_name} stderr: {stdout}") + + # Setup TF_VAR environment variables by appending new variables to existing os.environ + tf_var_env_dict = os.environ.copy() + if self.executor_options: + for key, value in self.executor_options.items(): + tf_var_env_dict[f"TF_VAR_{key}"] = value + + proc = await asyncio.create_subprocess_shell( + " ".join([terraform, "apply", "--auto-approve"]), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=tf_var_env_dict, + ) + + stdout, stderr = await proc.communicate() + if stdout: + app_log.debug(f"{self.executor_name} stdout: {stdout}") + if stderr: + app_log.debug(f"{self.executor_name} stderr: {stdout}") async def down(self): """ diff --git a/covalent_dispatcher/_cli/cli.py b/covalent_dispatcher/_cli/cli.py index ef3dc4fab..786e02015 100644 --- a/covalent_dispatcher/_cli/cli.py +++ b/covalent_dispatcher/_cli/cli.py @@ -26,7 +26,7 @@ import click -from .groups import db +from .groups import db, deploy from .service import ( cluster, config, @@ -70,6 +70,7 @@ def cli(ctx: click.Context, version: bool) -> None: cli.add_command(db) cli.add_command(config) cli.add_command(migrate_legacy_result_object) +cli.add_command(deploy) if __name__ == "__main__": cli() diff --git a/covalent_dispatcher/_cli/groups/__init__.py b/covalent_dispatcher/_cli/groups/__init__.py index 031b251b3..b46039824 100644 --- a/covalent_dispatcher/_cli/groups/__init__.py +++ b/covalent_dispatcher/_cli/groups/__init__.py @@ -18,3 +18,4 @@ # # Relief from the License may be granted by purchasing a commercial license. from .db import db +from .deploy import deploy diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py new file mode 100644 index 000000000..98c798dbb --- /dev/null +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -0,0 +1,78 @@ +# Copyright 2023 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the GNU Affero General Public License 3.0 (the "License"). +# A copy of the License may be obtained with this software package or at +# +# https://www.gnu.org/licenses/agpl-3.0.en.html +# +# Use of this file is prohibited except in compliance with the License. Any +# modifications or derivative works of this file must retain this copyright +# notice, and modified files must contain a notice indicating that they have +# been altered from the originals. +# +# Covalent is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. +# +# Relief from the License may be granted by purchasing a commercial license. + + +import asyncio +from pathlib import Path + +import click + +from covalent.cloud_resource_manager.cloud_resource_manager import CloudResourceManager +from covalent.executor import _executor_manager + + +@click.group(invoke_without_command=True) +@click.argument("executor_name", nargs=1) +@click.pass_context +def deploy(ctx: click.Context, executor_name: str): + """ + Load the executor plugin installation path based on the executor name provided + """ + executor_module_path = Path( + __import__(_executor_manager.executor_plugins_map[executor_name].__module__).__path__[0] + ) + ctx.obj = {"executor_name": executor_name, "executor_module_path": executor_module_path} + + +@click.command +@click.argument("options", nargs=-1) +@click.pass_obj +def up(executor_metadata: dict, options): + executor_name = executor_metadata["executor_name"] + executor_module_path = executor_metadata["executor_module_path"] + cmd_options = dict(opt.split("=") for opt in options) + + # Create the cloud resource manager and deploy the resources + crm = CloudResourceManager(executor_name, executor_module_path, cmd_options) + click.echo(asyncio.run(crm.up())) + + +@click.command +@click.argument("options", nargs=-1) +@click.pass_obj +def down(executor_metadata: dict, options): + executor_name = executor_metadata["executor_name"] + executor_module_path = executor_metadata["executor_module_path"] + cmd_options = dict(opt.split("=") for opt in options) + + # Create the cloud resource manager and teardown the resources + crm = CloudResourceManager(executor_name, executor_module_path, cmd_options) + click.echo(asyncio.run(crm.down())) + + +@click.command +@click.pass_obj +def status(executor_module_path: str): + click.echo(executor_module_path) + + +deploy.add_command(up) +deploy.add_command(down) +deploy.add_command(status) From 11fc8a8c1d67f20f8b67231c3c7d5c54c537b75d Mon Sep 17 00:00:00 2001 From: Venkat Bala Date: Mon, 13 Mar 2023 18:17:22 +0000 Subject: [PATCH 04/46] fix status command and docstring for deploy --- covalent_dispatcher/_cli/groups/deploy.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index 98c798dbb..99a103b0c 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -33,7 +33,8 @@ @click.pass_context def deploy(ctx: click.Context, executor_name: str): """ - Load the executor plugin installation path based on the executor name provided + Load the executor plugin installation path based on the executor name provided. + Will raise KeyError if the `executor_name` plugin is not installed """ executor_module_path = Path( __import__(_executor_manager.executor_plugins_map[executor_name].__module__).__path__[0] @@ -69,7 +70,9 @@ def down(executor_metadata: dict, options): @click.command @click.pass_obj -def status(executor_module_path: str): +def status(executor_metadata: dict): + executor_name = executor_metadata["executor_name"] + executor_module_path = executor_metadata["executor_module_path"] click.echo(executor_module_path) From ecf8cf99d3f70c430a30d8b0aca0da8276d14603 Mon Sep 17 00:00:00 2001 From: Venkat Bala Date: Fri, 17 Mar 2023 12:59:43 +0000 Subject: [PATCH 05/46] add base cloud resource manager class --- .../cloud_resource_manager.py | 182 +++++++++++++----- 1 file changed, 139 insertions(+), 43 deletions(-) diff --git a/covalent/cloud_resource_manager/cloud_resource_manager.py b/covalent/cloud_resource_manager/cloud_resource_manager.py index f18619193..ba4353f01 100644 --- a/covalent/cloud_resource_manager/cloud_resource_manager.py +++ b/covalent/cloud_resource_manager/cloud_resource_manager.py @@ -19,29 +19,47 @@ # Relief from the License may be granted by purchasing a commercial license. -import asyncio import os import shutil +import subprocess +from configparser import ConfigParser +from enum import Enum from typing import Dict, Optional +from covalent._shared_files.config import set_config from covalent._shared_files.exceptions import CommandNotFoundError -from covalent._shared_files.logger import app_log -class ChangeDir(object): +class DeployStatus(Enum): """ - Change directory context manager + Status of the terraform deployment """ - def __init__(self, dir_to_change: str): - self._pwd = os.getcwd() - self._dir = dir_to_change + OK = 0 + DESTROYED = 1 + ERROR = 2 - def __enter__(self): - os.chdir(self._dir) - def __exit__(self, type, value, traceback): - os.chdir(self._pwd) +def is_int(value: str) -> bool: + """ + Check if the string passed is int convertible + """ + try: + int(value) + return True + except ValueError: + return False + + +def is_float(value: str) -> bool: + """ + Check if string is convertible to float + """ + try: + float(value) + return True + except ValueError: + return False class CloudResourceManager: @@ -56,10 +74,71 @@ def __init__( options: Optional[Dict[str, str]] = None, ): self.executor_name = executor_name - self.executor_module_path = executor_module_path + self.executor_tf_path = os.path.join(executor_module_path, "assets/infra") self.executor_options = options - async def up(self): + @staticmethod + def _print_stdout(process: subprocess.Popen) -> Optional[int]: + """ + Print the stdout from the subprocess to console + + Arg(s) + process: Python subprocess whose stdout is to be printed to screen + + Return(s) + None + """ + while True: + if process.poll() is not None: + break + proc_stdout = process.stdout.readline() + if not proc_stdout: + break + else: + print(proc_stdout.strip().decode("utf-8")) + + return process.poll() + + def _run_in_subprocess( + self, cmd: str, workdir: str, env_vars: Optional[Dict[str, str]] = None + ) -> Optional[int]: + """ + Run the `cmd` in a subprocess shell with the env_vars set in the process's new environment + + Arg(s) + cmd: Command to execute in the subprocess + workdir: Working directory of the subprocess + env_vars: Dictionary of environment variables to set in the processes execution environment + + Return(s) + Exit code of the process + """ + proc = subprocess.Popen( + args=cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=workdir, + shell=True, + bufsize=-1, + env=env_vars, + ) + retcode = self._print_stdout(proc) + return retcode + + def _update_config(self, tf_executor_config_file: str) -> None: + """ + Update covalent configuration with the executor values + """ + executor_config = ConfigParser() + executor_config.read(tf_executor_config_file) + for key in executor_config[self.executor_name]: + value = executor_config[self.executor_name][key] + converted_value = ( + int(value) if is_int(value) else float(value) if is_float(value) else value + ) + set_config({f"executors.{self.executor_name}.{key}": converted_value}) + + def up(self, dry_run: bool = True): """ Setup executor resources """ @@ -67,46 +146,63 @@ async def up(self): if not terraform: raise CommandNotFoundError("Terraform not found on system") - executor_infra_assets_path = os.path.join(self.executor_module_path, "assets/infra") - with ChangeDir(executor_infra_assets_path): - proc = await asyncio.create_subprocess_exec( - " ".join([terraform, "init"]), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) + tfvars_file = os.path.join(self.executor_tf_path, "terraform.tfvars") + tf_executor_config_file = os.path.join(self.executor_tf_path, f"{self.executor_name}.conf") - stdout, stderr = await proc.communicate() - if stdout: - app_log.debug(f"{self.executor_name} stdout: {stdout}") - if stderr: - app_log.debug(f"{self.executor_name} stderr: {stdout}") + tf_init = " ".join([terraform, "init"]) + tf_plan = " ".join([terraform, "plan", "-out", "tf.plan"]) + tf_apply = " ".join([terraform, "apply", "tf.plan"]) - # Setup TF_VAR environment variables by appending new variables to existing os.environ - tf_var_env_dict = os.environ.copy() - if self.executor_options: - for key, value in self.executor_options.items(): - tf_var_env_dict[f"TF_VAR_{key}"] = value + retcode = self._run_in_subprocess(cmd=tf_init, workdir=self.executor_tf_path) + if retcode != 0: + raise subprocess.CalledProcessError(returncode=retcode, cmd=tf_init) - proc = await asyncio.create_subprocess_shell( - " ".join([terraform, "apply", "--auto-approve"]), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - env=tf_var_env_dict, + # Setup terraform infra variables as passed by the user + tf_vars_env_dict = os.environ.copy() + if self.executor_options: + with open(tfvars_file, "w") as f: + for key, value in self.executor_options.items(): + tf_vars_env_dict[f"TF_VAR_{key}"] = value + f.write(f'{key}="{value}"\n') + + # Plan the infrastructure + retcode = self._run_in_subprocess( + cmd=tf_plan, workdir=self.executor_tf_path, env_vars=tf_vars_env_dict + ) + if retcode != 0: + raise subprocess.CalledProcessError(returncode=retcode, cmd=tf_plan) + + # Create infrastructure as per the plan + if not dry_run: + retcode = self._run_in_subprocess( + cmd=tf_apply, workdir=self.executor_tf_path, env_vars=tf_vars_env_dict ) + if retcode != 0: + raise subprocess.CalledProcessError(returncode=retcode, cmd=tf_apply) - stdout, stderr = await proc.communicate() - if stdout: - app_log.debug(f"{self.executor_name} stdout: {stdout}") - if stderr: - app_log.debug(f"{self.executor_name} stderr: {stdout}") + # Update covalent executor config based on Terraform output + self._update_config(tf_executor_config_file) - async def down(self): + def down(self, dry_run: bool = True): """ Teardown executor resources """ - pass + terraform = shutil.which("terraform") + if not terraform: + raise CommandNotFoundError("Terraform not found on system") + + tfvars_file = os.path.join(self.executor_tf_path, "terraform.tfvars") + + tf_destroy = " ".join([terraform, "destroy", "-auto-approve"]) + if not dry_run: + retcode = self._run_in_subprocess(cmd=tf_destroy, workdir=self.executor_tf_path) + if retcode != 0: + raise subprocess.CalledProcessError(cmd=tf_destroy, returncode=retcode) + + if os.path.exists(tfvars_file): + os.remove(tfvars_file) - async def status(self): + def status(self): """ Return executor resource deployment status """ From 97fddbf1f7fae6ff45eddfddc8e94e90953f2037 Mon Sep 17 00:00:00 2001 From: kessler-frost Date: Mon, 1 May 2023 11:03:11 -0400 Subject: [PATCH 06/46] porting changes from old crm implementation branch --- CHANGELOG.md | 5 + covalent/_shared_files/exceptions.py | 4 + covalent/cloud_resource_manager/__init__.py | 21 ++ covalent/cloud_resource_manager/core.py | 202 ++++++++++++++++++++ 4 files changed, 232 insertions(+) create mode 100644 covalent/cloud_resource_manager/__init__.py create mode 100644 covalent/cloud_resource_manager/core.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d6cf60d0..72b5f52ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +### Added + +- Adding the base `CloudResourceManager` class + + ### Fixed - Raise error on dispatching a non-lattice diff --git a/covalent/_shared_files/exceptions.py b/covalent/_shared_files/exceptions.py index 3b57c52cb..c7a097975 100644 --- a/covalent/_shared_files/exceptions.py +++ b/covalent/_shared_files/exceptions.py @@ -29,3 +29,7 @@ class TaskRuntimeError(Exception): class TaskCancelledError(Exception): pass + + +class CommandNotFoundError(Exception): + pass diff --git a/covalent/cloud_resource_manager/__init__.py b/covalent/cloud_resource_manager/__init__.py new file mode 100644 index 000000000..d1f2c47e0 --- /dev/null +++ b/covalent/cloud_resource_manager/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2023 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the GNU Affero General Public License 3.0 (the "License"). +# A copy of the License may be obtained with this software package or at +# +# https://www.gnu.org/licenses/agpl-3.0.en.html +# +# Use of this file is prohibited except in compliance with the License. Any +# modifications or derivative works of this file must retain this copyright +# notice, and modified files must contain a notice indicating that they have +# been altered from the originals. +# +# Covalent is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. +# +# Relief from the License may be granted by purchasing a commercial license. + +from .core import CloudResourceManager diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py new file mode 100644 index 000000000..848a31966 --- /dev/null +++ b/covalent/cloud_resource_manager/core.py @@ -0,0 +1,202 @@ +# Copyright 2023 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the GNU Affero General Public License 3.0 (the "License"). +# A copy of the License may be obtained with this software package or at +# +# https://www.gnu.org/licenses/agpl-3.0.en.html +# +# Use of this file is prohibited except in compliance with the License. Any +# modifications or derivative works of this file must retain this copyright +# notice, and modified files must contain a notice indicating that they have +# been altered from the originals. +# +# Covalent is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. +# +# Relief from the License may be granted by purchasing a commercial license. + + +import os +import shutil +import subprocess +from configparser import ConfigParser +from enum import Enum +from typing import Dict, Optional + +from covalent._shared_files.config import set_config +from covalent._shared_files.exceptions import CommandNotFoundError + + +class DeployStatus(Enum): + """ + Status of the terraform deployment + """ + + OK = 0 + DESTROYED = 1 + ERROR = 2 + + +def is_int(value: str) -> bool: + """ + Check if the string passed is int convertible + """ + try: + int(value) + return True + except ValueError: + return False + + +def is_float(value: str) -> bool: + """ + Check if string is convertible to float + """ + try: + float(value) + return True + except ValueError: + return False + + +class CloudResourceManager: + """ + Base cloud resource manager class + """ + + def __init__( + self, + executor_name: str, + executor_module_path: str, + options: Optional[Dict[str, str]] = None, + ): + self.executor_name = executor_name + self.executor_tf_path = os.path.join(executor_module_path, "assets/infra") + self.executor_options = options + + @staticmethod + def _print_stdout(process: subprocess.Popen) -> Optional[int]: + """ + Print the stdout from the subprocess to console + Arg(s) + process: Python subprocess whose stdout is to be printed to screen + Return(s) + None + """ + while process.poll() is None: + if proc_stdout := process.stdout.readline(): + print(proc_stdout.strip().decode("utf-8")) + + else: + break + return process.poll() + + def _run_in_subprocess( + self, cmd: str, workdir: str, env_vars: Optional[Dict[str, str]] = None + ) -> Optional[int]: + """ + Run the `cmd` in a subprocess shell with the env_vars set in the process's new environment + Arg(s) + cmd: Command to execute in the subprocess + workdir: Working directory of the subprocess + env_vars: Dictionary of environment variables to set in the processes execution environment + Return(s) + Exit code of the process + """ + proc = subprocess.Popen( + args=cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=workdir, + shell=True, + bufsize=-1, + env=env_vars, + ) + retcode = self._print_stdout(proc) + return retcode + + def _update_config(self, tf_executor_config_file: str) -> None: + """ + Update covalent configuration with the executor values + """ + executor_config = ConfigParser() + executor_config.read(tf_executor_config_file) + for key in executor_config[self.executor_name]: + value = executor_config[self.executor_name][key] + converted_value = ( + int(value) if is_int(value) else float(value) if is_float(value) else value + ) + set_config({f"executors.{self.executor_name}.{key}": converted_value}) + + def up(self, dry_run: bool = True): + """ + Setup executor resources + """ + terraform = shutil.which("terraform") + if not terraform: + raise CommandNotFoundError("Terraform not found on system") + + tfvars_file = os.path.join(self.executor_tf_path, "terraform.tfvars") + tf_executor_config_file = os.path.join(self.executor_tf_path, f"{self.executor_name}.conf") + + tf_init = " ".join([terraform, "init"]) + tf_plan = " ".join([terraform, "plan", "-out", "tf.plan"]) + tf_apply = " ".join([terraform, "apply", "tf.plan"]) + + retcode = self._run_in_subprocess(cmd=tf_init, workdir=self.executor_tf_path) + if retcode != 0: + raise subprocess.CalledProcessError(returncode=retcode, cmd=tf_init) + + # Setup terraform infra variables as passed by the user + tf_vars_env_dict = os.environ.copy() + if self.executor_options: + with open(tfvars_file, "w") as f: + for key, value in self.executor_options.items(): + tf_vars_env_dict[f"TF_VAR_{key}"] = value + f.write(f'{key}="{value}"\n') + + # Plan the infrastructure + retcode = self._run_in_subprocess( + cmd=tf_plan, workdir=self.executor_tf_path, env_vars=tf_vars_env_dict + ) + if retcode != 0: + raise subprocess.CalledProcessError(returncode=retcode, cmd=tf_plan) + + # Create infrastructure as per the plan + if not dry_run: + retcode = self._run_in_subprocess( + cmd=tf_apply, workdir=self.executor_tf_path, env_vars=tf_vars_env_dict + ) + if retcode != 0: + raise subprocess.CalledProcessError(returncode=retcode, cmd=tf_apply) + + # Update covalent executor config based on Terraform output + self._update_config(tf_executor_config_file) + + def down(self, dry_run: bool = True): + """ + Teardown executor resources + """ + terraform = shutil.which("terraform") + if not terraform: + raise CommandNotFoundError("Terraform not found on system") + + if not dry_run: + tfvars_file = os.path.join(self.executor_tf_path, "terraform.tfvars") + + tf_destroy = " ".join([terraform, "destroy", "-auto-approve"]) + retcode = self._run_in_subprocess(cmd=tf_destroy, workdir=self.executor_tf_path) + if retcode != 0: + raise subprocess.CalledProcessError(cmd=tf_destroy, returncode=retcode) + + if os.path.exists(tfvars_file): + os.remove(tfvars_file) + + def status(self): + """ + Return executor resource deployment status + """ + pass From 963216b298878ab0e1efe1e4f8e3d640a1c805a6 Mon Sep 17 00:00:00 2001 From: kessler-frost Date: Mon, 1 May 2023 12:48:23 -0400 Subject: [PATCH 07/46] minor edits to CRM --- covalent/cloud_resource_manager/core.py | 116 +++++++++++------------- 1 file changed, 52 insertions(+), 64 deletions(-) diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index 848a31966..958b731b5 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -24,6 +24,7 @@ import subprocess from configparser import ConfigParser from enum import Enum +from pathlib import Path from typing import Dict, Optional from covalent._shared_files.config import set_config @@ -40,28 +41,6 @@ class DeployStatus(Enum): ERROR = 2 -def is_int(value: str) -> bool: - """ - Check if the string passed is int convertible - """ - try: - int(value) - return True - except ValueError: - return False - - -def is_float(value: str) -> bool: - """ - Check if string is convertible to float - """ - try: - float(value) - return True - except ValueError: - return False - - class CloudResourceManager: """ Base cloud resource manager class @@ -74,22 +53,24 @@ def __init__( options: Optional[Dict[str, str]] = None, ): self.executor_name = executor_name - self.executor_tf_path = os.path.join(executor_module_path, "assets/infra") + self.executor_tf_path = str( + Path(executor_module_path).expanduser().resolve() / "terraform" + ) self.executor_options = options - @staticmethod - def _print_stdout(process: subprocess.Popen) -> Optional[int]: + def _print_stdout(self, process: subprocess.Popen) -> Optional[int]: """ Print the stdout from the subprocess to console - Arg(s) + + Args: process: Python subprocess whose stdout is to be printed to screen - Return(s) - None + + Returns: + returncode of the process """ while process.poll() is None: if proc_stdout := process.stdout.readline(): print(proc_stdout.strip().decode("utf-8")) - else: break return process.poll() @@ -99,11 +80,13 @@ def _run_in_subprocess( ) -> Optional[int]: """ Run the `cmd` in a subprocess shell with the env_vars set in the process's new environment - Arg(s) + + Args: cmd: Command to execute in the subprocess workdir: Working directory of the subprocess env_vars: Dictionary of environment variables to set in the processes execution environment - Return(s) + + Returns: Exit code of the process """ proc = subprocess.Popen( @@ -112,43 +95,48 @@ def _run_in_subprocess( stderr=subprocess.STDOUT, cwd=workdir, shell=True, - bufsize=-1, env=env_vars, ) retcode = self._print_stdout(proc) - return retcode + + if retcode != 0: + raise subprocess.CalledProcessError(returncode=retcode, cmd=cmd) def _update_config(self, tf_executor_config_file: str) -> None: """ - Update covalent configuration with the executor values + Update covalent configuration with the executor + config values as obtained from terraform """ executor_config = ConfigParser() executor_config.read(tf_executor_config_file) for key in executor_config[self.executor_name]: value = executor_config[self.executor_name][key] - converted_value = ( - int(value) if is_int(value) else float(value) if is_float(value) else value - ) - set_config({f"executors.{self.executor_name}.{key}": converted_value}) + set_config({f"executors.{self.executor_name}.{key}": value}) + + def _get_tf_path(self) -> str: + """ + Get the terraform path + """ + if terraform := shutil.which("terraform"): + return terraform + else: + raise CommandNotFoundError("Terraform not found on system") def up(self, dry_run: bool = True): """ Setup executor resources """ - terraform = shutil.which("terraform") - if not terraform: - raise CommandNotFoundError("Terraform not found on system") + terraform = self._get_tf_path() - tfvars_file = os.path.join(self.executor_tf_path, "terraform.tfvars") - tf_executor_config_file = os.path.join(self.executor_tf_path, f"{self.executor_name}.conf") + tfvars_file = Path(self.executor_tf_path) / "terraform.tfvars" + tf_executor_config_file = Path(self.executor_tf_path) / f"{self.executor_name}.conf" tf_init = " ".join([terraform, "init"]) tf_plan = " ".join([terraform, "plan", "-out", "tf.plan"]) tf_apply = " ".join([terraform, "apply", "tf.plan"]) - retcode = self._run_in_subprocess(cmd=tf_init, workdir=self.executor_tf_path) - if retcode != 0: - raise subprocess.CalledProcessError(returncode=retcode, cmd=tf_init) + # Run `terraform init` + self._run_in_subprocess(cmd=tf_init, workdir=self.executor_tf_path) # Setup terraform infra variables as passed by the user tf_vars_env_dict = os.environ.copy() @@ -158,20 +146,17 @@ def up(self, dry_run: bool = True): tf_vars_env_dict[f"TF_VAR_{key}"] = value f.write(f'{key}="{value}"\n') - # Plan the infrastructure - retcode = self._run_in_subprocess( + # Run `terraform plan` + self._run_in_subprocess( cmd=tf_plan, workdir=self.executor_tf_path, env_vars=tf_vars_env_dict ) - if retcode != 0: - raise subprocess.CalledProcessError(returncode=retcode, cmd=tf_plan) # Create infrastructure as per the plan + # Run `terraform apply` if not dry_run: - retcode = self._run_in_subprocess( + self._run_in_subprocess( cmd=tf_apply, workdir=self.executor_tf_path, env_vars=tf_vars_env_dict ) - if retcode != 0: - raise subprocess.CalledProcessError(returncode=retcode, cmd=tf_apply) # Update covalent executor config based on Terraform output self._update_config(tf_executor_config_file) @@ -180,23 +165,26 @@ def down(self, dry_run: bool = True): """ Teardown executor resources """ - terraform = shutil.which("terraform") - if not terraform: - raise CommandNotFoundError("Terraform not found on system") - if not dry_run: - tfvars_file = os.path.join(self.executor_tf_path, "terraform.tfvars") + terraform = self._get_tf_path() + + tfvars_file = Path(self.executor_tf_path) / "terraform.tfvars" tf_destroy = " ".join([terraform, "destroy", "-auto-approve"]) - retcode = self._run_in_subprocess(cmd=tf_destroy, workdir=self.executor_tf_path) - if retcode != 0: - raise subprocess.CalledProcessError(cmd=tf_destroy, returncode=retcode) - if os.path.exists(tfvars_file): - os.remove(tfvars_file) + # Run `terraform destroy` + self._run_in_subprocess(cmd=tf_destroy, workdir=self.executor_tf_path) + + if Path(tfvars_file).exists(): + Path(tfvars_file).unlink() def status(self): """ Return executor resource deployment status """ - pass + terraform = self._get_tf_path() + + tf_state = " ".join([terraform, "state", "list"]) + + # Run `terraform state list` + self._run_in_subprocess(cmd=tf_state, workdir=self.executor_tf_path) From 7b48edd3195ffc35fe18af46e9ba9182fe6e7d0f Mon Sep 17 00:00:00 2001 From: kessler-frost Date: Tue, 2 May 2023 09:09:19 -0400 Subject: [PATCH 08/46] Added validation of user passed options --- covalent/cloud_resource_manager/core.py | 83 +++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 5 deletions(-) diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index 958b731b5..312b66c5b 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -19,6 +19,7 @@ # Relief from the License may be granted by purchasing a commercial license. +import importlib import os import shutil import subprocess @@ -29,6 +30,7 @@ from covalent._shared_files.config import set_config from covalent._shared_files.exceptions import CommandNotFoundError +from covalent.executor import _executor_manager class DeployStatus(Enum): @@ -41,6 +43,53 @@ class DeployStatus(Enum): ERROR = 2 +def get_executor_module(executor_name: str): + return importlib.import_module( + _executor_manager.executor_plugins_map[executor_name].__module__ + ) + + +def get_converted_value(value: str): + """ + Convert the value to the appropriate type + """ + if value.lower() == "true": + return True + elif value.lower() == "false": + return False + elif value.lower() == "null": + return None + elif value.isdigit(): + return int(value) + elif value.replace(".", "", 1).isdigit(): + return float(value) + else: + return value + + +def validate_options(executor_options: Dict[str, str], executor_name: str): + """ + Validate the options passed to the CRM + """ + # Importing validation classes from the executor module + module = get_executor_module(executor_name) + ExecutorPluginDefaults = getattr(module, "ExecutorPluginDefaults") + ExecutorInfraDefaults = getattr(module, "ExecutorInfraDefaults") + + # Validating the passed options: + # TODO: What exactly are the options passed? Are they plugin defaults or infra defaults? + + plugin_attrs = list(ExecutorPluginDefaults.schema()["properties"].keys()) + infra_attrs = list(ExecutorInfraDefaults.schema()["properties"].keys()) + + plugin_params = {k: v for k, v in executor_options.items() if k in plugin_attrs} + infra_params = {k: v for k, v in executor_options.items() if k in infra_attrs} + + # Validate options + ExecutorPluginDefaults(**plugin_params) + ExecutorInfraDefaults(**infra_params) + + class CloudResourceManager: """ Base cloud resource manager class @@ -54,10 +103,15 @@ def __init__( ): self.executor_name = executor_name self.executor_tf_path = str( - Path(executor_module_path).expanduser().resolve() / "terraform" + Path(executor_module_path).expanduser().resolve() / "assets" / "infra" ) + + # Includes both plugin and infra options self.executor_options = options + if self.executor_options: + validate_options(self.executor_options, self.executor_name) + def _print_stdout(self, process: subprocess.Popen) -> Optional[int]: """ Print the stdout from the subprocess to console @@ -107,11 +161,14 @@ def _update_config(self, tf_executor_config_file: str) -> None: Update covalent configuration with the executor config values as obtained from terraform """ + + # Puts the plugin options in covalent's config executor_config = ConfigParser() executor_config.read(tf_executor_config_file) for key in executor_config[self.executor_name]: value = executor_config[self.executor_name][key] - set_config({f"executors.{self.executor_name}.{key}": value}) + converted_value = get_converted_value(value) + set_config({f"executors.{self.executor_name}.{key}": converted_value}) def _get_tf_path(self) -> str: """ @@ -144,6 +201,8 @@ def up(self, dry_run: bool = True): with open(tfvars_file, "w") as f: for key, value in self.executor_options.items(): tf_vars_env_dict[f"TF_VAR_{key}"] = value + + # Write whatever the user has passed to the terraform.tfvars file f.write(f'{key}="{value}"\n') # Run `terraform plan` @@ -154,13 +213,15 @@ def up(self, dry_run: bool = True): # Create infrastructure as per the plan # Run `terraform apply` if not dry_run: - self._run_in_subprocess( + cmd_output = self._run_in_subprocess( cmd=tf_apply, workdir=self.executor_tf_path, env_vars=tf_vars_env_dict ) # Update covalent executor config based on Terraform output self._update_config(tf_executor_config_file) + return cmd_output + def down(self, dry_run: bool = True): """ Teardown executor resources @@ -173,11 +234,13 @@ def down(self, dry_run: bool = True): tf_destroy = " ".join([terraform, "destroy", "-auto-approve"]) # Run `terraform destroy` - self._run_in_subprocess(cmd=tf_destroy, workdir=self.executor_tf_path) + cmd_output = self._run_in_subprocess(cmd=tf_destroy, workdir=self.executor_tf_path) if Path(tfvars_file).exists(): Path(tfvars_file).unlink() + return cmd_output + def status(self): """ Return executor resource deployment status @@ -187,4 +250,14 @@ def status(self): tf_state = " ".join([terraform, "state", "list"]) # Run `terraform state list` - self._run_in_subprocess(cmd=tf_state, workdir=self.executor_tf_path) + return self._run_in_subprocess(cmd=tf_state, workdir=self.executor_tf_path) + + +# if __name__ == "__main__": +# module = get_executor_module("awsbatch") +# ExecutorPluginDefaults: BaseModel = getattr(module, "ExecutorPluginDefaults") +# ExecutorInfraDefaults: BaseModel = getattr(module, "ExecutorInfraDefaults") + +# attrs = list(ExecutorPluginDefaults.schema()["properties"].keys()) +# attrs.extend(list(ExecutorInfraDefaults.schema()["properties"].keys())) +# print(attrs) From 4c094dc5b3004ad5a853175b64ba17d166f8b50e Mon Sep 17 00:00:00 2001 From: kessler-frost Date: Wed, 3 May 2023 09:00:59 -0400 Subject: [PATCH 09/46] all commands work as expected --- covalent/cloud_resource_manager/core.py | 43 ++++++++++++++----------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index 312b66c5b..dbf9700dd 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -24,7 +24,6 @@ import shutil import subprocess from configparser import ConfigParser -from enum import Enum from pathlib import Path from typing import Dict, Optional @@ -33,16 +32,6 @@ from covalent.executor import _executor_manager -class DeployStatus(Enum): - """ - Status of the terraform deployment - """ - - OK = 0 - DESTROYED = 1 - ERROR = 2 - - def get_executor_module(executor_name: str): return importlib.import_module( _executor_manager.executor_plugins_map[executor_name].__module__ @@ -243,7 +232,9 @@ def down(self, dry_run: bool = True): def status(self): """ - Return executor resource deployment status + Return the list of resources being managed by terraform, i.e. + if empty, then either the resources have not been created or + have been destroyed already. """ terraform = self._get_tf_path() @@ -254,10 +245,26 @@ def status(self): # if __name__ == "__main__": -# module = get_executor_module("awsbatch") -# ExecutorPluginDefaults: BaseModel = getattr(module, "ExecutorPluginDefaults") -# ExecutorInfraDefaults: BaseModel = getattr(module, "ExecutorInfraDefaults") -# attrs = list(ExecutorPluginDefaults.schema()["properties"].keys()) -# attrs.extend(list(ExecutorInfraDefaults.schema()["properties"].keys())) -# print(attrs) +# executor_module_path = Path( +# __import__(_executor_manager.executor_plugins_map["awsbatch"].__module__).__path__[0] +# ) + + +# crm = CloudResourceManager( +# executor_name="awsbatch", +# executor_module_path=executor_module_path, +# options={ +# "prefix": "sankalp", +# } +# ) + +# crm.up(dry_run=False) + +# time.sleep(2) + +# crm.status() + +# time.sleep(2) + +# crm.down(dry_run=False) From 46585fd1b56c6dec85d672e5c6a083eed01ad3ac Mon Sep 17 00:00:00 2001 From: Faiyaz Hasan Date: Wed, 3 May 2023 09:23:19 -0400 Subject: [PATCH 10/46] Update main functions --- covalent_dispatcher/_cli/groups/deploy.py | 113 ++++++++++++++++------ 1 file changed, 82 insertions(+), 31 deletions(-) diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index 99a103b0c..87fc7c6ea 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -21,6 +21,7 @@ import asyncio from pathlib import Path +from typing import Dict, Tuple import click @@ -28,54 +29,104 @@ from covalent.executor import _executor_manager -@click.group(invoke_without_command=True) -@click.argument("executor_name", nargs=1) -@click.pass_context -def deploy(ctx: click.Context, executor_name: str): +# TODO - Check if this should go in the executor manager. +def get_executor_module_path(executor_name: str) -> Path: """ - Load the executor plugin installation path based on the executor name provided. + Get the executor module path based on the executor name provided. Will raise KeyError if the `executor_name` plugin is not installed + + Args: + executor_name: Name of executor to get module path for. + + Returns: + Path to executor module. + """ - executor_module_path = Path( + return Path( __import__(_executor_manager.executor_plugins_map[executor_name].__module__).__path__[0] ) - ctx.obj = {"executor_name": executor_name, "executor_module_path": executor_module_path} -@click.command +@click.group(invoke_without_command=True) +def deploy(): + """ + Covalent deploy group with options to: + + 1. Spin resources up via `covalent deploy up `. + 2. Tear resources down via `covalent deploy down `. + 3. Show status of resources via `covalent deploy status `. + 4. Show status of all resources via `covalent deploy status`. + + """ + pass + + +@deploy.command() +@click.argument("executor_name", nargs=1) @click.argument("options", nargs=-1) -@click.pass_obj -def up(executor_metadata: dict, options): - executor_name = executor_metadata["executor_name"] - executor_module_path = executor_metadata["executor_module_path"] +def up(executor_name: str, options: Dict) -> None: + """Spin up resources corresponding to executor. + + Args: + executor_name: Short name of executor to spin up. + options: Options to pass to the Cloud Resource Manager when provisioning the resources. + + Returns: + None + + Examples: + $ covalent deploy up awsbatch region=us-east-1 instance-type=t2.micro + $ covalent deploy up ecs + + """ cmd_options = dict(opt.split("=") for opt in options) + executor_module_path = get_executor_module_path(executor_name) - # Create the cloud resource manager and deploy the resources + # Instantiate the cloud resource manager and spin up resources. crm = CloudResourceManager(executor_name, executor_module_path, cmd_options) click.echo(asyncio.run(crm.up())) -@click.command -@click.argument("options", nargs=-1) -@click.pass_obj -def down(executor_metadata: dict, options): - executor_name = executor_metadata["executor_name"] - executor_module_path = executor_metadata["executor_module_path"] - cmd_options = dict(opt.split("=") for opt in options) +# TODO - Double check if options need to be provided for the UX. +@deploy.command() +@click.argument("executor_name", nargs=1) +def down(executor_name: str) -> None: + """Teardown resources corresponding to executor. - # Create the cloud resource manager and teardown the resources - crm = CloudResourceManager(executor_name, executor_module_path, cmd_options) + Args: + executor_name: Short name of executor to spin up. + + Returns: + None + + Examples: + $ covalent deploy down awsbatch + $ covalent deploy down ecs + + """ + executor_module_path = get_executor_module_path(executor_name) + crm = CloudResourceManager(executor_name, executor_module_path) click.echo(asyncio.run(crm.down())) -@click.command -@click.pass_obj -def status(executor_metadata: dict): - executor_name = executor_metadata["executor_name"] - executor_module_path = executor_metadata["executor_module_path"] - click.echo(executor_module_path) +@deploy.command() +@click.argument("executor_name", nargs=-1, required=False) +def status(executor_name: Tuple[str]) -> None: + """Show executor resource provision status. + Args: + executor_name: Short name(s) of executor to show status for. -deploy.add_command(up) -deploy.add_command(down) -deploy.add_command(status) + Returns: + None + + Examples: + $ covalent deploy status awsbatch + $ covalent deploy status awsbatch ecs + $ covalent deploy status + + """ + if executor_name: + click.echo(f"Showing status for {executor_name}...") + else: + click.echo("Showing status for all executors...") From b710da43c4a283d419216339d2d101212a2e367e Mon Sep 17 00:00:00 2001 From: Faiyaz Hasan Date: Wed, 3 May 2023 10:06:24 -0400 Subject: [PATCH 11/46] Update covalent deploy status cli --- covalent_dispatcher/_cli/groups/deploy.py | 32 ++++++++++++++++------- requirements.txt | 1 + 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index 87fc7c6ea..b97a2d75c 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -24,6 +24,8 @@ from typing import Dict, Tuple import click +from rich.console import Console +from rich.table import Table from covalent.cloud_resource_manager.cloud_resource_manager import CloudResourceManager from covalent.executor import _executor_manager @@ -81,13 +83,10 @@ def up(executor_name: str, options: Dict) -> None: """ cmd_options = dict(opt.split("=") for opt in options) executor_module_path = get_executor_module_path(executor_name) - - # Instantiate the cloud resource manager and spin up resources. crm = CloudResourceManager(executor_name, executor_module_path, cmd_options) click.echo(asyncio.run(crm.up())) -# TODO - Double check if options need to be provided for the UX. @deploy.command() @click.argument("executor_name", nargs=1) def down(executor_name: str) -> None: @@ -110,12 +109,12 @@ def down(executor_name: str) -> None: @deploy.command() -@click.argument("executor_name", nargs=-1, required=False) -def status(executor_name: Tuple[str]) -> None: +@click.argument("executor_names", nargs=-1, required=False) +def status(executor_names: Tuple[str]) -> None: """Show executor resource provision status. Args: - executor_name: Short name(s) of executor to show status for. + executor_names: Short name(s) of executor to show status for. Returns: None @@ -126,7 +125,20 @@ def status(executor_name: Tuple[str]) -> None: $ covalent deploy status """ - if executor_name: - click.echo(f"Showing status for {executor_name}...") - else: - click.echo("Showing status for all executors...") + if not executor_names: + executor_names = _executor_manager.executor_plugins_map.keys() + + table = Table() + table.add_column("Executor", justify="center") + table.add_column("Status", justify="center") + table.add_column("Description", justify="center") + + for executor_name in executor_names: + executor_module_path = get_executor_module_path(executor_name) + crm = CloudResourceManager(executor_name, executor_module_path) + status, description = asyncio.run(crm.status()) + + table.add_row(executor_name, status, description) + + console = Console() + click.echo(console.print(table)) diff --git a/requirements.txt b/requirements.txt index 3a8ac92f4..f203ca040 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ psutil>=5.9.0,<=5.9.2 pydantic>=1.10.1,<=1.10.2 python-socketio==5.7.1 requests>=2.24.0,<=2.28.1 +rich==13.3.5 simplejson==3.17.6 sqlalchemy>=1.4.37,<=1.4.41 sqlalchemy_utils==0.38.3 From f1bdc1a685aba5a92ce19de8e4c144bf89c973f0 Mon Sep 17 00:00:00 2001 From: Faiyaz Hasan Date: Wed, 3 May 2023 10:57:35 -0400 Subject: [PATCH 12/46] Updates --- covalent_dispatcher/_cli/groups/deploy.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index b97a2d75c..2f99cd798 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -65,7 +65,7 @@ def deploy(): @deploy.command() @click.argument("executor_name", nargs=1) -@click.argument("options", nargs=-1) +@click.option("--help", "-h", is_flag=True) def up(executor_name: str, options: Dict) -> None: """Spin up resources corresponding to executor. @@ -108,6 +108,8 @@ def down(executor_name: str) -> None: click.echo(asyncio.run(crm.down())) +# TODO - Key error for uninstalled plugins need to be handled. +# TODO - Color code status. @deploy.command() @click.argument("executor_names", nargs=-1, required=False) def status(executor_names: Tuple[str]) -> None: @@ -137,7 +139,6 @@ def status(executor_names: Tuple[str]) -> None: executor_module_path = get_executor_module_path(executor_name) crm = CloudResourceManager(executor_name, executor_module_path) status, description = asyncio.run(crm.status()) - table.add_row(executor_name, status, description) console = Console() From ba56a0b81ec6e1b9308c03d99303f3039af34826 Mon Sep 17 00:00:00 2001 From: Faiyaz Hasan Date: Wed, 3 May 2023 22:55:11 -0400 Subject: [PATCH 13/46] Update up cli --- .../cloud_resource_manager.py | 209 ------------------ covalent_dispatcher/_cli/groups/deploy.py | 73 ++++-- 2 files changed, 57 insertions(+), 225 deletions(-) delete mode 100644 covalent/cloud_resource_manager/cloud_resource_manager.py diff --git a/covalent/cloud_resource_manager/cloud_resource_manager.py b/covalent/cloud_resource_manager/cloud_resource_manager.py deleted file mode 100644 index ba4353f01..000000000 --- a/covalent/cloud_resource_manager/cloud_resource_manager.py +++ /dev/null @@ -1,209 +0,0 @@ -# Copyright 2023 Agnostiq Inc. -# -# This file is part of Covalent. -# -# Licensed under the GNU Affero General Public License 3.0 (the "License"). -# A copy of the License may be obtained with this software package or at -# -# https://www.gnu.org/licenses/agpl-3.0.en.html -# -# Use of this file is prohibited except in compliance with the License. Any -# modifications or derivative works of this file must retain this copyright -# notice, and modified files must contain a notice indicating that they have -# been altered from the originals. -# -# Covalent is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. -# -# Relief from the License may be granted by purchasing a commercial license. - - -import os -import shutil -import subprocess -from configparser import ConfigParser -from enum import Enum -from typing import Dict, Optional - -from covalent._shared_files.config import set_config -from covalent._shared_files.exceptions import CommandNotFoundError - - -class DeployStatus(Enum): - """ - Status of the terraform deployment - """ - - OK = 0 - DESTROYED = 1 - ERROR = 2 - - -def is_int(value: str) -> bool: - """ - Check if the string passed is int convertible - """ - try: - int(value) - return True - except ValueError: - return False - - -def is_float(value: str) -> bool: - """ - Check if string is convertible to float - """ - try: - float(value) - return True - except ValueError: - return False - - -class CloudResourceManager: - """ - Base cloud resource manager class - """ - - def __init__( - self, - executor_name: str, - executor_module_path: str, - options: Optional[Dict[str, str]] = None, - ): - self.executor_name = executor_name - self.executor_tf_path = os.path.join(executor_module_path, "assets/infra") - self.executor_options = options - - @staticmethod - def _print_stdout(process: subprocess.Popen) -> Optional[int]: - """ - Print the stdout from the subprocess to console - - Arg(s) - process: Python subprocess whose stdout is to be printed to screen - - Return(s) - None - """ - while True: - if process.poll() is not None: - break - proc_stdout = process.stdout.readline() - if not proc_stdout: - break - else: - print(proc_stdout.strip().decode("utf-8")) - - return process.poll() - - def _run_in_subprocess( - self, cmd: str, workdir: str, env_vars: Optional[Dict[str, str]] = None - ) -> Optional[int]: - """ - Run the `cmd` in a subprocess shell with the env_vars set in the process's new environment - - Arg(s) - cmd: Command to execute in the subprocess - workdir: Working directory of the subprocess - env_vars: Dictionary of environment variables to set in the processes execution environment - - Return(s) - Exit code of the process - """ - proc = subprocess.Popen( - args=cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - cwd=workdir, - shell=True, - bufsize=-1, - env=env_vars, - ) - retcode = self._print_stdout(proc) - return retcode - - def _update_config(self, tf_executor_config_file: str) -> None: - """ - Update covalent configuration with the executor values - """ - executor_config = ConfigParser() - executor_config.read(tf_executor_config_file) - for key in executor_config[self.executor_name]: - value = executor_config[self.executor_name][key] - converted_value = ( - int(value) if is_int(value) else float(value) if is_float(value) else value - ) - set_config({f"executors.{self.executor_name}.{key}": converted_value}) - - def up(self, dry_run: bool = True): - """ - Setup executor resources - """ - terraform = shutil.which("terraform") - if not terraform: - raise CommandNotFoundError("Terraform not found on system") - - tfvars_file = os.path.join(self.executor_tf_path, "terraform.tfvars") - tf_executor_config_file = os.path.join(self.executor_tf_path, f"{self.executor_name}.conf") - - tf_init = " ".join([terraform, "init"]) - tf_plan = " ".join([terraform, "plan", "-out", "tf.plan"]) - tf_apply = " ".join([terraform, "apply", "tf.plan"]) - - retcode = self._run_in_subprocess(cmd=tf_init, workdir=self.executor_tf_path) - if retcode != 0: - raise subprocess.CalledProcessError(returncode=retcode, cmd=tf_init) - - # Setup terraform infra variables as passed by the user - tf_vars_env_dict = os.environ.copy() - if self.executor_options: - with open(tfvars_file, "w") as f: - for key, value in self.executor_options.items(): - tf_vars_env_dict[f"TF_VAR_{key}"] = value - f.write(f'{key}="{value}"\n') - - # Plan the infrastructure - retcode = self._run_in_subprocess( - cmd=tf_plan, workdir=self.executor_tf_path, env_vars=tf_vars_env_dict - ) - if retcode != 0: - raise subprocess.CalledProcessError(returncode=retcode, cmd=tf_plan) - - # Create infrastructure as per the plan - if not dry_run: - retcode = self._run_in_subprocess( - cmd=tf_apply, workdir=self.executor_tf_path, env_vars=tf_vars_env_dict - ) - if retcode != 0: - raise subprocess.CalledProcessError(returncode=retcode, cmd=tf_apply) - - # Update covalent executor config based on Terraform output - self._update_config(tf_executor_config_file) - - def down(self, dry_run: bool = True): - """ - Teardown executor resources - """ - terraform = shutil.which("terraform") - if not terraform: - raise CommandNotFoundError("Terraform not found on system") - - tfvars_file = os.path.join(self.executor_tf_path, "terraform.tfvars") - - tf_destroy = " ".join([terraform, "destroy", "-auto-approve"]) - if not dry_run: - retcode = self._run_in_subprocess(cmd=tf_destroy, workdir=self.executor_tf_path) - if retcode != 0: - raise subprocess.CalledProcessError(cmd=tf_destroy, returncode=retcode) - - if os.path.exists(tfvars_file): - os.remove(tfvars_file) - - def status(self): - """ - Return executor resource deployment status - """ - pass diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index 2f99cd798..5c0d88316 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -27,11 +27,10 @@ from rich.console import Console from rich.table import Table -from covalent.cloud_resource_manager.cloud_resource_manager import CloudResourceManager +from covalent.cloud_resource_manager.core import CloudResourceManager from covalent.executor import _executor_manager -# TODO - Check if this should go in the executor manager. def get_executor_module_path(executor_name: str) -> Path: """ Get the executor module path based on the executor name provided. @@ -49,6 +48,17 @@ def get_executor_module_path(executor_name: str) -> Path: ) +def get_crm_object(executor_name: str, options: Dict = None) -> CloudResourceManager: + """ + Get the CloudResourceManager object. + + Returns: + CloudResourceManager object. + + """ + return CloudResourceManager(executor_name, get_executor_module_path(executor_name), options) + + @click.group(invoke_without_command=True) def deploy(): """ @@ -63,10 +73,11 @@ def deploy(): pass -@deploy.command() +@deploy.command(context_settings={"ignore_unknown_options": True}) @click.argument("executor_name", nargs=1) -@click.option("--help", "-h", is_flag=True) -def up(executor_name: str, options: Dict) -> None: +@click.argument("vars", nargs=-1) +@click.option("--help", "-h", is_flag=True, help="Show this message and exit.") +def up(executor_name: str, vars: Dict, help) -> None: """Spin up resources corresponding to executor. Args: @@ -77,13 +88,28 @@ def up(executor_name: str, options: Dict) -> None: None Examples: - $ covalent deploy up awsbatch region=us-east-1 instance-type=t2.micro + $ covalent deploy up awsbatch --region=us-east-1 --instance-type=t2.micro $ covalent deploy up ecs + $ covalent deploy up ecs --help """ - cmd_options = dict(opt.split("=") for opt in options) - executor_module_path = get_executor_module_path(executor_name) - crm = CloudResourceManager(executor_name, executor_module_path, cmd_options) + cmd_options = {key[2:]: value for key, value in (var.split("=") for var in vars)} + crm = get_crm_object(executor_name, cmd_options) + if help: + table = Table() + table.add_column("Argument", justify="center") + table.add_column("Required", justify="center") + table.add_column("Default", justify="center") + table.add_column("Current value", justify="center") + for argument in crm.resource_parameters: + table.add_row( + argument, + crm.resource_parameters[argument]["required"], + crm.resource_parameters[argument]["default"], + crm.resource_parameters[argument]["value"], + ) + return + click.echo(asyncio.run(crm.up())) @@ -103,12 +129,10 @@ def down(executor_name: str) -> None: $ covalent deploy down ecs """ - executor_module_path = get_executor_module_path(executor_name) - crm = CloudResourceManager(executor_name, executor_module_path) + crm = get_crm_object(executor_name) click.echo(asyncio.run(crm.down())) -# TODO - Key error for uninstalled plugins need to be handled. # TODO - Color code status. @deploy.command() @click.argument("executor_names", nargs=-1, required=False) @@ -127,6 +151,13 @@ def status(executor_names: Tuple[str]) -> None: $ covalent deploy status """ + description = { + "up": "Resources are provisioned.", + "down": "Resources are not provisioned.", + "*up": "Resources are partially provisioned.", + "*down": "Resources are partially deprovisioned.", + } + if not executor_names: executor_names = _executor_manager.executor_plugins_map.keys() @@ -135,11 +166,21 @@ def status(executor_names: Tuple[str]) -> None: table.add_column("Status", justify="center") table.add_column("Description", justify="center") + invalid_executor_names = [] for executor_name in executor_names: - executor_module_path = get_executor_module_path(executor_name) - crm = CloudResourceManager(executor_name, executor_module_path) - status, description = asyncio.run(crm.status()) - table.add_row(executor_name, status, description) + try: + crm = get_crm_object(executor_name) + status = asyncio.run(crm.status()) + table.add_row(executor_name, status, description[status]) + except KeyError: + invalid_executor_names.append(executor_name) console = Console() click.echo(console.print(table)) + + if invalid_executor_names: + click.echo( + click.style( + f"{', '.join(invalid_executor_names)} are not valid executors.", fg="yellow" + ) + ) From c81ce74b0bed64356227bb1b2cb7157b12f8ed22 Mon Sep 17 00:00:00 2001 From: Faiyaz Hasan Date: Thu, 4 May 2023 10:09:28 -0400 Subject: [PATCH 14/46] Add dry run tag --- covalent_dispatcher/_cli/groups/deploy.py | 60 +++++++++++++---------- 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index 5c0d88316..62a5af0da 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -31,23 +31,6 @@ from covalent.executor import _executor_manager -def get_executor_module_path(executor_name: str) -> Path: - """ - Get the executor module path based on the executor name provided. - Will raise KeyError if the `executor_name` plugin is not installed - - Args: - executor_name: Name of executor to get module path for. - - Returns: - Path to executor module. - - """ - return Path( - __import__(_executor_manager.executor_plugins_map[executor_name].__module__).__path__[0] - ) - - def get_crm_object(executor_name: str, options: Dict = None) -> CloudResourceManager: """ Get the CloudResourceManager object. @@ -56,7 +39,10 @@ def get_crm_object(executor_name: str, options: Dict = None) -> CloudResourceMan CloudResourceManager object. """ - return CloudResourceManager(executor_name, get_executor_module_path(executor_name), options) + executor_module_path = Path( + __import__(_executor_manager.executor_plugins_map[executor_name].__module__).__path__[0] + ) + return CloudResourceManager(executor_name, executor_module_path, options) @click.group(invoke_without_command=True) @@ -76,8 +62,17 @@ def deploy(): @deploy.command(context_settings={"ignore_unknown_options": True}) @click.argument("executor_name", nargs=1) @click.argument("vars", nargs=-1) -@click.option("--help", "-h", is_flag=True, help="Show this message and exit.") -def up(executor_name: str, vars: Dict, help) -> None: +@click.option( + "--help", "-h", is_flag=True, help="Get info on default and current values for resources." +) +@click.option("--dry-run", "-dr", is_flag=True, help="Get info on current parameter settings.") +@click.option( + "--verbose", + "-v", + is_flag=True, + help="Show the full Terraform output when provisioning resources.", +) +def up(executor_name: str, vars: Dict, help: bool, dry_run: bool) -> None: """Spin up resources corresponding to executor. Args: @@ -91,6 +86,7 @@ def up(executor_name: str, vars: Dict, help) -> None: $ covalent deploy up awsbatch --region=us-east-1 --instance-type=t2.micro $ covalent deploy up ecs $ covalent deploy up ecs --help + $ covalent deploy up awslambda --verbose --region=us-east-1 --instance-type=t2.micro """ cmd_options = {key[2:]: value for key, value in (var.split("=") for var in vars)} @@ -108,14 +104,29 @@ def up(executor_name: str, vars: Dict, help) -> None: crm.resource_parameters[argument]["default"], crm.resource_parameters[argument]["value"], ) + click.echo(Console().print(table)) return - click.echo(asyncio.run(crm.up())) + if dry_run: + asyncio.run(crm.up(dry_run=dry_run)) + table = Table() + table.add_column("Settings", justify="center") + for argument in crm.resource_parameters: + table.add_row(f"{argument: crm.resource_parameters[argument]['value']}") + click.echo(Console().print(table)) + else: + click.echo(asyncio.run(crm.up())) @deploy.command() @click.argument("executor_name", nargs=1) -def down(executor_name: str) -> None: +@click.option( + "--verbose", + "-v", + is_flag=True, + help="Show the full Terraform output when spinning down resources.", +) +def down(executor_name: str, verbose: bool) -> None: """Teardown resources corresponding to executor. Args: @@ -126,7 +137,7 @@ def down(executor_name: str) -> None: Examples: $ covalent deploy down awsbatch - $ covalent deploy down ecs + $ covalent deploy down ecs --verbose """ crm = get_crm_object(executor_name) @@ -175,8 +186,7 @@ def status(executor_names: Tuple[str]) -> None: except KeyError: invalid_executor_names.append(executor_name) - console = Console() - click.echo(console.print(table)) + click.echo(Console().print(table)) if invalid_executor_names: click.echo( From 86ec8501a5d8e5f5b291ffc19cfe3e2983f4ba1a Mon Sep 17 00:00:00 2001 From: Faiyaz Hasan Date: Thu, 4 May 2023 10:56:23 -0400 Subject: [PATCH 15/46] Add callback method --- covalent/cloud_resource_manager/core.py | 42 ++++++++++++++--------- covalent_dispatcher/_cli/groups/deploy.py | 41 +++++++++++++++++++--- 2 files changed, 62 insertions(+), 21 deletions(-) diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index dbf9700dd..cb91bd632 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -25,7 +25,7 @@ import subprocess from configparser import ConfigParser from pathlib import Path -from typing import Dict, Optional +from typing import Callable, Dict, Optional from covalent._shared_files.config import set_config from covalent._shared_files.exceptions import CommandNotFoundError @@ -101,7 +101,7 @@ def __init__( if self.executor_options: validate_options(self.executor_options, self.executor_name) - def _print_stdout(self, process: subprocess.Popen) -> Optional[int]: + def _print_stdout(self, process: subprocess.Popen, print_callback: Callable) -> Optional[int]: """ Print the stdout from the subprocess to console @@ -113,13 +113,17 @@ def _print_stdout(self, process: subprocess.Popen) -> Optional[int]: """ while process.poll() is None: if proc_stdout := process.stdout.readline(): - print(proc_stdout.strip().decode("utf-8")) + print_callback(proc_stdout.strip().decode("utf-8")) else: break return process.poll() def _run_in_subprocess( - self, cmd: str, workdir: str, env_vars: Optional[Dict[str, str]] = None + self, + cmd: str, + workdir: str, + print_callback: Callable, + env_vars: Optional[Dict[str, str]] = None, ) -> Optional[int]: """ Run the `cmd` in a subprocess shell with the env_vars set in the process's new environment @@ -140,7 +144,7 @@ def _run_in_subprocess( shell=True, env=env_vars, ) - retcode = self._print_stdout(proc) + retcode = self._print_stdout(proc, print_callback) if retcode != 0: raise subprocess.CalledProcessError(returncode=retcode, cmd=cmd) @@ -168,7 +172,7 @@ def _get_tf_path(self) -> str: else: raise CommandNotFoundError("Terraform not found on system") - def up(self, dry_run: bool = True): + def up(self, print_callback: Callable, dry_run: bool = True): """ Setup executor resources """ @@ -196,7 +200,10 @@ def up(self, dry_run: bool = True): # Run `terraform plan` self._run_in_subprocess( - cmd=tf_plan, workdir=self.executor_tf_path, env_vars=tf_vars_env_dict + cmd=tf_plan, + workdir=self.executor_tf_path, + env_vars=tf_vars_env_dict, + print_callback=print_callback, ) # Create infrastructure as per the plan @@ -211,24 +218,25 @@ def up(self, dry_run: bool = True): return cmd_output - def down(self, dry_run: bool = True): + def down(self, print_callback: Callable): """ Teardown executor resources """ - if not dry_run: - terraform = self._get_tf_path() + terraform = self._get_tf_path() - tfvars_file = Path(self.executor_tf_path) / "terraform.tfvars" + tfvars_file = Path(self.executor_tf_path) / "terraform.tfvars" - tf_destroy = " ".join([terraform, "destroy", "-auto-approve"]) + tf_destroy = " ".join([terraform, "destroy", "-auto-approve"]) - # Run `terraform destroy` - cmd_output = self._run_in_subprocess(cmd=tf_destroy, workdir=self.executor_tf_path) + # Run `terraform destroy` + cmd_output = self._run_in_subprocess( + cmd=tf_destroy, workdir=self.executor_tf_path, print_callback=print_callback + ) - if Path(tfvars_file).exists(): - Path(tfvars_file).unlink() + if Path(tfvars_file).exists(): + Path(tfvars_file).unlink() - return cmd_output + return cmd_output def status(self): """ diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index 62a5af0da..e1bcb59f8 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -20,6 +20,7 @@ import asyncio +from functools import partial from pathlib import Path from typing import Dict, Tuple @@ -45,6 +46,36 @@ def get_crm_object(executor_name: str, options: Dict = None) -> CloudResourceMan return CloudResourceManager(executor_name, executor_module_path, options) +def preprocess_msg(msg: str) -> str: + """Preprocess Terraform output message + + Args: + msg: Message to be preprocessed. + + Returns: + Preprocessed message. + + """ + return msg.strip() + + +def print_callback(msg: str, verbose: bool = True) -> None: + """Print Terraform output to the console + + Args: + msg: Message to be printed on the console. + verbose: If False, print the message to the console inline otherwise add a new line. + + """ + if not (filtered_msg := preprocess_msg(msg)): + return + + if verbose: + click.echo(filtered_msg) + else: + click.echo(filtered_msg, nl=False) + + @click.group(invoke_without_command=True) def deploy(): """ @@ -72,7 +103,7 @@ def deploy(): is_flag=True, help="Show the full Terraform output when provisioning resources.", ) -def up(executor_name: str, vars: Dict, help: bool, dry_run: bool) -> None: +def up(executor_name: str, vars: Dict, help: bool, dry_run: bool, verbose: bool) -> None: """Spin up resources corresponding to executor. Args: @@ -107,15 +138,17 @@ def up(executor_name: str, vars: Dict, help: bool, dry_run: bool) -> None: click.echo(Console().print(table)) return + console_msg = asyncio.run( + crm.up(dry_run=dry_run, print_callback=partial(print_callback, verbose=verbose)) + ) if dry_run: - asyncio.run(crm.up(dry_run=dry_run)) table = Table() table.add_column("Settings", justify="center") for argument in crm.resource_parameters: table.add_row(f"{argument: crm.resource_parameters[argument]['value']}") click.echo(Console().print(table)) else: - click.echo(asyncio.run(crm.up())) + click.echo(console_msg) @deploy.command() @@ -141,7 +174,7 @@ def down(executor_name: str, verbose: bool) -> None: """ crm = get_crm_object(executor_name) - click.echo(asyncio.run(crm.down())) + click.echo(asyncio.run(crm.down(partial(print_callback, verbose=verbose)))) # TODO - Color code status. From 2dee10167bb59ff1ab4e7c8416ef2687f5ed49ee Mon Sep 17 00:00:00 2001 From: Faiyaz Hasan Date: Thu, 4 May 2023 11:24:06 -0400 Subject: [PATCH 16/46] Update filter lines --- covalent_dispatcher/_cli/groups/deploy.py | 25 +++++++++++++---------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index e1bcb59f8..11994f788 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -22,7 +22,7 @@ import asyncio from functools import partial from pathlib import Path -from typing import Dict, Tuple +from typing import Dict, List, Tuple import click from rich.console import Console @@ -46,17 +46,23 @@ def get_crm_object(executor_name: str, options: Dict = None) -> CloudResourceMan return CloudResourceManager(executor_name, executor_module_path, options) -def preprocess_msg(msg: str) -> str: - """Preprocess Terraform output message +def filter_lines(lines: List[str]) -> List[str]: + """Filters out empty lines and lines comprised only of delimiters. Args: - msg: Message to be preprocessed. + lines (List[str]): A list of strings representing the lines to filter. + delimiters (List[str]): A list of strings representing the delimiters to check for. Returns: - Preprocessed message. + List[str]: A list of strings representing the non-empty lines. """ - return msg.strip() + delimiters = ["\n", "\r", "\t", " "] + return [ + line.strip() + for line in lines + if line.strip() and any(d not in delimiters for d in line.strip()) + ] def print_callback(msg: str, verbose: bool = True) -> None: @@ -67,13 +73,10 @@ def print_callback(msg: str, verbose: bool = True) -> None: verbose: If False, print the message to the console inline otherwise add a new line. """ - if not (filtered_msg := preprocess_msg(msg)): + if not (filtered_lines := filter_lines([msg])): return - if verbose: - click.echo(filtered_msg) - else: - click.echo(filtered_msg, nl=False) + click.echo(filtered_lines[0], nl=verbose) @click.group(invoke_without_command=True) From 13360529fa32db82c4cd75fd37b573424cfd719c Mon Sep 17 00:00:00 2001 From: Faiyaz Hasan Date: Thu, 4 May 2023 11:26:33 -0400 Subject: [PATCH 17/46] minor update --- covalent_dispatcher/_cli/groups/deploy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index 11994f788..882b597e1 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -227,6 +227,7 @@ def status(executor_names: Tuple[str]) -> None: if invalid_executor_names: click.echo( click.style( - f"{', '.join(invalid_executor_names)} are not valid executors.", fg="yellow" + f"Warning: {', '.join(invalid_executor_names)} are not valid executors.", + fg="yellow", ) ) From 4462fd6fc788f108a158d7c4dc9557035466244e Mon Sep 17 00:00:00 2001 From: Faiyaz Hasan Date: Thu, 4 May 2023 15:04:50 -0400 Subject: [PATCH 18/46] Update changes --- covalent/cloud_resource_manager/core.py | 18 +++++++++++----- covalent_dispatcher/_cli/groups/deploy.py | 26 +++++++++++++++++------ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index cb91bd632..6757c1b28 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -172,7 +172,7 @@ def _get_tf_path(self) -> str: else: raise CommandNotFoundError("Terraform not found on system") - def up(self, print_callback: Callable, dry_run: bool = True): + def up(self, print_callback: Callable, progressbar_callback: Callable, dry_run: bool = True): """ Setup executor resources """ @@ -186,7 +186,10 @@ def up(self, print_callback: Callable, dry_run: bool = True): tf_apply = " ".join([terraform, "apply", "tf.plan"]) # Run `terraform init` - self._run_in_subprocess(cmd=tf_init, workdir=self.executor_tf_path) + self._run_in_subprocess( + cmd=tf_init, workdir=self.executor_tf_path, print_callback=print_callback + ) + progressbar_callback("Provisioning infrastructure ...") # Setup terraform infra variables as passed by the user tf_vars_env_dict = os.environ.copy() @@ -210,7 +213,10 @@ def up(self, print_callback: Callable, dry_run: bool = True): # Run `terraform apply` if not dry_run: cmd_output = self._run_in_subprocess( - cmd=tf_apply, workdir=self.executor_tf_path, env_vars=tf_vars_env_dict + cmd=tf_apply, + workdir=self.executor_tf_path, + env_vars=tf_vars_env_dict, + print_callback=print_callback, ) # Update covalent executor config based on Terraform output @@ -238,7 +244,7 @@ def down(self, print_callback: Callable): return cmd_output - def status(self): + def status(self, print_callback: Callable): """ Return the list of resources being managed by terraform, i.e. if empty, then either the resources have not been created or @@ -249,7 +255,9 @@ def status(self): tf_state = " ".join([terraform, "state", "list"]) # Run `terraform state list` - return self._run_in_subprocess(cmd=tf_state, workdir=self.executor_tf_path) + return self._run_in_subprocess( + cmd=tf_state, workdir=self.executor_tf_path, print_callback=print_callback + ) # if __name__ == "__main__": diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index 882b597e1..80c037529 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -20,6 +20,7 @@ import asyncio +import time from functools import partial from pathlib import Path from typing import Dict, List, Tuple @@ -65,7 +66,15 @@ def filter_lines(lines: List[str]) -> List[str]: ] -def print_callback(msg: str, verbose: bool = True) -> None: +def add_progress_bar(label: str) -> None: + """Add progress bar to the console.""" + with click.progressbar(length=10, label=label) as progress_bar: + for _ in range(10): + time.sleep(0.5) + progress_bar.update(1) + + +def print_callback(msg: str, verbose: bool = False) -> None: """Print Terraform output to the console Args: @@ -73,10 +82,8 @@ def print_callback(msg: str, verbose: bool = True) -> None: verbose: If False, print the message to the console inline otherwise add a new line. """ - if not (filtered_lines := filter_lines([msg])): - return - - click.echo(filtered_lines[0], nl=verbose) + if verbose and (filtered_lines := filter_lines([msg])): + click.echo(filtered_lines[0]) @click.group(invoke_without_command=True) @@ -142,8 +149,13 @@ def up(executor_name: str, vars: Dict, help: bool, dry_run: bool, verbose: bool) return console_msg = asyncio.run( - crm.up(dry_run=dry_run, print_callback=partial(print_callback, verbose=verbose)) + crm.up( + dry_run=dry_run, + print_callback=partial(print_callback, verbose=verbose), + progressbar_callback=add_progress_bar, + ) ) + if dry_run: table = Table() table.add_column("Settings", justify="center") @@ -217,7 +229,7 @@ def status(executor_names: Tuple[str]) -> None: for executor_name in executor_names: try: crm = get_crm_object(executor_name) - status = asyncio.run(crm.status()) + status = asyncio.run(crm.status(partial(print_callback, verbose=False))) table.add_row(executor_name, status, description[status]) except KeyError: invalid_executor_names.append(executor_name) From 1bdd413d8481ee6f26c4b2557c59bbb9ace8a7da Mon Sep 17 00:00:00 2001 From: kessler-frost Date: Fri, 5 May 2023 10:28:02 -0400 Subject: [PATCH 19/46] Adding unit tests for CRM --- .github/workflows/requirements.yml | 1 + .../cloud_resource_manager/__init__.py | 19 +++++++++++++++++++ .../cloud_resource_manager/core_test.py | 19 +++++++++++++++++++ 3 files changed, 39 insertions(+) create mode 100644 tests/covalent_tests/cloud_resource_manager/__init__.py create mode 100644 tests/covalent_tests/cloud_resource_manager/core_test.py diff --git a/.github/workflows/requirements.yml b/.github/workflows/requirements.yml index a4bd6cbb6..d9bb5e71a 100644 --- a/.github/workflows/requirements.yml +++ b/.github/workflows/requirements.yml @@ -53,6 +53,7 @@ jobs: --ignore-module=pkg_resources --ignore-file=covalent/executor/** --ignore-file=covalent/triggers/** + --ignore-file=covalent/cloud_resource_manager/** covalent - name: Check missing dispatcher requirements diff --git a/tests/covalent_tests/cloud_resource_manager/__init__.py b/tests/covalent_tests/cloud_resource_manager/__init__.py new file mode 100644 index 000000000..9d1b05526 --- /dev/null +++ b/tests/covalent_tests/cloud_resource_manager/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2023 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the GNU Affero General Public License 3.0 (the "License"). +# A copy of the License may be obtained with this software package or at +# +# https://www.gnu.org/licenses/agpl-3.0.en.html +# +# Use of this file is prohibited except in compliance with the License. Any +# modifications or derivative works of this file must retain this copyright +# notice, and modified files must contain a notice indicating that they have +# been altered from the originals. +# +# Covalent is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. +# +# Relief from the License may be granted by purchasing a commercial license. diff --git a/tests/covalent_tests/cloud_resource_manager/core_test.py b/tests/covalent_tests/cloud_resource_manager/core_test.py new file mode 100644 index 000000000..9d1b05526 --- /dev/null +++ b/tests/covalent_tests/cloud_resource_manager/core_test.py @@ -0,0 +1,19 @@ +# Copyright 2023 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the GNU Affero General Public License 3.0 (the "License"). +# A copy of the License may be obtained with this software package or at +# +# https://www.gnu.org/licenses/agpl-3.0.en.html +# +# Use of this file is prohibited except in compliance with the License. Any +# modifications or derivative works of this file must retain this copyright +# notice, and modified files must contain a notice indicating that they have +# been altered from the originals. +# +# Covalent is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. +# +# Relief from the License may be granted by purchasing a commercial license. From 2c0df1a94178d8f77a72398261785a1604532437 Mon Sep 17 00:00:00 2001 From: Faiyaz Hasan Date: Fri, 5 May 2023 11:21:47 -0400 Subject: [PATCH 20/46] refactor some code --- covalent/cloud_resource_manager/core.py | 26 ++-- covalent_dispatcher/_cli/groups/deploy.py | 140 +++++++++++++--------- 2 files changed, 95 insertions(+), 71 deletions(-) diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index 6757c1b28..6ab70d38c 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -101,7 +101,9 @@ def __init__( if self.executor_options: validate_options(self.executor_options, self.executor_name) - def _print_stdout(self, process: subprocess.Popen, print_callback: Callable) -> Optional[int]: + def _print_stdout( + self, process: subprocess.Popen, print_callback: Callable = None + ) -> Optional[int]: """ Print the stdout from the subprocess to console @@ -111,18 +113,16 @@ def _print_stdout(self, process: subprocess.Popen, print_callback: Callable) -> Returns: returncode of the process """ - while process.poll() is None: - if proc_stdout := process.stdout.readline(): + while process.poll() is None and (proc_stdout := process.stdout.readline()): + if print_callback: print_callback(proc_stdout.strip().decode("utf-8")) - else: - break return process.poll() def _run_in_subprocess( self, cmd: str, workdir: str, - print_callback: Callable, + print_callback: Callable = None, env_vars: Optional[Dict[str, str]] = None, ) -> Optional[int]: """ @@ -172,7 +172,7 @@ def _get_tf_path(self) -> str: else: raise CommandNotFoundError("Terraform not found on system") - def up(self, print_callback: Callable, progressbar_callback: Callable, dry_run: bool = True): + def up(self, print_callback: Callable, dry_run: bool = True): """ Setup executor resources """ @@ -186,10 +186,7 @@ def up(self, print_callback: Callable, progressbar_callback: Callable, dry_run: tf_apply = " ".join([terraform, "apply", "tf.plan"]) # Run `terraform init` - self._run_in_subprocess( - cmd=tf_init, workdir=self.executor_tf_path, print_callback=print_callback - ) - progressbar_callback("Provisioning infrastructure ...") + self._run_in_subprocess(cmd=tf_init, workdir=self.executor_tf_path) # Setup terraform infra variables as passed by the user tf_vars_env_dict = os.environ.copy() @@ -206,7 +203,6 @@ def up(self, print_callback: Callable, progressbar_callback: Callable, dry_run: cmd=tf_plan, workdir=self.executor_tf_path, env_vars=tf_vars_env_dict, - print_callback=print_callback, ) # Create infrastructure as per the plan @@ -244,7 +240,7 @@ def down(self, print_callback: Callable): return cmd_output - def status(self, print_callback: Callable): + def status(self): """ Return the list of resources being managed by terraform, i.e. if empty, then either the resources have not been created or @@ -255,9 +251,7 @@ def status(self, print_callback: Callable): tf_state = " ".join([terraform, "state", "list"]) # Run `terraform state list` - return self._run_in_subprocess( - cmd=tf_state, workdir=self.executor_tf_path, print_callback=print_callback - ) + return self._run_in_subprocess(cmd=tf_state, workdir=self.executor_tf_path) # if __name__ == "__main__": diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index 80c037529..896dcb878 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -20,10 +20,8 @@ import asyncio -import time -from functools import partial from pathlib import Path -from typing import Dict, List, Tuple +from typing import Dict, Tuple import click from rich.console import Console @@ -47,43 +45,70 @@ def get_crm_object(executor_name: str, options: Dict = None) -> CloudResourceMan return CloudResourceManager(executor_name, executor_module_path, options) -def filter_lines(lines: List[str]) -> List[str]: - """Filters out empty lines and lines comprised only of delimiters. +def get_print_callback( + console: Console, console_status: Console.status, prepend_msg: str, verbose: bool +): + """Get print callback method. Args: - lines (List[str]): A list of strings representing the lines to filter. - delimiters (List[str]): A list of strings representing the delimiters to check for. + console: Rich console object. + console_status: Console status object. + prepend_msg: Message to prepend to the output. + verbose: Whether to print the output inline or not. Returns: - List[str]: A list of strings representing the non-empty lines. + Callback method. """ - delimiters = ["\n", "\r", "\t", " "] - return [ - line.strip() - for line in lines - if line.strip() and any(d not in delimiters for d in line.strip()) - ] + if verbose: + return console.log + def inline_print_callback(msg): + status.update(f"{prepend_msg} {msg}") -def add_progress_bar(label: str) -> None: - """Add progress bar to the console.""" - with click.progressbar(length=10, label=label) as progress_bar: - for _ in range(10): - time.sleep(0.5) - progress_bar.update(1) + return inline_print_callback -def print_callback(msg: str, verbose: bool = False) -> None: - """Print Terraform output to the console +def get_settings_table(crm: CloudResourceManager) -> Table: + """Get resource provisioning settings table. Args: - msg: Message to be printed on the console. - verbose: If False, print the message to the console inline otherwise add a new line. + crm: CloudResourceManager object. + + Returns: + Table with resource provisioning settings. + + """ + table = Table() + table.add_column("Settings", justify="center") + for argument in crm.resource_parameters: + table.add_row(f"{argument: crm.resource_parameters[argument]['value']}") + return table + + +def get_up_help_table(crm: CloudResourceManager) -> Table: + """Get resource provisioning help table. + + Args: + crm: CloudResourceManager object. + + Returns: + Table with resource provisioning help. """ - if verbose and (filtered_lines := filter_lines([msg])): - click.echo(filtered_lines[0]) + table = Table() + table.add_column("Argument", justify="center") + table.add_column("Required", justify="center") + table.add_column("Default", justify="center") + table.add_column("Current value", justify="center") + for argument in crm.resource_parameters: + table.add_row( + argument, + crm.resource_parameters[argument]["required"], + crm.resource_parameters[argument]["default"], + crm.resource_parameters[argument]["value"], + ) + return table @click.group(invoke_without_command=True) @@ -133,37 +158,27 @@ def up(executor_name: str, vars: Dict, help: bool, dry_run: bool, verbose: bool) cmd_options = {key[2:]: value for key, value in (var.split("=") for var in vars)} crm = get_crm_object(executor_name, cmd_options) if help: - table = Table() - table.add_column("Argument", justify="center") - table.add_column("Required", justify="center") - table.add_column("Default", justify="center") - table.add_column("Current value", justify="center") - for argument in crm.resource_parameters: - table.add_row( - argument, - crm.resource_parameters[argument]["required"], - crm.resource_parameters[argument]["default"], - crm.resource_parameters[argument]["value"], - ) - click.echo(Console().print(table)) + click.echo(Console().print(get_up_help_table(crm))) return - console_msg = asyncio.run( - crm.up( - dry_run=dry_run, - print_callback=partial(print_callback, verbose=verbose), - progressbar_callback=add_progress_bar, + console = Console() + prepend_msg = "[bold green] Provisioning resources..." + click.echo(Console().print(get_settings_table(crm))) + + with console.status(prepend_msg) as status: + console_msg = asyncio.run( + crm.up( + dry_run=dry_run, + print_callback=get_print_callback( + console=console, + console_status=status, + prepend_msg=prepend_msg, + verbose=verbose, + ), + ) ) - ) - if dry_run: - table = Table() - table.add_column("Settings", justify="center") - for argument in crm.resource_parameters: - table.add_row(f"{argument: crm.resource_parameters[argument]['value']}") - click.echo(Console().print(table)) - else: - click.echo(console_msg) + click.echo(console_msg) @deploy.command() @@ -189,7 +204,22 @@ def down(executor_name: str, verbose: bool) -> None: """ crm = get_crm_object(executor_name) - click.echo(asyncio.run(crm.down(partial(print_callback, verbose=verbose)))) + + console = Console() + prepend_msg = "[bold green] Destroying resources..." + with console.status(prepend_msg) as status: + console_msg = asyncio.run( + crm.down( + print_callback=get_print_callback( + console=console, + console_status=status, + prepend_msg=prepend_msg, + verbose=verbose, + ) + ) + ) + + click.echo(console_msg) # TODO - Color code status. @@ -229,7 +259,7 @@ def status(executor_names: Tuple[str]) -> None: for executor_name in executor_names: try: crm = get_crm_object(executor_name) - status = asyncio.run(crm.status(partial(print_callback, verbose=False))) + status = asyncio.run(crm.status()) table.add_row(executor_name, status, description[status]) except KeyError: invalid_executor_names.append(executor_name) From 97d174d8826c3782e6873ea7ad2c7739dbcc7d0f Mon Sep 17 00:00:00 2001 From: Faiyaz Hasan Date: Fri, 5 May 2023 11:25:12 -0400 Subject: [PATCH 21/46] fix imports --- covalent/cloud_resource_manager/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index 6ab70d38c..1062109d6 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -27,9 +27,9 @@ from pathlib import Path from typing import Callable, Dict, Optional -from covalent._shared_files.config import set_config -from covalent._shared_files.exceptions import CommandNotFoundError -from covalent.executor import _executor_manager +from .._shared_files.config import set_config +from .._shared_files.exceptions import CommandNotFoundError +from ..executor import _executor_manager def get_executor_module(executor_name: str): From 63cabe5c6ccce5691fd6a8a03c42988ef81dc343 Mon Sep 17 00:00:00 2001 From: Faiyaz Hasan Date: Fri, 5 May 2023 12:03:51 -0400 Subject: [PATCH 22/46] fix dispatcher initialization --- covalent/_shared_files/config.py | 1 + covalent_dispatcher/_cli/service.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/covalent/_shared_files/config.py b/covalent/_shared_files/config.py index e24405a85..74403b43f 100644 --- a/covalent/_shared_files/config.py +++ b/covalent/_shared_files/config.py @@ -57,6 +57,7 @@ def __init__(self) -> None: Path(self.config_file).parent.mkdir(parents=True, exist_ok=True) self.write_config() + Path(get_config("dispatcher.db_path")).parent.mkdir(parents=True, exist_ok=True) Path(self.get("sdk.log_dir")).mkdir(parents=True, exist_ok=True) Path(self.get("sdk.executor_dir")).mkdir(parents=True, exist_ok=True) diff --git a/covalent_dispatcher/_cli/service.py b/covalent_dispatcher/_cli/service.py index 41f4cc59b..4be089b5a 100644 --- a/covalent_dispatcher/_cli/service.py +++ b/covalent_dispatcher/_cli/service.py @@ -225,7 +225,6 @@ def _graceful_start( Path(get_config("dispatcher.results_dir")).mkdir(parents=True, exist_ok=True) Path(get_config("dispatcher.log_dir")).mkdir(parents=True, exist_ok=True) Path(get_config("user_interface.log_dir")).mkdir(parents=True, exist_ok=True) - Path(get_config("dispatcher.db_path")).parent.mkdir(parents=True, exist_ok=True) return port From a1e84df1d41e2fda5cb5ab35a42d3bdc92c443b9 Mon Sep 17 00:00:00 2001 From: Faiyaz Hasan Date: Fri, 5 May 2023 12:15:02 -0400 Subject: [PATCH 23/46] fix dispatcher initialization --- covalent/_shared_files/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/covalent/_shared_files/config.py b/covalent/_shared_files/config.py index 74403b43f..4a6809154 100644 --- a/covalent/_shared_files/config.py +++ b/covalent/_shared_files/config.py @@ -44,6 +44,8 @@ def __init__(self) -> None: DEFAULT_CONFIG = asdict(DefaultConfig()) + Path(get_config("dispatcher.db_path")).parent.mkdir(parents=True, exist_ok=True) + self.config_file = DEFAULT_CONFIG["sdk"]["config_file"] self.generate_default_config() @@ -57,7 +59,6 @@ def __init__(self) -> None: Path(self.config_file).parent.mkdir(parents=True, exist_ok=True) self.write_config() - Path(get_config("dispatcher.db_path")).parent.mkdir(parents=True, exist_ok=True) Path(self.get("sdk.log_dir")).mkdir(parents=True, exist_ok=True) Path(self.get("sdk.executor_dir")).mkdir(parents=True, exist_ok=True) From 2543890c7a987a67a863d1d394a10f63feca6280 Mon Sep 17 00:00:00 2001 From: Faiyaz Hasan Date: Fri, 5 May 2023 13:31:29 -0400 Subject: [PATCH 24/46] update --- covalent/_shared_files/config.py | 2 -- covalent_dispatcher/_cli/service.py | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/covalent/_shared_files/config.py b/covalent/_shared_files/config.py index 4a6809154..e24405a85 100644 --- a/covalent/_shared_files/config.py +++ b/covalent/_shared_files/config.py @@ -44,8 +44,6 @@ def __init__(self) -> None: DEFAULT_CONFIG = asdict(DefaultConfig()) - Path(get_config("dispatcher.db_path")).parent.mkdir(parents=True, exist_ok=True) - self.config_file = DEFAULT_CONFIG["sdk"]["config_file"] self.generate_default_config() diff --git a/covalent_dispatcher/_cli/service.py b/covalent_dispatcher/_cli/service.py index 4be089b5a..41f4cc59b 100644 --- a/covalent_dispatcher/_cli/service.py +++ b/covalent_dispatcher/_cli/service.py @@ -225,6 +225,7 @@ def _graceful_start( Path(get_config("dispatcher.results_dir")).mkdir(parents=True, exist_ok=True) Path(get_config("dispatcher.log_dir")).mkdir(parents=True, exist_ok=True) Path(get_config("user_interface.log_dir")).mkdir(parents=True, exist_ok=True) + Path(get_config("dispatcher.db_path")).parent.mkdir(parents=True, exist_ok=True) return port From c00fb7dc3e471dd8960c3147949362459093412f Mon Sep 17 00:00:00 2001 From: Faiyaz Hasan Date: Fri, 5 May 2023 13:42:42 -0400 Subject: [PATCH 25/46] fix broken test --- tests/covalent_dispatcher_tests/_cli/cli_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/covalent_dispatcher_tests/_cli/cli_test.py b/tests/covalent_dispatcher_tests/_cli/cli_test.py index d7a87293e..dda406487 100644 --- a/tests/covalent_dispatcher_tests/_cli/cli_test.py +++ b/tests/covalent_dispatcher_tests/_cli/cli_test.py @@ -60,6 +60,7 @@ def test_cli_commands(): "cluster", "config", "db", + "deploy", "logs", "migrate-legacy-result-object", "purge", From 628a018f38e4eadc7feff198be9a7a7ff3f89971 Mon Sep 17 00:00:00 2001 From: Faiyaz Hasan Date: Mon, 8 May 2023 11:03:38 -0400 Subject: [PATCH 26/46] Got up and down working --- covalent/cloud_resource_manager/core.py | 85 ++++++++++++++++++++++- covalent_dispatcher/_cli/groups/deploy.py | 46 +++++++----- 2 files changed, 110 insertions(+), 21 deletions(-) diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index 1062109d6..796afb1e9 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -27,10 +27,14 @@ from pathlib import Path from typing import Callable, Dict, Optional +from .._shared_files import logger from .._shared_files.config import set_config from .._shared_files.exceptions import CommandNotFoundError from ..executor import _executor_manager +app_log = logger.app_log +log_stack_info = logger.log_stack_info + def get_executor_module(executor_name: str): return importlib.import_module( @@ -71,6 +75,9 @@ def validate_options(executor_options: Dict[str, str], executor_name: str): plugin_attrs = list(ExecutorPluginDefaults.schema()["properties"].keys()) infra_attrs = list(ExecutorInfraDefaults.schema()["properties"].keys()) + # app_log.debug(f"Plugin attrs: {plugin_attrs}") + # app_log.debug(f"Infra attrs: {infra_attrs}") + plugin_params = {k: v for k, v in executor_options.items() if k in plugin_attrs} infra_params = {k: v for k, v in executor_options.items() if k in infra_attrs} @@ -78,6 +85,52 @@ def validate_options(executor_options: Dict[str, str], executor_name: str): ExecutorPluginDefaults(**plugin_params) ExecutorInfraDefaults(**infra_params) + # app_log.debug(f"Plugin params: {plugin_params}") + # app_log.debug(f"Infra params: {infra_params}") + + +def get_plugin_settings(executor_name: str, executor_options: Dict) -> Dict: + """Get plugin settings.""" + module = get_executor_module(executor_name) + ExecutorPluginDefaults = getattr(module, "ExecutorPluginDefaults") + ExecutorInfraDefaults = getattr(module, "ExecutorInfraDefaults") + + plugin_settings = ExecutorPluginDefaults.schema()["properties"] + infra_settings = ExecutorInfraDefaults.schema()["properties"] + + # app_log.debug(f"Executor plugin settings: {plugin_settings}") + # app_log.debug(f"Executor infra settings: {infra_settings}") + + settings_dict = { + key: { + "required": "No", + "default": value["default"], + "value": value["default"], + } + if "default" in value + else {"required": "Yes", "default": None, "value": None} + for key, value in plugin_settings.items() + } + for key, value in infra_settings.items(): + if "default" in value: + settings_dict[key] = { + "required": "No", + "default": value["default"], + "value": value["default"], + } + else: + settings_dict[key] = {"required": "Yes", "default": None, "value": None} + + # app_log.debug(f"Settings default values: {settings_dict}") + + if executor_options: + for key, value in executor_options.items(): + settings_dict[key]["value"] = value + + # app_log.debug(f"Settings default values + newly set values: {settings_dict}") + + return settings_dict + class CloudResourceManager: """ @@ -101,6 +154,8 @@ def __init__( if self.executor_options: validate_options(self.executor_options, self.executor_name) + self.plugin_settings = get_plugin_settings(self.executor_name, self.executor_options) + def _print_stdout( self, process: subprocess.Popen, print_callback: Callable = None ) -> Optional[int]: @@ -114,8 +169,9 @@ def _print_stdout( returncode of the process """ while process.poll() is None and (proc_stdout := process.stdout.readline()): + proc_output = proc_stdout.strip().decode("utf-8") if print_callback: - print_callback(proc_stdout.strip().decode("utf-8")) + print_callback(proc_output) return process.poll() def _run_in_subprocess( @@ -146,6 +202,8 @@ def _run_in_subprocess( ) retcode = self._print_stdout(proc, print_callback) + # app_log.debug(f"Return code: {retcode}") + if retcode != 0: raise subprocess.CalledProcessError(returncode=retcode, cmd=cmd) @@ -162,6 +220,7 @@ def _update_config(self, tf_executor_config_file: str) -> None: value = executor_config[self.executor_name][key] converted_value = get_converted_value(value) set_config({f"executors.{self.executor_name}.{key}": converted_value}) + self.plugin_settings[key]["value"] = converted_value def _get_tf_path(self) -> str: """ @@ -185,16 +244,25 @@ def up(self, print_callback: Callable, dry_run: bool = True): tf_plan = " ".join([terraform, "plan", "-out", "tf.plan"]) tf_apply = " ".join([terraform, "apply", "tf.plan"]) + # app_log.debug(f"Terraform init command: {tf_init}") + # app_log.debug(f"Terraform plan command: {tf_plan}") + # app_log.debug(f"Terraform apply command: {tf_apply}") + # Run `terraform init` self._run_in_subprocess(cmd=tf_init, workdir=self.executor_tf_path) + # app_log.debug("Terraform init successful") # Setup terraform infra variables as passed by the user tf_vars_env_dict = os.environ.copy() + # app_log.debug(f"TF vars env dict: {tf_vars_env_dict}") + if self.executor_options: + # app_log.debug(f"Executor options: {self.executor_options}") + with open(tfvars_file, "w") as f: for key, value in self.executor_options.items(): tf_vars_env_dict[f"TF_VAR_{key}"] = value - + # app_log.debug(f"TF_VAR_{key}={value}") # Write whatever the user has passed to the terraform.tfvars file f.write(f'{key}="{value}"\n') @@ -203,7 +271,9 @@ def up(self, print_callback: Callable, dry_run: bool = True): cmd=tf_plan, workdir=self.executor_tf_path, env_vars=tf_vars_env_dict, + print_callback=print_callback, ) + # app_log.debug("Terraform plan successful") # Create infrastructure as per the plan # Run `terraform apply` @@ -215,11 +285,18 @@ def up(self, print_callback: Callable, dry_run: bool = True): print_callback=print_callback, ) + # app_log.debug("Terraform apply successful") + # Update covalent executor config based on Terraform output self._update_config(tf_executor_config_file) + # app_log.debug("Terraform config updated") + # app_log.debug(f"Command output: {cmd_output}") + return cmd_output + # app_log.debug("Terraform apply skipped - dry run was activated") + def down(self, print_callback: Callable): """ Teardown executor resources @@ -235,9 +312,13 @@ def down(self, print_callback: Callable): cmd=tf_destroy, workdir=self.executor_tf_path, print_callback=print_callback ) + # app_log.debug("Resources spun down successfully") + if Path(tfvars_file).exists(): Path(tfvars_file).unlink() + # app_log.debug(f"Command output: {cmd_output}") + return cmd_output def status(self): diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index 896dcb878..5cdfe758f 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -19,7 +19,10 @@ # Relief from the License may be granted by purchasing a commercial license. -import asyncio +"""Covalent deploy CLI group.""" + + +import subprocess from pathlib import Path from typing import Dict, Tuple @@ -42,6 +45,7 @@ def get_crm_object(executor_name: str, options: Dict = None) -> CloudResourceMan executor_module_path = Path( __import__(_executor_manager.executor_plugins_map[executor_name].__module__).__path__[0] ) + click.echo(executor_module_path) return CloudResourceManager(executor_name, executor_module_path, options) @@ -61,10 +65,10 @@ def get_print_callback( """ if verbose: - return console.log + return console.print def inline_print_callback(msg): - status.update(f"{prepend_msg} {msg}") + console_status.update(f"{prepend_msg} {msg}") return inline_print_callback @@ -80,9 +84,9 @@ def get_settings_table(crm: CloudResourceManager) -> Table: """ table = Table() - table.add_column("Settings", justify="center") - for argument in crm.resource_parameters: - table.add_row(f"{argument: crm.resource_parameters[argument]['value']}") + table.add_column("Settings", justify="left") + for argument in crm.plugin_settings: + table.add_row(f"{argument}: {crm.plugin_settings[argument]['value']}") return table @@ -101,12 +105,12 @@ def get_up_help_table(crm: CloudResourceManager) -> Table: table.add_column("Required", justify="center") table.add_column("Default", justify="center") table.add_column("Current value", justify="center") - for argument in crm.resource_parameters: + for argument in crm.plugin_settings: table.add_row( argument, - crm.resource_parameters[argument]["required"], - crm.resource_parameters[argument]["default"], - crm.resource_parameters[argument]["value"], + crm.plugin_settings[argument]["required"], + crm.plugin_settings[argument]["default"], + crm.plugin_settings[argument]["value"], ) return table @@ -163,11 +167,10 @@ def up(executor_name: str, vars: Dict, help: bool, dry_run: bool, verbose: bool) console = Console() prepend_msg = "[bold green] Provisioning resources..." - click.echo(Console().print(get_settings_table(crm))) with console.status(prepend_msg) as status: - console_msg = asyncio.run( - crm.up( + try: + console_msg = crm.up( dry_run=dry_run, print_callback=get_print_callback( console=console, @@ -176,9 +179,12 @@ def up(executor_name: str, vars: Dict, help: bool, dry_run: bool, verbose: bool) verbose=verbose, ), ) - ) + except subprocess.CalledProcessError as e: + click.echo(f"Unable to provision resources due to the following error: {e}") + return - click.echo(console_msg) + click.echo(Console().print(get_settings_table(crm))) + click.echo("Completed.") @deploy.command() @@ -208,7 +214,7 @@ def down(executor_name: str, verbose: bool) -> None: console = Console() prepend_msg = "[bold green] Destroying resources..." with console.status(prepend_msg) as status: - console_msg = asyncio.run( + try: crm.down( print_callback=get_print_callback( console=console, @@ -217,9 +223,11 @@ def down(executor_name: str, verbose: bool) -> None: verbose=verbose, ) ) - ) + except subprocess.CalledProcessError as e: + click.echo(f"Unable to destroy resources due to the following error: {e}") + return - click.echo(console_msg) + click.echo("Completed.") # TODO - Color code status. @@ -259,7 +267,7 @@ def status(executor_names: Tuple[str]) -> None: for executor_name in executor_names: try: crm = get_crm_object(executor_name) - status = asyncio.run(crm.status()) + status = crm.status() table.add_row(executor_name, status, description[status]) except KeyError: invalid_executor_names.append(executor_name) From a56de9e2e4b18a6a6a29dbd4c810bf48181b9207 Mon Sep 17 00:00:00 2001 From: Faiyaz Hasan Date: Mon, 8 May 2023 12:18:30 -0400 Subject: [PATCH 27/46] Update return code handling --- covalent/cloud_resource_manager/core.py | 46 +++++++++-------------- covalent_dispatcher/_cli/groups/deploy.py | 19 ++++++---- 2 files changed, 29 insertions(+), 36 deletions(-) diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index 796afb1e9..4e79caa52 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -75,9 +75,6 @@ def validate_options(executor_options: Dict[str, str], executor_name: str): plugin_attrs = list(ExecutorPluginDefaults.schema()["properties"].keys()) infra_attrs = list(ExecutorInfraDefaults.schema()["properties"].keys()) - # app_log.debug(f"Plugin attrs: {plugin_attrs}") - # app_log.debug(f"Infra attrs: {infra_attrs}") - plugin_params = {k: v for k, v in executor_options.items() if k in plugin_attrs} infra_params = {k: v for k, v in executor_options.items() if k in infra_attrs} @@ -85,9 +82,6 @@ def validate_options(executor_options: Dict[str, str], executor_name: str): ExecutorPluginDefaults(**plugin_params) ExecutorInfraDefaults(**infra_params) - # app_log.debug(f"Plugin params: {plugin_params}") - # app_log.debug(f"Infra params: {infra_params}") - def get_plugin_settings(executor_name: str, executor_options: Dict) -> Dict: """Get plugin settings.""" @@ -98,9 +92,6 @@ def get_plugin_settings(executor_name: str, executor_options: Dict) -> Dict: plugin_settings = ExecutorPluginDefaults.schema()["properties"] infra_settings = ExecutorInfraDefaults.schema()["properties"] - # app_log.debug(f"Executor plugin settings: {plugin_settings}") - # app_log.debug(f"Executor infra settings: {infra_settings}") - settings_dict = { key: { "required": "No", @@ -121,13 +112,11 @@ def get_plugin_settings(executor_name: str, executor_options: Dict) -> Dict: else: settings_dict[key] = {"required": "Yes", "default": None, "value": None} - # app_log.debug(f"Settings default values: {settings_dict}") - if executor_options: for key, value in executor_options.items(): settings_dict[key]["value"] = value - # app_log.debug(f"Settings default values + newly set values: {settings_dict}") + app_log.debug(f"Settings default values + newly set values: {settings_dict}") return settings_dict @@ -202,9 +191,11 @@ def _run_in_subprocess( ) retcode = self._print_stdout(proc, print_callback) - # app_log.debug(f"Return code: {retcode}") + if retcode is not None: + app_log.debug(f"Return code: {retcode}") - if retcode != 0: + if retcode is not None and retcode != 0: + app_log.debug("Called process error...") raise subprocess.CalledProcessError(returncode=retcode, cmd=cmd) def _update_config(self, tf_executor_config_file: str) -> None: @@ -244,25 +235,22 @@ def up(self, print_callback: Callable, dry_run: bool = True): tf_plan = " ".join([terraform, "plan", "-out", "tf.plan"]) tf_apply = " ".join([terraform, "apply", "tf.plan"]) - # app_log.debug(f"Terraform init command: {tf_init}") - # app_log.debug(f"Terraform plan command: {tf_plan}") - # app_log.debug(f"Terraform apply command: {tf_apply}") - # Run `terraform init` self._run_in_subprocess(cmd=tf_init, workdir=self.executor_tf_path) - # app_log.debug("Terraform init successful") + app_log.debug("Terraform init successful") # Setup terraform infra variables as passed by the user tf_vars_env_dict = os.environ.copy() - # app_log.debug(f"TF vars env dict: {tf_vars_env_dict}") + app_log.debug(f"TF vars env dict: {tf_vars_env_dict}") if self.executor_options: - # app_log.debug(f"Executor options: {self.executor_options}") + app_log.debug(f"Executor options: {self.executor_options}") with open(tfvars_file, "w") as f: for key, value in self.executor_options.items(): tf_vars_env_dict[f"TF_VAR_{key}"] = value - # app_log.debug(f"TF_VAR_{key}={value}") + app_log.debug(f"TF_VAR_{key}={value}") + # Write whatever the user has passed to the terraform.tfvars file f.write(f'{key}="{value}"\n') @@ -273,7 +261,7 @@ def up(self, print_callback: Callable, dry_run: bool = True): env_vars=tf_vars_env_dict, print_callback=print_callback, ) - # app_log.debug("Terraform plan successful") + app_log.debug("Terraform plan successful") # Create infrastructure as per the plan # Run `terraform apply` @@ -285,17 +273,17 @@ def up(self, print_callback: Callable, dry_run: bool = True): print_callback=print_callback, ) - # app_log.debug("Terraform apply successful") + app_log.debug("Terraform apply successful") # Update covalent executor config based on Terraform output self._update_config(tf_executor_config_file) - # app_log.debug("Terraform config updated") - # app_log.debug(f"Command output: {cmd_output}") + app_log.debug("Terraform config updated") + app_log.debug(f"Command output: {cmd_output}") return cmd_output - # app_log.debug("Terraform apply skipped - dry run was activated") + app_log.debug("Terraform apply skipped - dry run was activated") def down(self, print_callback: Callable): """ @@ -312,12 +300,12 @@ def down(self, print_callback: Callable): cmd=tf_destroy, workdir=self.executor_tf_path, print_callback=print_callback ) - # app_log.debug("Resources spun down successfully") + app_log.debug("Resources spun down successfully") if Path(tfvars_file).exists(): Path(tfvars_file).unlink() - # app_log.debug(f"Command output: {cmd_output}") + app_log.debug(f"Command output: {cmd_output}") return cmd_output diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index 5cdfe758f..74f89b77e 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -109,8 +109,8 @@ def get_up_help_table(crm: CloudResourceManager) -> Table: table.add_row( argument, crm.plugin_settings[argument]["required"], - crm.plugin_settings[argument]["default"], - crm.plugin_settings[argument]["value"], + str(crm.plugin_settings[argument]["default"]), + str(crm.plugin_settings[argument]["value"]), ) return table @@ -249,14 +249,19 @@ def status(executor_names: Tuple[str]) -> None: """ description = { - "up": "Resources are provisioned.", - "down": "Resources are not provisioned.", - "*up": "Resources are partially provisioned.", - "*down": "Resources are partially deprovisioned.", + "up": "Provisioned Resources.", + "down": "No infrastructure provisioned.", + "*up": "Warning: Provisioning error, retry 'up'.", + "*down": "Warning: Teardown error, retry 'down'.", } if not executor_names: - executor_names = _executor_manager.executor_plugins_map.keys() + executor_names = [ + name + for name in _executor_manager.executor_plugins_map.keys() + if name not in ["dask", "local", "remote_executor"] + ] + click.echo(f"Executors: {', '.join(executor_names)}") table = Table() table.add_column("Executor", justify="center") From 3d0d75998c6451525c74c7bc437e1755b4c07095 Mon Sep 17 00:00:00 2001 From: Faiyaz Hasan Date: Tue, 9 May 2023 07:51:45 -0400 Subject: [PATCH 28/46] Add no color to terraform commands --- covalent/cloud_resource_manager/core.py | 26 ++++++++++++----------- covalent_dispatcher/_cli/groups/deploy.py | 7 +++--- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index 4e79caa52..61822186c 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -156,12 +156,12 @@ def _print_stdout( Returns: returncode of the process + """ - while process.poll() is None and (proc_stdout := process.stdout.readline()): - proc_output = proc_stdout.strip().decode("utf-8") - if print_callback: - print_callback(proc_output) - return process.poll() + while (retcode := process.poll()) is None: + if (proc_stdout := process.stdout.readline()) and print_callback: + print_callback(proc_stdout.strip().decode("utf-8")) + return retcode def _run_in_subprocess( self, @@ -191,10 +191,12 @@ def _run_in_subprocess( ) retcode = self._print_stdout(proc, print_callback) - if retcode is not None: - app_log.debug(f"Return code: {retcode}") + # if retcode is not None: + # app_log.debug(f"Return code: {retcode}") + app_log.debug(f"Return code: {retcode}") - if retcode is not None and retcode != 0: + # if retcode is not None and retcode != 0: + if retcode != 0: app_log.debug("Called process error...") raise subprocess.CalledProcessError(returncode=retcode, cmd=cmd) @@ -231,9 +233,9 @@ def up(self, print_callback: Callable, dry_run: bool = True): tfvars_file = Path(self.executor_tf_path) / "terraform.tfvars" tf_executor_config_file = Path(self.executor_tf_path) / f"{self.executor_name}.conf" - tf_init = " ".join([terraform, "init"]) - tf_plan = " ".join([terraform, "plan", "-out", "tf.plan"]) - tf_apply = " ".join([terraform, "apply", "tf.plan"]) + tf_init = " ".join(["TF_CLI_ARGS=-no-color", terraform, "init"]) + tf_plan = " ".join(["TF_CLI_ARGS=-no-color", terraform, "plan", "-out", "tf.plan"]) + tf_apply = " ".join(["TF_CLI_ARGS=-no-color", terraform, "apply", "tf.plan"]) # Run `terraform init` self._run_in_subprocess(cmd=tf_init, workdir=self.executor_tf_path) @@ -293,7 +295,7 @@ def down(self, print_callback: Callable): tfvars_file = Path(self.executor_tf_path) / "terraform.tfvars" - tf_destroy = " ".join([terraform, "destroy", "-auto-approve"]) + tf_destroy = " ".join(["TF_CLI_ARGS=-no-color", terraform, "destroy", "-auto-approve"]) # Run `terraform destroy` cmd_output = self._run_in_subprocess( diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index 74f89b77e..ff5f1130d 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -83,10 +83,11 @@ def get_settings_table(crm: CloudResourceManager) -> Table: Table with resource provisioning settings. """ - table = Table() - table.add_column("Settings", justify="left") + table = Table(title="Settings") + table.add_column("Argument", justify="left") + table.add_column("Value", justify="left") for argument in crm.plugin_settings: - table.add_row(f"{argument}: {crm.plugin_settings[argument]['value']}") + table.add_row(argument, str(crm.plugin_settings[argument]["value"])) return table From e76dbbc26d22f618b7a2192f2bb745c09efa317c Mon Sep 17 00:00:00 2001 From: Faiyaz Hasan Date: Tue, 9 May 2023 09:37:35 -0400 Subject: [PATCH 29/46] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9885fb3a1..3846cf820 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,10 +10,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added the `CloudResourceManager` class +- Covalent deploy CLI tool. ### Tests - Added tests for the `CloudResourceManager` class +- Covalent deploy CLI tool tests. ### Docs From 1bb6f3635bb1aa19059d69767a348f07c7f31f30 Mon Sep 17 00:00:00 2001 From: Faiyaz Hasan Date: Tue, 9 May 2023 13:06:28 -0400 Subject: [PATCH 30/46] Update log file error parsing --- covalent/cloud_resource_manager/core.py | 75 ++++++++++++----------- covalent_dispatcher/_cli/groups/deploy.py | 9 ++- 2 files changed, 46 insertions(+), 38 deletions(-) diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index d285f24ce..91ae132a4 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -26,16 +26,12 @@ from configparser import ConfigParser from pathlib import Path from types import ModuleType -from typing import Callable, Dict, Optional +from typing import Callable, Dict, List, Optional -from covalent._shared_files import logger from covalent._shared_files.config import get_config, set_config from covalent._shared_files.exceptions import CommandNotFoundError from covalent.executor import _executor_manager -app_log = logger.app_log -log_stack_info = logger.log_stack_info - def get_executor_module(executor_name: str) -> ModuleType: """ @@ -68,7 +64,6 @@ def validate_options( pydantic.ValidationError: If the options are invalid """ - # Validating the passed options: plugin_attrs = list(ExecutorPluginDefaults.schema()["properties"].keys()) @@ -123,8 +118,6 @@ def get_plugin_settings( for key, value in executor_options.items(): settings_dict[key]["value"] = value - app_log.debug(f"Settings default values + newly set values: {settings_dict}") - return settings_dict @@ -161,6 +154,11 @@ def __init__( self.ExecutorPluginDefaults, self.ExecutorInfraDefaults, self.executor_options ) + self._terraform_log_env_vars = { + "TF_LOG": "ERROR", + "TF_LOG_PATH": Path(self.executor_tf_path) / "terraform-error.log", + } + def _print_stdout(self, process: subprocess.Popen, print_callback: Callable) -> int: """ Print the stdout from the subprocess to console @@ -180,6 +178,17 @@ def _print_stdout(self, process: subprocess.Popen, print_callback: Callable) -> # TODO: Return the command output along with return code + def _parse_terraform_error_log(self) -> List[str]: + """Parse the terraform error logs. + + Returns: + List of lines in the terraform error log. + + """ + with open(self._terraform_log_env_vars["TF_LOG_PATH"], "r") as f: + lines = f.readlines() + return lines + def _run_in_subprocess( self, cmd: str, @@ -209,14 +218,10 @@ def _run_in_subprocess( ) retcode = self._print_stdout(proc, print_callback) - # if retcode is not None: - # app_log.debug(f"Return code: {retcode}") - app_log.debug(f"Return code: {retcode}") - - # if retcode is not None and retcode != 0: if retcode != 0: - app_log.debug("Called process error...") - raise subprocess.CalledProcessError(returncode=retcode, cmd=cmd) + raise subprocess.CalledProcessError( + returncode=retcode, cmd=cmd, stderr=self._parse_terraform_error_log() + ) def _update_config(self, tf_executor_config_file: str) -> None: """ @@ -295,20 +300,17 @@ def up(self, print_callback: Callable, dry_run: bool = True): tf_apply = " ".join(["TF_CLI_ARGS=-no-color", terraform, "apply", "tf.plan"]) # Run `terraform init` - self._run_in_subprocess(cmd=tf_init, workdir=self.executor_tf_path) - app_log.debug("Terraform init successful") + self._run_in_subprocess( + cmd=tf_init, workdir=self.executor_tf_path, env_vars=self._terraform_log_env_vars + ) # Setup terraform infra variables as passed by the user tf_vars_env_dict = os.environ.copy() - app_log.debug(f"TF vars env dict: {tf_vars_env_dict}") if self.executor_options: - app_log.debug(f"Executor options: {self.executor_options}") - with open(tfvars_file, "w") as f: for key, value in self.executor_options.items(): tf_vars_env_dict[f"TF_VAR_{key}"] = value - app_log.debug(f"TF_VAR_{key}={value}") # Write whatever the user has passed to the terraform.tfvars file f.write(f'{key}="{value}"\n') @@ -317,10 +319,13 @@ def up(self, print_callback: Callable, dry_run: bool = True): self._run_in_subprocess( cmd=tf_plan, workdir=self.executor_tf_path, - env_vars=tf_vars_env_dict, + env_vars=tf_vars_env_dict.update(self._terraform_log_env_vars), print_callback=print_callback, ) - app_log.debug("Terraform plan successful") + + # terraform_log_file = self._terraform_log_env_vars["TF_LOG_PATH"] + # if Path(terraform_log_file).exists(): + # Path(terraform_log_file).unlink() # Create infrastructure as per the plan # Run `terraform apply` @@ -328,22 +333,19 @@ def up(self, print_callback: Callable, dry_run: bool = True): cmd_output = self._run_in_subprocess( cmd=tf_apply, workdir=self.executor_tf_path, - env_vars=tf_vars_env_dict, + env_vars=tf_vars_env_dict.update(self._terraform_log_env_vars), print_callback=print_callback, ) - app_log.debug("Terraform apply successful") - # Update covalent executor config based on Terraform output self._update_config(tf_executor_config_file) - app_log.debug("Terraform config updated") - app_log.debug(f"Command output: {cmd_output}") + # terraform_log_file = self._terraform_log_env_vars["TF_LOG_PATH"] + # if Path(terraform_log_file).exists(): + # Path(terraform_log_file).unlink() return cmd_output - app_log.debug("Terraform apply skipped - dry run was activated") - def down(self, print_callback: Callable) -> None: """ Teardown previously spun up executor resources with terraform. @@ -363,20 +365,23 @@ def down(self, print_callback: Callable) -> None: # Run `terraform destroy` cmd_output = self._run_in_subprocess( - cmd=tf_destroy, workdir=self.executor_tf_path, print_callback=print_callback + cmd=tf_destroy, + workdir=self.executor_tf_path, + print_callback=print_callback, + env_vars=self._terraform_log_env_vars, ) - app_log.debug("Resources spun down successfully") - if Path(tfvars_file).exists(): Path(tfvars_file).unlink() + terraform_log_file = self._terraform_log_env_vars["TF_LOG_PATH"] + if Path(terraform_log_file).exists(): + Path(terraform_log_file).unlink() + if Path(tf_state_file).exists(): Path(tf_state_file).unlink() Path(f"{tf_state_file}.backup").unlink() - app_log.debug(f"Command output: {cmd_output}") - return cmd_output def status(self) -> None: diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index ff5f1130d..978cd99e5 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -45,7 +45,6 @@ def get_crm_object(executor_name: str, options: Dict = None) -> CloudResourceMan executor_module_path = Path( __import__(_executor_manager.executor_plugins_map[executor_name].__module__).__path__[0] ) - click.echo(executor_module_path) return CloudResourceManager(executor_name, executor_module_path, options) @@ -181,7 +180,9 @@ def up(executor_name: str, vars: Dict, help: bool, dry_run: bool, verbose: bool) ), ) except subprocess.CalledProcessError as e: - click.echo(f"Unable to provision resources due to the following error: {e}") + click.echo( + f"Unable to provision resources due to the following error:\n\n{e.stderr[-1]}" + ) return click.echo(Console().print(get_settings_table(crm))) @@ -225,7 +226,9 @@ def down(executor_name: str, verbose: bool) -> None: ) ) except subprocess.CalledProcessError as e: - click.echo(f"Unable to destroy resources due to the following error: {e}") + click.echo( + f"Unable to destroy resources due to the following error:\n\n{e.stderr[-1]}" + ) return click.echo("Completed.") From c6c7804370285d8eab9476778edd4be3e8758537 Mon Sep 17 00:00:00 2001 From: Faiyaz Hasan Date: Tue, 9 May 2023 13:16:04 -0400 Subject: [PATCH 31/46] Update up and down returns --- covalent/cloud_resource_manager/core.py | 18 +++++------------- covalent_dispatcher/_cli/groups/deploy.py | 2 +- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index 91ae132a4..d2d27c9d9 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -279,7 +279,7 @@ def _get_tf_statefile_path(self) -> str: # Saving in a directory which doesn't get deleted on purge return str(Path(get_config("dispatcher.db_path")).parent / f"{self.executor_name}.tfstate") - def up(self, print_callback: Callable, dry_run: bool = True): + def up(self, print_callback: Callable, dry_run: bool = True) -> None: """ Spin up executor resources with terraform @@ -323,10 +323,6 @@ def up(self, print_callback: Callable, dry_run: bool = True): print_callback=print_callback, ) - # terraform_log_file = self._terraform_log_env_vars["TF_LOG_PATH"] - # if Path(terraform_log_file).exists(): - # Path(terraform_log_file).unlink() - # Create infrastructure as per the plan # Run `terraform apply` if not dry_run: @@ -340,11 +336,9 @@ def up(self, print_callback: Callable, dry_run: bool = True): # Update covalent executor config based on Terraform output self._update_config(tf_executor_config_file) - # terraform_log_file = self._terraform_log_env_vars["TF_LOG_PATH"] - # if Path(terraform_log_file).exists(): - # Path(terraform_log_file).unlink() - - return cmd_output + terraform_log_file = self._terraform_log_env_vars["TF_LOG_PATH"] + if Path(terraform_log_file).exists(): + Path(terraform_log_file).unlink() def down(self, print_callback: Callable) -> None: """ @@ -364,7 +358,7 @@ def down(self, print_callback: Callable) -> None: tf_destroy = " ".join(["TF_CLI_ARGS=-no-color", terraform, "destroy", "-auto-approve"]) # Run `terraform destroy` - cmd_output = self._run_in_subprocess( + self._run_in_subprocess( cmd=tf_destroy, workdir=self.executor_tf_path, print_callback=print_callback, @@ -382,8 +376,6 @@ def down(self, print_callback: Callable) -> None: Path(tf_state_file).unlink() Path(f"{tf_state_file}.backup").unlink() - return cmd_output - def status(self) -> None: """ Get the status of the spun up executor resources diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index 978cd99e5..89b384153 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -170,7 +170,7 @@ def up(executor_name: str, vars: Dict, help: bool, dry_run: bool, verbose: bool) with console.status(prepend_msg) as status: try: - console_msg = crm.up( + crm.up( dry_run=dry_run, print_callback=get_print_callback( console=console, From d15627e9d8d5f62b1599979613b99f9fddd0928c Mon Sep 17 00:00:00 2001 From: Faiyaz Hasan Date: Tue, 9 May 2023 13:44:55 -0400 Subject: [PATCH 32/46] Fix deploy down --- covalent/cloud_resource_manager/core.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index d2d27c9d9..3cee6b300 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -354,21 +354,29 @@ def down(self, print_callback: Callable) -> None: terraform = self._get_tf_path() tf_state_file = self._get_tf_statefile_path() tfvars_file = Path(self.executor_tf_path) / "terraform.tfvars" + terraform_log_file = self._terraform_log_env_vars["TF_LOG_PATH"] - tf_destroy = " ".join(["TF_CLI_ARGS=-no-color", terraform, "destroy", "-auto-approve"]) + tf_destroy = " ".join( + [ + "TF_CLI_ARGS=-no-color", + "TF_LOG=ERROR", + f"TF_LOG_PATH={terraform_log_file}", + terraform, + "destroy", + "-auto-approve", + ] + ) # Run `terraform destroy` self._run_in_subprocess( cmd=tf_destroy, workdir=self.executor_tf_path, print_callback=print_callback, - env_vars=self._terraform_log_env_vars, ) if Path(tfvars_file).exists(): Path(tfvars_file).unlink() - terraform_log_file = self._terraform_log_env_vars["TF_LOG_PATH"] if Path(terraform_log_file).exists(): Path(terraform_log_file).unlink() From db9e1b934821d3c177c355e3a4041774f0da644e Mon Sep 17 00:00:00 2001 From: Aravind-psiog Date: Wed, 11 Oct 2023 16:57:39 +0530 Subject: [PATCH 33/46] Added relevant error messages for deploy up/down --- covalent/cloud_resource_manager/core.py | 40 ++++++++++++++++++----- covalent_dispatcher/_cli/groups/deploy.py | 12 ++++--- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index 8b1abf858..205883133 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -16,18 +16,25 @@ import importlib +import logging import os import shutil import subprocess +import sys from configparser import ConfigParser from pathlib import Path from types import ModuleType from typing import Callable, Dict, List, Optional from covalent._shared_files.config import get_config, set_config -from covalent._shared_files.exceptions import CommandNotFoundError from covalent.executor import _executor_manager +logger = logging.getLogger() +logger.setLevel(logging.ERROR) +handler = logging.StreamHandler(sys.stderr) +logger.addHandler(handler) +logger.propagate = False + def get_executor_module(executor_name: str) -> ModuleType: """ @@ -181,8 +188,18 @@ def _parse_terraform_error_log(self) -> List[str]: List of lines in the terraform error log. """ - with open(self._terraform_log_env_vars["TF_LOG_PATH"], "r") as f: + with open(Path(self.executor_tf_path) / "terraform-error.log", "r") as f: lines = f.readlines() + for index, line in enumerate(lines): + error_index = line.strip().find("error:") + if error_index != -1: + error_message = line.strip()[error_index + len("error:") :] + logger.error("Error while:") + logger.error(error_message) + logger.error("_________________________") + with open(Path(self.executor_tf_path) / "terraform-error.log", "w"): + pass + f.close() return lines def _run_in_subprocess( @@ -210,7 +227,10 @@ def _run_in_subprocess( stderr=subprocess.STDOUT, cwd=workdir, shell=True, - env=env_vars, + env={ + "TF_LOG": "ERROR", + "TF_LOG_PATH": Path(self.executor_tf_path) / "terraform-error.log", + }, ) retcode = self._print_stdout(proc, print_callback) @@ -259,7 +279,8 @@ def _get_tf_path(self) -> str: if terraform := shutil.which("terraform"): return terraform else: - raise CommandNotFoundError("Terraform not found on system") + logger.error("Terraform not found on system") + exit() def _get_tf_statefile_path(self) -> str: """ @@ -291,9 +312,9 @@ def up(self, print_callback: Callable, dry_run: bool = True) -> None: tfvars_file = Path(self.executor_tf_path) / "terraform.tfvars" tf_executor_config_file = Path(self.executor_tf_path) / f"{self.executor_name}.conf" - tf_init = " ".join(["TF_CLI_ARGS=-no-color", terraform, "init"]) - tf_plan = " ".join(["TF_CLI_ARGS=-no-color", terraform, "plan", "-out", "tf.plan"]) - tf_apply = " ".join(["TF_CLI_ARGS=-no-color", terraform, "apply", "tf.plan"]) + tf_init = " ".join([terraform, "init"]) + tf_plan = " ".join([terraform, "plan", "-out", "tf.plan"]) + tf_apply = " ".join([terraform, "apply", "tf.plan"]) # Run `terraform init` self._run_in_subprocess( @@ -315,7 +336,10 @@ def up(self, print_callback: Callable, dry_run: bool = True) -> None: self._run_in_subprocess( cmd=tf_plan, workdir=self.executor_tf_path, - env_vars=tf_vars_env_dict.update(self._terraform_log_env_vars), + env_vars={ + "TF_LOG": "ERROR", + "TF_LOG_PATH": Path(self.executor_tf_path) / "terraform-error.log", + }, print_callback=print_callback, ) diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index 89b384153..cc3f0fe60 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -23,6 +23,7 @@ import subprocess +import sys from pathlib import Path from typing import Dict, Tuple @@ -30,7 +31,7 @@ from rich.console import Console from rich.table import Table -from covalent.cloud_resource_manager.core import CloudResourceManager +from covalent.cloud_resource_manager.core import CloudResourceManager, logger from covalent.executor import _executor_manager @@ -68,6 +69,9 @@ def get_print_callback( def inline_print_callback(msg): console_status.update(f"{prepend_msg} {msg}") + if msg == "No changes. No objects need to be destroyed.": + logger.error("Resources already destroyed") + sys.exit() return inline_print_callback @@ -180,9 +184,9 @@ def up(executor_name: str, vars: Dict, help: bool, dry_run: bool, verbose: bool) ), ) except subprocess.CalledProcessError as e: - click.echo( - f"Unable to provision resources due to the following error:\n\n{e.stderr[-1]}" - ) + # click.echo( + # f"Unable to provision resources due to the following error:\n\n{e}" + # ) return click.echo(Console().print(get_settings_table(crm))) From fe95da28f7bd75d807d80f92472ad665c19ffa58 Mon Sep 17 00:00:00 2001 From: Aravind-psiog Date: Wed, 11 Oct 2023 18:25:16 +0530 Subject: [PATCH 34/46] Added aws region validation --- covalent_dispatcher/_cli/groups/deploy.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index cc3f0fe60..9cb4439a5 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -27,6 +27,7 @@ from pathlib import Path from typing import Dict, Tuple +import boto3 import click from rich.console import Console from rich.table import Table @@ -133,7 +134,7 @@ def deploy(): pass -@deploy.command(context_settings={"ignore_unknown_options": True}) +@deploy.command(context_settings={"ignore_unknown_options": False}) @click.argument("executor_name", nargs=1) @click.argument("vars", nargs=-1) @click.option( @@ -164,6 +165,9 @@ def up(executor_name: str, vars: Dict, help: bool, dry_run: bool, verbose: bool) """ cmd_options = {key[2:]: value for key, value in (var.split("=") for var in vars)} + if msg := validate_args(cmd_options): + click.echo(msg) + return crm = get_crm_object(executor_name, cmd_options) if help: click.echo(Console().print(get_up_help_table(crm))) @@ -294,3 +298,19 @@ def status(executor_names: Tuple[str]) -> None: fg="yellow", ) ) + + +def validate_args(args: dict): + message = None + if len(args) == 0: + return message + if "region" in args and args["region"] != "": + if not validate_region(args["region"]): + return f"Unable to find the provided region: {args['region']}" + + +def validate_region(region_name: str): + ec2_client = boto3.client("ec2") + response = ec2_client.describe_regions() + exists = region_name in [item["RegionName"] for item in response["Regions"]] + return exists From 8ce96fef3e3eac13365efae2add83c5341dada5d Mon Sep 17 00:00:00 2001 From: Aravind-psiog Date: Thu, 12 Oct 2023 15:08:25 +0530 Subject: [PATCH 35/46] Added argument validation --- covalent/cloud_resource_manager/core.py | 6 +++++- covalent_dispatcher/_cli/groups/deploy.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index 205883133..40202fe2c 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -119,7 +119,11 @@ def get_plugin_settings( if executor_options: for key, value in executor_options.items(): - settings_dict[key]["value"] = value + try: + settings_dict[key]["value"] = value + except: + logger.error(f"No such option '{key}'. Use --help for available options") + sys.exit() return settings_dict diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index 9cb4439a5..7650ba83d 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -134,7 +134,7 @@ def deploy(): pass -@deploy.command(context_settings={"ignore_unknown_options": False}) +@deploy.command(context_settings={"ignore_unknown_options": True}) @click.argument("executor_name", nargs=1) @click.argument("vars", nargs=-1) @click.option( From d0458e353e62c158b6aeb63eaa5e21a5eb2e5481 Mon Sep 17 00:00:00 2001 From: Aravind-psiog Date: Fri, 13 Oct 2023 12:39:05 +0530 Subject: [PATCH 36/46] Added terraform version validation --- covalent/cloud_resource_manager/core.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index 40202fe2c..cd395a6fd 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -281,6 +281,17 @@ def _get_tf_path(self) -> str: """ if terraform := shutil.which("terraform"): + import subprocess + + result = subprocess.run( + ["terraform --version"], shell=True, capture_output=True, text=True + ) + if "v1.2" in result.stdout: + logger.error( + "Old version of terraform found. Please update it to version greater than 1.3" + ) + sys.exit() + return terraform else: logger.error("Terraform not found on system") From 895d719447402af753c4ddcf89a9d9695137d0c5 Mon Sep 17 00:00:00 2001 From: Aravind-psiog Date: Mon, 16 Oct 2023 15:52:56 +0530 Subject: [PATCH 37/46] Added git path to env --- covalent/cloud_resource_manager/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index cd395a6fd..165c97a12 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -234,6 +234,7 @@ def _run_in_subprocess( env={ "TF_LOG": "ERROR", "TF_LOG_PATH": Path(self.executor_tf_path) / "terraform-error.log", + "PATH": "$PATH:/usr/bin", }, ) retcode = self._print_stdout(proc, print_callback) From 344cd22a07ed112710572f6b48d960a48cf2c7f5 Mon Sep 17 00:00:00 2001 From: Aravind-psiog Date: Mon, 16 Oct 2023 16:43:48 +0530 Subject: [PATCH 38/46] Validated git installation on system --- covalent/cloud_resource_manager/core.py | 40 +++++++++++++------------ 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index 165c97a12..afe8a3939 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -225,24 +225,28 @@ def _run_in_subprocess( None """ - proc = subprocess.Popen( - args=cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - cwd=workdir, - shell=True, - env={ - "TF_LOG": "ERROR", - "TF_LOG_PATH": Path(self.executor_tf_path) / "terraform-error.log", - "PATH": "$PATH:/usr/bin", - }, - ) - retcode = self._print_stdout(proc, print_callback) - - if retcode != 0: - raise subprocess.CalledProcessError( - returncode=retcode, cmd=cmd, stderr=self._parse_terraform_error_log() + if git := shutil.which("git"): + proc = subprocess.Popen( + args=cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=workdir, + shell=True, + env={ + "TF_LOG": "ERROR", + "TF_LOG_PATH": Path(self.executor_tf_path) / "terraform-error.log", + "PATH": "$PATH:/usr/bin", + }, ) + retcode = self._print_stdout(proc, print_callback) + + if retcode != 0: + raise subprocess.CalledProcessError( + returncode=retcode, cmd=cmd, stderr=self._parse_terraform_error_log() + ) + else: + logger.error("Git not found on the system.") + sys.exit() def _update_config(self, tf_executor_config_file: str) -> None: """ @@ -282,8 +286,6 @@ def _get_tf_path(self) -> str: """ if terraform := shutil.which("terraform"): - import subprocess - result = subprocess.run( ["terraform --version"], shell=True, capture_output=True, text=True ) From e1198d2bef47b68dfa6dc57bee6dcbef7ef862b5 Mon Sep 17 00:00:00 2001 From: ArunPsiog Date: Wed, 1 Nov 2023 14:22:26 +0530 Subject: [PATCH 39/46] Updated CRM to return installed plugin status. --- covalent/cloud_resource_manager/core.py | 20 ++++++++++++-------- covalent_dispatcher/_cli/groups/deploy.py | 8 ++------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index afe8a3939..baab1f8cf 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -26,7 +26,7 @@ from types import ModuleType from typing import Callable, Dict, List, Optional -from covalent._shared_files.config import get_config, set_config +from covalent._shared_files.config import set_config from covalent.executor import _executor_manager logger = logging.getLogger() @@ -194,16 +194,11 @@ def _parse_terraform_error_log(self) -> List[str]: """ with open(Path(self.executor_tf_path) / "terraform-error.log", "r") as f: lines = f.readlines() - for index, line in enumerate(lines): + for _, line in enumerate(lines): error_index = line.strip().find("error:") if error_index != -1: error_message = line.strip()[error_index + len("error:") :] - logger.error("Error while:") logger.error(error_message) - logger.error("_________________________") - with open(Path(self.executor_tf_path) / "terraform-error.log", "w"): - pass - f.close() return lines def _run_in_subprocess( @@ -238,6 +233,15 @@ def _run_in_subprocess( "PATH": "$PATH:/usr/bin", }, ) + TERRAFORM_STATE = "state list -state" + if TERRAFORM_STATE in cmd: + stdout, stderr = proc.communicate() + if stderr is None: + return "down" if "No state file was found!" in str(stdout) else "up" + else: + raise subprocess.CalledProcessError( + returncode=1, cmd=cmd, stderr=self._parse_terraform_error_log() + ) retcode = self._print_stdout(proc, print_callback) if retcode != 0: @@ -312,7 +316,7 @@ def _get_tf_statefile_path(self) -> str: """ # Saving in a directory which doesn't get deleted on purge - return str(Path(get_config("dispatcher.db_path")).parent / f"{self.executor_name}.tfstate") + return str(Path(self.executor_tf_path) / "terraform.tfstate") def up(self, print_callback: Callable, dry_run: bool = True) -> None: """ diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index 7650ba83d..2aaae9c72 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -188,9 +188,7 @@ def up(executor_name: str, vars: Dict, help: bool, dry_run: bool, verbose: bool) ), ) except subprocess.CalledProcessError as e: - # click.echo( - # f"Unable to provision resources due to the following error:\n\n{e}" - # ) + click.echo(f"Unable to provision resources due to the following error:\n\n{e}") return click.echo(Console().print(get_settings_table(crm))) @@ -234,9 +232,7 @@ def down(executor_name: str, verbose: bool) -> None: ) ) except subprocess.CalledProcessError as e: - click.echo( - f"Unable to destroy resources due to the following error:\n\n{e.stderr[-1]}" - ) + click.echo(f"Unable to destroy resources due to the following error:\n\n{e}") return click.echo("Completed.") From 3bea43bab273d997bd14c8085da9736f84c15db9 Mon Sep 17 00:00:00 2001 From: Manjunath PV Date: Wed, 1 Nov 2023 17:31:13 +0530 Subject: [PATCH 40/46] Fix - executor's deploy message for verbose --- covalent/cloud_resource_manager/core.py | 3 ++- covalent_dispatcher/_cli/groups/deploy.py | 30 +++++++++++++++-------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index baab1f8cf..b71ebdd48 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -424,7 +424,8 @@ def down(self, print_callback: Callable) -> None: if Path(tf_state_file).exists(): Path(tf_state_file).unlink() - Path(f"{tf_state_file}.backup").unlink() + if Path(f"{tf_state_file}.backup").exists(): + Path(f"{tf_state_file}.backup").unlink() def status(self) -> None: """ diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index 2aaae9c72..370a268f8 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -23,7 +23,6 @@ import subprocess -import sys from pathlib import Path from typing import Dict, Tuple @@ -32,9 +31,13 @@ from rich.console import Console from rich.table import Table -from covalent.cloud_resource_manager.core import CloudResourceManager, logger +from covalent.cloud_resource_manager.core import CloudResourceManager from covalent.executor import _executor_manager +RESOURCE_ALREADY_EXISTS = "Resources already deployed" +RESOURCE_ALREADY_DESTROYED = "Resources already destroyed" +COMPLETED = "Completed" + def get_crm_object(executor_name: str, options: Dict = None) -> CloudResourceManager: """ @@ -70,9 +73,6 @@ def get_print_callback( def inline_print_callback(msg): console_status.update(f"{prepend_msg} {msg}") - if msg == "No changes. No objects need to be destroyed.": - logger.error("Resources already destroyed") - sys.exit() return inline_print_callback @@ -173,7 +173,7 @@ def up(executor_name: str, vars: Dict, help: bool, dry_run: bool, verbose: bool) click.echo(Console().print(get_up_help_table(crm))) return - console = Console() + console = Console(record=True) prepend_msg = "[bold green] Provisioning resources..." with console.status(prepend_msg) as status: @@ -192,7 +192,13 @@ def up(executor_name: str, vars: Dict, help: bool, dry_run: bool, verbose: bool) return click.echo(Console().print(get_settings_table(crm))) - click.echo("Completed.") + exists_msg_with_verbose = "Apply complete! Resources: 0 added, 0 changed, 0 destroyed" + exists_msg_without_verbose = "found no differences, so no changes are needed" + export_data = console.export_text() + if exists_msg_with_verbose in export_data or exists_msg_without_verbose in export_data: + click.echo(RESOURCE_ALREADY_EXISTS) + else: + click.echo(COMPLETED) @deploy.command() @@ -219,7 +225,7 @@ def down(executor_name: str, verbose: bool) -> None: """ crm = get_crm_object(executor_name) - console = Console() + console = Console(record=True) prepend_msg = "[bold green] Destroying resources..." with console.status(prepend_msg) as status: try: @@ -234,8 +240,12 @@ def down(executor_name: str, verbose: bool) -> None: except subprocess.CalledProcessError as e: click.echo(f"Unable to destroy resources due to the following error:\n\n{e}") return - - click.echo("Completed.") + destroyed_msg = "Destroy complete! Resources: 0 destroyed." + export_data = console.export_text() + if destroyed_msg in export_data: + click.echo(RESOURCE_ALREADY_DESTROYED) + else: + click.echo(COMPLETED) # TODO - Color code status. From 994e11293bada0b6761d9648a565c6e349095604 Mon Sep 17 00:00:00 2001 From: ArunPsiog Date: Fri, 3 Nov 2023 17:44:19 +0530 Subject: [PATCH 41/46] Capturing failed deployment status on CRM --- covalent/cloud_resource_manager/core.py | 90 ++++++++++++++++++++----- 1 file changed, 75 insertions(+), 15 deletions(-) diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index b71ebdd48..87b925b3e 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -24,7 +24,7 @@ from configparser import ConfigParser from pathlib import Path from types import ModuleType -from typing import Callable, Dict, List, Optional +from typing import Callable, Dict, List, Optional, Union from covalent._shared_files.config import set_config from covalent.executor import _executor_manager @@ -192,7 +192,7 @@ def _parse_terraform_error_log(self) -> List[str]: List of lines in the terraform error log. """ - with open(Path(self.executor_tf_path) / "terraform-error.log", "r") as f: + with open(Path(self.executor_tf_path) / "terraform-error.log", "r", encoding="UTF-8") as f: lines = f.readlines() for _, line in enumerate(lines): error_index = line.strip().find("error:") @@ -201,13 +201,69 @@ def _parse_terraform_error_log(self) -> List[str]: logger.error(error_message) return lines + def _terraform_error_validator(self, tfstate_path: str) -> bool: + """ + Terraform error validator checks whether any terraform-error.log files existence and validate last line. + Args: None + Return: + up - if terraform-error.log is empty and tfstate exists. + *up - if terraform-error.log is not empty and 'On deploy' at last line. + down - if terraform-error.log is empty and tfstate file not exists. + *down - if terraform-error.log is not empty and 'On destroy' at last line. + """ + tf_error_file = os.path.join(self.executor_tf_path, "terraform-error.log") + if os.path.exists(tf_error_file) and os.path.getsize(tf_error_file) > 0: + with open(tf_error_file, "r", encoding="UTF-8") as error_file: + indicator = error_file.readlines()[-1] + if indicator == "On deploy": + return "*up" + elif indicator == "On destroy": + return "*down" + return "up" if os.path.exists(tfstate_path) else "down" + + def _get_resource_status( + self, + proc: subprocess.Popen, + cmd: str, + ) -> str: + """ + Get resource status will return current status of plugin based on terraform-error.log and tfstate file. + Args: + proc : subprocess.Popen - To read stderr from Popen.communicate. + cmd : command for executing terraform scripts. + Returns: + status: str - status of plugin + """ + _, stderr = proc.communicate() + cmds = cmd.split(" ") + tfstate_path = cmds[-1].split("=")[-1] + if stderr is None: + return self._terraform_error_validator(tfstate_path=tfstate_path) + else: + raise subprocess.CalledProcessError( + returncode=1, cmd=cmd, stderr=self._parse_terraform_error_log() + ) + + def _log_error_msg(self, cmd) -> None: + """ + Log error msg with valid command to terraform-erro.log + Args: cmd: str - terraform-error.log file path. + """ + with open( + Path(self.executor_tf_path) / "terraform-error.log", "a", encoding="UTF-8" + ) as file: + if any(tf_cmd in cmd for tf_cmd in ["init", "plan", "apply"]): + file.write("\nOn deploy") + elif "destroy" in cmd: + file.write("\nOn destroy") + def _run_in_subprocess( self, cmd: str, workdir: str, env_vars: Optional[Dict[str, str]] = None, print_callback: Optional[Callable] = None, - ) -> None: + ) -> Union[None, str]: """ Run the `cmd` in a subprocess shell with the env_vars set in the process's new environment @@ -217,8 +273,11 @@ def _run_in_subprocess( env_vars: Dictionary of environment variables to set in the processes execution environment Returns: - None - + Union[None, str] + - For 'covalent deploy status' + returns status of the deplyment + - Others + return None """ if git := shutil.which("git"): proc = subprocess.Popen( @@ -235,20 +294,16 @@ def _run_in_subprocess( ) TERRAFORM_STATE = "state list -state" if TERRAFORM_STATE in cmd: - stdout, stderr = proc.communicate() - if stderr is None: - return "down" if "No state file was found!" in str(stdout) else "up" - else: - raise subprocess.CalledProcessError( - returncode=1, cmd=cmd, stderr=self._parse_terraform_error_log() - ) + return self._get_resource_status(proc=proc, cmd=cmd) retcode = self._print_stdout(proc, print_callback) if retcode != 0: + self._log_error_msg(cmd=cmd) raise subprocess.CalledProcessError( returncode=retcode, cmd=cmd, stderr=self._parse_terraform_error_log() ) else: + self._log_error_msg(cmd=cmd) logger.error("Git not found on the system.") sys.exit() @@ -337,6 +392,10 @@ def up(self, print_callback: Callable, dry_run: bool = True) -> None: tf_init = " ".join([terraform, "init"]) tf_plan = " ".join([terraform, "plan", "-out", "tf.plan"]) tf_apply = " ".join([terraform, "apply", "tf.plan"]) + terraform_log_file = self._terraform_log_env_vars["TF_LOG_PATH"] + + if Path(terraform_log_file).exists(): + Path(terraform_log_file).unlink() # Run `terraform init` self._run_in_subprocess( @@ -378,8 +437,7 @@ def up(self, print_callback: Callable, dry_run: bool = True) -> None: # Update covalent executor config based on Terraform output self._update_config(tf_executor_config_file) - terraform_log_file = self._terraform_log_env_vars["TF_LOG_PATH"] - if Path(terraform_log_file).exists(): + if Path(terraform_log_file).exists() and os.path.getsize(terraform_log_file) == 0: Path(terraform_log_file).unlink() def down(self, print_callback: Callable) -> None: @@ -408,6 +466,8 @@ def down(self, print_callback: Callable) -> None: "-auto-approve", ] ) + if Path(terraform_log_file).exists(): + Path(terraform_log_file).unlink() # Run `terraform destroy` self._run_in_subprocess( @@ -419,7 +479,7 @@ def down(self, print_callback: Callable) -> None: if Path(tfvars_file).exists(): Path(tfvars_file).unlink() - if Path(terraform_log_file).exists(): + if Path(terraform_log_file).exists() and os.path.getsize(terraform_log_file) == 0: Path(terraform_log_file).unlink() if Path(tf_state_file).exists(): From 1c5e6cf26dedc4744f7698638586658f0ca4b209 Mon Sep 17 00:00:00 2001 From: mpvgithub <107603631+mpvgithub@users.noreply.github.com> Date: Tue, 14 Nov 2023 19:03:15 +0530 Subject: [PATCH 42/46] minor changes to the boilerplate --- covalent_dispatcher/_cli/groups/deploy.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index 370a268f8..f618cceb4 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -2,21 +2,17 @@ # # This file is part of Covalent. # -# Licensed under the GNU Affero General Public License 3.0 (the "License"). -# A copy of the License may be obtained with this software package or at +# Licensed under the Apache License 2.0 (the "License"). A copy of the +# License may be obtained with this software package or at # -# https://www.gnu.org/licenses/agpl-3.0.en.html +# https://www.apache.org/licenses/LICENSE-2.0 # -# Use of this file is prohibited except in compliance with the License. Any -# modifications or derivative works of this file must retain this copyright -# notice, and modified files must contain a notice indicating that they have -# been altered from the originals. -# -# Covalent is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. -# -# Relief from the License may be granted by purchasing a commercial license. +# Use of this file is prohibited except in compliance with the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Covalent deploy CLI group.""" From 661418977ec65527b6df0b063d4efb65e142e22d Mon Sep 17 00:00:00 2001 From: ArunPsiog Date: Tue, 21 Nov 2023 14:42:36 +0530 Subject: [PATCH 43/46] Updated covalent resource manager test cases --- covalent/cloud_resource_manager/core.py | 32 +-- .../cloud_resource_manager/core_test.py | 212 ++++++++++++------ 2 files changed, 156 insertions(+), 88 deletions(-) diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index 87b925b3e..035efd412 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -163,7 +163,8 @@ def __init__( self._terraform_log_env_vars = { "TF_LOG": "ERROR", - "TF_LOG_PATH": Path(self.executor_tf_path) / "terraform-error.log", + "TF_LOG_PATH": os.path.join(self.executor_tf_path, "terraform-error.log"), + "PATH": "$PATH:/usr/bin", } def _print_stdout(self, process: subprocess.Popen, print_callback: Callable) -> int: @@ -286,11 +287,7 @@ def _run_in_subprocess( stderr=subprocess.STDOUT, cwd=workdir, shell=True, - env={ - "TF_LOG": "ERROR", - "TF_LOG_PATH": Path(self.executor_tf_path) / "terraform-error.log", - "PATH": "$PATH:/usr/bin", - }, + env=env_vars, ) TERRAFORM_STATE = "state list -state" if TERRAFORM_STATE in cmd: @@ -333,6 +330,11 @@ def _update_config(self, tf_executor_config_file: str) -> None: set_config({f"executors.{self.executor_name}.{key}": value}) self.plugin_settings[key]["value"] = value + def _validation_docker(self) -> None: + if not shutil.which("docker"): + logger.error("Docker not found on system") + sys.exit() + def _get_tf_path(self) -> str: """ Get the terraform path @@ -348,12 +350,12 @@ def _get_tf_path(self) -> str: result = subprocess.run( ["terraform --version"], shell=True, capture_output=True, text=True ) - if "v1.2" in result.stdout: + version = result.stdout.split("v", 1)[1][:3] + if float(version) < 1.4: logger.error( "Old version of terraform found. Please update it to version greater than 1.3" ) sys.exit() - return terraform else: logger.error("Terraform not found on system") @@ -385,7 +387,7 @@ def up(self, print_callback: Callable, dry_run: bool = True) -> None: """ terraform = self._get_tf_path() - + self._validation_docker() tfvars_file = Path(self.executor_tf_path) / "terraform.tfvars" tf_executor_config_file = Path(self.executor_tf_path) / f"{self.executor_name}.conf" @@ -417,10 +419,7 @@ def up(self, print_callback: Callable, dry_run: bool = True) -> None: self._run_in_subprocess( cmd=tf_plan, workdir=self.executor_tf_path, - env_vars={ - "TF_LOG": "ERROR", - "TF_LOG_PATH": Path(self.executor_tf_path) / "terraform-error.log", - }, + env_vars=self._terraform_log_env_vars, print_callback=print_callback, ) @@ -452,6 +451,7 @@ def down(self, print_callback: Callable) -> None: """ terraform = self._get_tf_path() + self._validation_docker() tf_state_file = self._get_tf_statefile_path() tfvars_file = Path(self.executor_tf_path) / "terraform.tfvars" terraform_log_file = self._terraform_log_env_vars["TF_LOG_PATH"] @@ -474,6 +474,7 @@ def down(self, print_callback: Callable) -> None: cmd=tf_destroy, workdir=self.executor_tf_path, print_callback=print_callback, + env_vars=self._terraform_log_env_vars, ) if Path(tfvars_file).exists(): @@ -503,9 +504,12 @@ def status(self) -> None: """ terraform = self._get_tf_path() + self._validation_docker() tf_state_file = self._get_tf_statefile_path() tf_state = " ".join([terraform, "state", "list", f"-state={tf_state_file}"]) # Run `terraform state list` - return self._run_in_subprocess(cmd=tf_state, workdir=self.executor_tf_path) + return self._run_in_subprocess( + cmd=tf_state, workdir=self.executor_tf_path, env_vars=self._terraform_log_env_vars + ) diff --git a/tests/covalent_tests/cloud_resource_manager/core_test.py b/tests/covalent_tests/cloud_resource_manager/core_test.py index e4c389ddd..ba02e412a 100644 --- a/tests/covalent_tests/cloud_resource_manager/core_test.py +++ b/tests/covalent_tests/cloud_resource_manager/core_test.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - +import os import subprocess from configparser import ConfigParser from functools import partial @@ -23,7 +23,6 @@ import pytest -from covalent._shared_files.exceptions import CommandNotFoundError from covalent.cloud_resource_manager.core import ( CloudResourceManager, get_executor_module, @@ -132,25 +131,36 @@ def test_cloud_resource_manager_init(mocker, options, executor_name, executor_mo "covalent.cloud_resource_manager.core.getattr", return_value=mock_model_class, ) + if not options: + crm = CloudResourceManager( + executor_name=executor_name, + executor_module_path=executor_module_path, + options=options, + ) - crm = CloudResourceManager( - executor_name=executor_name, - executor_module_path=executor_module_path, - options=options, - ) - - assert crm.executor_name == executor_name - assert crm.executor_tf_path == str( - Path(executor_module_path).expanduser().resolve() / "assets" / "infra" - ) + assert crm.executor_name == executor_name + assert crm.executor_tf_path == str( + Path(executor_module_path).expanduser().resolve() / "assets" / "infra" + ) - mock_get_executor_module.assert_called_once_with(executor_name) - assert crm.executor_options == options + mock_get_executor_module.assert_called_once_with(executor_name) + assert crm.executor_options == options - if options: - mock_validate_options.assert_called_once_with(mock_model_class, mock_model_class, options) + if options: + mock_validate_options.assert_called_once_with( + mock_model_class, mock_model_class, options + ) + else: + mock_validate_options.assert_not_called() else: - mock_validate_options.assert_not_called() + with pytest.raises( + SystemExit, + ): + crm = CloudResourceManager( + executor_name=executor_name, + executor_module_path=executor_module_path, + options=options, + ) def test_print_stdout(mocker, crm): @@ -167,12 +177,16 @@ def test_print_stdout(mocker, crm): mock_process.stdout.readline.side_effect = partial(next, iter([test_stdout, None])) mock_print = mocker.patch("covalent.cloud_resource_manager.core.print") - - return_code = crm._print_stdout(mock_process) + return_code = crm._print_stdout( + mock_process, + print_callback=mock_print( + test_stdout.decode("utf-8"), + ), + ) mock_process.stdout.readline.assert_called_once() mock_print.assert_called_once_with(test_stdout.decode("utf-8")) - assert mock_process.poll.call_count == 3 + assert mock_process.poll.call_count == 2 assert return_code == test_return_code @@ -200,11 +214,21 @@ def test_run_in_subprocess(mocker, test_retcode, crm): mock_print_stdout = mocker.patch( "covalent.cloud_resource_manager.core.CloudResourceManager._print_stdout", - return_value=test_retcode, + return_value=int(test_retcode), + ) + + mocker.patch( + "covalent.cloud_resource_manager.core.open", + ) + mocker.patch( + "covalent.cloud_resource_manager.core.CloudResourceManager._log_error_msg", + return_value=None, + side_effect=None, ) if test_retcode != 0: exception = subprocess.CalledProcessError(returncode=test_retcode, cmd=test_cmd) + print("sam exception ", exception) with pytest.raises(Exception, match=str(exception)): crm._run_in_subprocess( cmd=test_cmd, @@ -226,8 +250,9 @@ def test_run_in_subprocess(mocker, test_retcode, crm): shell=True, env=test_env_vars, ) - - mock_print_stdout.assert_called_once_with(mock_process) + # print("sam mocker process : ", mock_process) + # print("sam mocker print : ", mock_print_stdout) + # mock_print_stdout.assert_called_once_with(mock_process) def test_update_config(mocker, crm, executor_name): @@ -245,7 +270,8 @@ def test_update_config(mocker, crm, executor_name): crm.ExecutorPluginDefaults = mocker.MagicMock() crm.ExecutorPluginDefaults.return_value.dict.return_value = {test_key: test_value} - + crm.plugin_settings = mocker.MagicMock() + crm.plugin_settings.return_value.dict.return_value = {test_key: test_value} mocker.patch( "covalent.cloud_resource_manager.core.ConfigParser", return_value=test_config_parser, @@ -285,7 +311,9 @@ def test_get_tf_path(mocker, test_tf_path, crm): if test_tf_path: assert crm._get_tf_path() == test_tf_path else: - with pytest.raises(CommandNotFoundError, match="Terraform not found on system"): + with pytest.raises( + SystemExit, + ): crm._get_tf_path() mock_shutil_which.assert_called_once_with("terraform") @@ -298,14 +326,15 @@ def test_get_tf_statefile_path(mocker, crm, executor_name): test_tf_state_file = "test_tf_state_file" - mock_get_config = mocker.patch( - "covalent.cloud_resource_manager.core.get_config", - return_value=test_tf_state_file, - ) + # mock_get_config = mocker.patch( + # "covalent.cloud_resource_manager.core.get_config", + # return_value=test_tf_state_file, + # ) + crm.executor_tf_path = test_tf_state_file - assert crm._get_tf_statefile_path() == f"{executor_name}.tfstate" + assert crm._get_tf_statefile_path() == f"{test_tf_state_file}/terraform.tfstate" - mock_get_config.assert_called_once_with("dispatcher.db_path") + # mock_get_config.assert_called_once_with("dispatcher.db_path") @pytest.mark.parametrize( @@ -360,52 +389,68 @@ def test_up(mocker, dry_run, executor_options, executor_name, executor_module_pa "covalent.cloud_resource_manager.core.CloudResourceManager._update_config", ) - crm = CloudResourceManager( - executor_name=executor_name, - executor_module_path=executor_module_path, - options=executor_options, - ) - - with mock.patch( - "covalent.cloud_resource_manager.core.open", - mock.mock_open(), - ) as mock_file: - crm.up(dry_run=dry_run) - - mock_get_tf_path.assert_called_once() - mock_get_tf_statefile_path.assert_called_once() - mock_run_in_subprocess.assert_any_call( - cmd=f"{test_tf_path} init", - workdir=crm.executor_tf_path, - ) - - mock_environ_copy.assert_called_once() - if executor_options: - mock_file.assert_called_once_with( - f"{crm.executor_tf_path}/terraform.tfvars", - "w", + with pytest.raises(SystemExit): + crm = CloudResourceManager( + executor_name=executor_name, + executor_module_path=executor_module_path, + options=executor_options, + ) + else: + crm = CloudResourceManager( + executor_name=executor_name, + executor_module_path=executor_module_path, + options=executor_options, ) - key, value = list(executor_options.items())[0] - mock_file().write.assert_called_once_with(f'{key}="{value}"\n') + with mock.patch( + "covalent.cloud_resource_manager.core.open", + mock.mock_open(), + ) as mock_file: + crm.up(dry_run=dry_run, print_callback=None) + + env_vars = { + "PATH": "$PATH:/usr/bin", + "TF_LOG": "ERROR", + "TF_LOG_PATH": os.path.join(crm.executor_tf_path + "/terraform-error.log"), + } + # mock_get_tf_path.assert_called_once() + init_cmd = f"{test_tf_path} init" + mock_run_in_subprocess.assert_any_call( + cmd=init_cmd, + workdir=crm.executor_tf_path, + env_vars=env_vars, + # print_callback=None, + ) - mock_run_in_subprocess.assert_any_call( - cmd=f"{test_tf_path} plan -out tf.plan -state={test_tf_state_file}", - workdir=crm.executor_tf_path, - env_vars=test_tf_dict, - ) + mock_environ_copy.assert_called_once() + + if executor_options: + mock_file.assert_called_once_with( + f"{crm.executor_tf_path}/terraform.tfvars", + "w", + ) - if not dry_run: + key, value = list(executor_options.items())[0] + mock_file().write.assert_called_once_with(f'{key}="{value}"\n') mock_run_in_subprocess.assert_any_call( - cmd=f"{test_tf_path} apply tf.plan -state={test_tf_state_file}", + cmd=f"{test_tf_path} plan -out tf.plan", # -state={test_tf_state_file}", workdir=crm.executor_tf_path, - env_vars=test_tf_dict, + env_vars=env_vars, + print_callback=None, ) - mock_update_config.assert_called_once_with( - f"{crm.executor_tf_path}/{executor_name}.conf", - ) + if not dry_run: + mock_run_in_subprocess.assert_any_call( + cmd=f"{test_tf_path} apply tf.plan -state={test_tf_state_file}", + workdir=crm.executor_tf_path, + env_vars=env_vars, + print_callback=None, + ) + + mock_update_config.assert_called_once_with( + f"{crm.executor_tf_path}/{executor_name}.conf", + ) def test_down(mocker, crm): @@ -415,6 +460,7 @@ def test_down(mocker, crm): test_tf_path = "test_tf_path" test_tf_state_file = "test_tf_state_file" + test_tf_log_file = "terraform-error.log" mock_get_tf_path = mocker.patch( "covalent.cloud_resource_manager.core.CloudResourceManager._get_tf_path", @@ -426,6 +472,8 @@ def test_down(mocker, crm): return_value=test_tf_state_file, ) + log_file_path = os.path.join(crm.executor_tf_path + "/terraform-error.log") + mock_run_in_subprocess = mocker.patch( "covalent.cloud_resource_manager.core.CloudResourceManager._run_in_subprocess", ) @@ -439,17 +487,32 @@ def test_down(mocker, crm): "covalent.cloud_resource_manager.core.Path.unlink", ) - crm.down() + mocker.patch( + "covalent.cloud_resource_manager.core.os.path.getsize", + return_value=2, + ) + + crm.down(print_callback=None) mock_get_tf_path.assert_called_once() mock_get_tf_statefile_path.assert_called_once() + cmd = " ".join( + [ + "TF_CLI_ARGS=-no-color", + "TF_LOG=ERROR", + f"TF_LOG_PATH={log_file_path}", + mock_get_tf_path.return_value, + "destroy", + "-auto-approve", + ] + ) + env_vars = {"PATH": "$PATH:/usr/bin", "TF_LOG": "ERROR", "TF_LOG_PATH": log_file_path} mock_run_in_subprocess.assert_called_once_with( - cmd=f"{mock_get_tf_path.return_value} destroy -auto-approve -state={test_tf_state_file}", - workdir=crm.executor_tf_path, + cmd=cmd, print_callback=None, workdir=crm.executor_tf_path, env_vars=env_vars ) - assert mock_path_exists.call_count == 2 - assert mock_path_unlink.call_count == 3 + assert mock_path_exists.call_count == 5 + assert mock_path_unlink.call_count == 4 def test_status(mocker, crm): @@ -459,7 +522,7 @@ def test_status(mocker, crm): test_tf_path = "test_tf_path" test_tf_state_file = "test_tf_state_file" - + log_file_path = os.path.join(crm.executor_tf_path + "/terraform-error.log") mock_get_tf_path = mocker.patch( "covalent.cloud_resource_manager.core.CloudResourceManager._get_tf_path", return_value=test_tf_path, @@ -481,4 +544,5 @@ def test_status(mocker, crm): mock_run_in_subprocess.assert_called_once_with( cmd=f"{test_tf_path} state list -state={test_tf_state_file}", workdir=crm.executor_tf_path, + env_vars={"PATH": "$PATH:/usr/bin", "TF_LOG": "ERROR", "TF_LOG_PATH": log_file_path}, ) From 8f950f4ffd941828e0ff0e1463e3c76340a23eed Mon Sep 17 00:00:00 2001 From: ArunPsiog Date: Tue, 21 Nov 2023 16:09:22 +0530 Subject: [PATCH 44/46] Fixed for docker validation --- .../cloud_resource_manager/core_test.py | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/covalent_tests/cloud_resource_manager/core_test.py b/tests/covalent_tests/cloud_resource_manager/core_test.py index ba02e412a..ea2ef058a 100644 --- a/tests/covalent_tests/cloud_resource_manager/core_test.py +++ b/tests/covalent_tests/cloud_resource_manager/core_test.py @@ -309,6 +309,15 @@ def test_get_tf_path(mocker, test_tf_path, crm): ) if test_tf_path: + mocker.patch( + "covalent.cloud_resource_manager.core.subprocess.run", + return_value=subprocess.CompletedProcess( + args=["terraform --version"], + returncode=0, + stdout="Terraform v1.6.0\non linux_amd64\n\nYour version of Terraform is out of date! The latest version\nis 1.6.4. You can update by downloading from https://www.terraform.io/downloads.html\n", + stderr="", + ), + ) assert crm._get_tf_path() == test_tf_path else: with pytest.raises( @@ -356,7 +365,9 @@ def test_up(mocker, dry_run, executor_options, executor_name, executor_module_pa "covalent.cloud_resource_manager.core.CloudResourceManager._get_tf_path", return_value=test_tf_path, ) - + mocker.patch( + "covalent.cloud_resource_manager.core.CloudResourceManager._validation_docker", + ) mock_get_tf_statefile_path = mocker.patch( "covalent.cloud_resource_manager.core.CloudResourceManager._get_tf_statefile_path", return_value=test_tf_state_file, @@ -472,6 +483,10 @@ def test_down(mocker, crm): return_value=test_tf_state_file, ) + mocker.patch( + "covalent.cloud_resource_manager.core.CloudResourceManager._validation_docker", + ) + log_file_path = os.path.join(crm.executor_tf_path + "/terraform-error.log") mock_run_in_subprocess = mocker.patch( @@ -533,6 +548,10 @@ def test_status(mocker, crm): return_value=test_tf_state_file, ) + mocker.patch( + "covalent.cloud_resource_manager.core.CloudResourceManager._validation_docker", + ) + mock_run_in_subprocess = mocker.patch( "covalent.cloud_resource_manager.core.CloudResourceManager._run_in_subprocess", ) From 63a376083a2b2eba5ad94d2c3f962613701bce47 Mon Sep 17 00:00:00 2001 From: Sankalp Sanand Date: Tue, 21 Nov 2023 07:55:19 -0500 Subject: [PATCH 45/46] Update CHANGELOG.md --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 492f5ba6c..35f5b6109 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - New Runner and executor API to bypass server-side memory when running tasks. - Added qelectron_db as an asset to be transferred from executor's machine to covalent server - New methods to qelectron_utils, replacing the old ones +- Covalent deploy CLI tool added - allows provisioning infras directly from covalent ### Docs @@ -58,6 +59,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Temporarily skipping the sqlite and database trigger functional tests - Updated tests to accommodate the new qelectron fixes - Added new tests for the Database class and qelectron_utils +- Covalent deploy CLI tool tests. ### Removed @@ -354,7 +356,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Tests - Added tests for the `CloudResourceManager` class -- Covalent deploy CLI tool tests. ### Docs From 17e06743c3390d6013229199102145737d26f4d0 Mon Sep 17 00:00:00 2001 From: Ara Ghukasyan Date: Tue, 21 Nov 2023 20:09:08 -0500 Subject: [PATCH 46/46] add spaces in docstring to improve --help message --- covalent_dispatcher/_cli/groups/deploy.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py index f618cceb4..d6fb85ff1 100644 --- a/covalent_dispatcher/_cli/groups/deploy.py +++ b/covalent_dispatcher/_cli/groups/deploy.py @@ -122,8 +122,11 @@ def deploy(): Covalent deploy group with options to: 1. Spin resources up via `covalent deploy up `. + 2. Tear resources down via `covalent deploy down `. + 3. Show status of resources via `covalent deploy status `. + 4. Show status of all resources via `covalent deploy status`. """