diff --git a/.github/workflows/e2e-migration.yml b/.github/workflows/e2e-migration.yml new file mode 100644 index 00000000..8cc1e5d3 --- /dev/null +++ b/.github/workflows/e2e-migration.yml @@ -0,0 +1,98 @@ +name: E2E migration testing + +on: + push: + tags: + - v* + branches: + - main + - develop + workflow_dispatch: + +jobs: + e2e-migration-from-0_9_0-to-latest: + runs-on: ${{ matrix.os }} + + strategy: + matrix: + python-ver: ["3.11", "3.12"] + os: [ubuntu-latest] + # os: [self-hosted-arm64, ubuntu-latest] + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-ver }} + + - name: header + run: | + echo -e "\e[1;33m┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\e[0m + \e[1;33m┃\e[0m \e[1;36m Migration from v0.9.0 to latest \e[0m \e[1;33m┃\e[0m + \e[1;33m┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛\e[0m" + + - name: install frappe from v0.9.0 + run: | + cd /tmp + python -m pip install -U git+https://github.com/rtCamp/Frappe-Manager.git@v0.9.0 + + - name: frappe version + run: fm --version + + - name: e2e run + timeout-minutes: 20 + working-directory: test + run: | + ./migration_test.sh oldToNew + + - name: cleanup + if: always() + run: | + python -m pip uninstall -y frappe-manager + sudo rm -rf ~/frappe + + e2e-migration-from-before_latest-to-latest: + runs-on: ${{ matrix.os }} + + strategy: + matrix: + python-ver: ["3.11", "3.12"] + os: [ubuntu-latest] + # os: [self-hosted-arm64, ubuntu-latest] + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-ver }} + + - name: install jq + run: | + sudo apt upgrade -y + sudo apt install -y jq curl + - name: header + run: | + echo -e "\e[1;33m┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\e[0m + \e[1;33m┃\e[0m \e[1;36m Migration from latest~1 to latest \e[0m \e[1;33m┃\e[0m + \e[1;33m┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛\e[0m" + + - name: install frappe previous the latest release + run: | + python -m pip install -U git+https://github.com/rtCamp/Frappe-Manager.git@$(curl --silent https://api.github.com/repos/rtCamp/Frappe-Manager/tags | jq -r '.[1].name') + + - name: frappe version + run: fm --version + + - name: e2e run + timeout-minutes: 30 + working-directory: test + run: | + ./migration_test.sh semiNewToNew + + - name: cleanup + if: always() + run: | + python -m pip uninstall -y frappe-manager + sudo rm -rf ~/frappe diff --git a/.github/workflows/e2e-site.yaml b/.github/workflows/e2e-site.yaml new file mode 100644 index 00000000..e00c0772 --- /dev/null +++ b/.github/workflows/e2e-site.yaml @@ -0,0 +1,54 @@ +name: E2E site testing + +on: + push: + tags: + - v* + branches: + - main + - develop + workflow_dispatch: + +jobs: + e2e-current: + name: e2e current latest branch + runs-on: ${{ matrix.os }} + + strategy: + matrix: + python-ver: ["3.11", "3.12"] + os: [ubuntu-latest] + # os: [self-hosted-arm64, ubuntu-latest] + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-ver }} + + - name: header + run: | + echo -e "\e[1;33m┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\e[0m + \e[1;33m┃\e[0m \e[1;36m E2E Test \e[0m \e[1;33m┃\e[0m + \e[1;33m┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛\e[0m" + + - name: install frappe from current commit + run: | + cd .. + python -m pip install --upgrade ./Frappe-Manager + + - name: frappe version + run: fm --version + + - name: e2e run + timeout-minutes: 20 + working-directory: test + run: | + ./e2e_test.sh + + - name: cleanup + if: always() + run: | + python -m pip uninstall -y frappe-manager + sudo rm -rf ~/frappe diff --git a/frappe_manager/commands.py b/frappe_manager/commands.py index 7628ff3c..dcb3f563 100644 --- a/frappe_manager/commands.py +++ b/frappe_manager/commands.py @@ -25,7 +25,7 @@ from frappe_manager.migration_manager.migration_executor import MigrationExecutor from frappe_manager.site_manager.site import Bench from frappe_manager.site_manager.workers_manager.SiteWorker import BenchWorkers -from frappe_manager.ssl_manager import SUPPORTED_SSL_TYPES +from frappe_manager.ssl_manager import LETSENCRYPT_PREFERRED_CHALLENGE, SUPPORTED_SSL_TYPES from frappe_manager.ssl_manager.certificate import SSLCertificate from frappe_manager.ssl_manager.letsencrypt_certificate import LetsencryptSSLCertificate from frappe_manager.utils.callbacks import ( @@ -105,35 +105,38 @@ def app_callback( fm_config_manager: FMConfigManager = FMConfigManager.import_from_toml() + # docker pull if first_time_install: if not fm_config_manager.root_path.exists(): - richprint.print('🔍 It seems like the first installation. Pulling images... 🖼️') + richprint.print("It seems like the first installation. Pulling docker images...️", "🔍") + completed_status = pull_docker_images() if not completed_status: shutil.rmtree(CLI_DIR) richprint.exit("Aborting. [bold][blue]fm[/blue][/bold] will not be able to work without images. 🖼️") + current_version = Version(get_current_fm_version()) fm_config_manager.version = current_version fm_config_manager.export_to_toml() migrations = MigrationExecutor(fm_config_manager) migration_status = migrations.execute() + if not migration_status: richprint.exit(f"Rollbacked to previous version of fm {migrations.prev_version}.") services_manager: ServicesManager = ServicesManager(verbose=verbose) + services_manager.set_typer_context(ctx) + services_manager.init() + try: - services_manager.entrypoint_checks() + services_manager.entrypoint_checks(start=True) except ServicesNotCreated as e: services_manager.remove_itself() richprint.exit(f"Not able to create services. {e}") - if not services_manager.compose_project.running: - services_manager.are_ports_free() - services_manager.compose_project.start_service() - ctx.obj["services"] = services_manager ctx.obj["verbose"] = verbose ctx.obj['fm_config_manager'] = fm_config_manager @@ -156,6 +159,14 @@ def create( environment: Annotated[ FMBenchEnvType, typer.Option("--environment", "--env", help="Select bench environment type.") ] = FMBenchEnvType.dev, + letsencrypt_preferred_challenge: Annotated[ + Optional[LETSENCRYPT_PREFERRED_CHALLENGE], + typer.Option(help="Select preferred letsencrypt challenge.", show_default=False), + ] = None, + letsencrypt_email: Annotated[ + Optional[str], + typer.Option(help="Specify email for letsencrypt", show_default=False), + ] = None, developer_mode: Annotated[ EnableDisableOptionsEnum, typer.Option(help="Toggle frappe developer mode.") ] = EnableDisableOptionsEnum.disable, @@ -205,18 +216,36 @@ def create( bench_config_path = bench_path / CLI_BENCH_CONFIG_FILE_NAME if ssl == SUPPORTED_SSL_TYPES.le: - if fm_config_manager.le_email == 'dummy@fm.fm': - email = richprint.prompt_ask(prompt='Please enter [bold][green]email[/bold][/green] for Let\'s Encrypt') + if not letsencrypt_preferred_challenge: + if fm_config_manager.letsencrypt.exists: + if letsencrypt_preferred_challenge is None: + letsencrypt_preferred_challenge = LETSENCRYPT_PREFERRED_CHALLENGE.dns01 + + if not letsencrypt_preferred_challenge: + letsencrypt_preferred_challenge = LETSENCRYPT_PREFERRED_CHALLENGE.http01 + + if fm_config_manager.letsencrypt.email == 'dummy@fm.fm' or fm_config_manager.letsencrypt.email is None: + if not letsencrypt_email: + richprint.stop() + raise typer.BadParameter("No email provided, required by certbot.", param_hint='--letsencrypt-email') + else: + email = letsencrypt_email + validate_email(email, check_deliverability=False) - fm_config_manager.le_email = email - fm_config_manager.export_to_toml() - richprint.print("Let's Encrypt email saved to configuration. It will be used automatically from now on.") else: - richprint.print("Using Let's Encrypt email from configuration.") - email = fm_config_manager.le_email - fm_config_manager.export_to_toml() - - ssl_certificate = LetsencryptSSLCertificate(domain=benchname, ssl_type=ssl, email=email) + richprint.print( + "Defaulting to Let's Encrypt email from [blue]fm_config.toml[/blue] since [blue]'--letsencrypt-email'[/blue] is not given." + ) + email = fm_config_manager.letsencrypt.email + + ssl_certificate = LetsencryptSSLCertificate( + domain=benchname, + ssl_type=ssl, + email=email, + preferred_challenge=letsencrypt_preferred_challenge, + api_key=fm_config_manager.letsencrypt.api_key, + api_token=fm_config_manager.letsencrypt.api_token, + ) elif ssl == SUPPORTED_SSL_TYPES.none: ssl_certificate = SSLCertificate(domain=benchname, ssl_type=ssl) @@ -483,9 +512,17 @@ def update( Optional[EnableDisableOptionsEnum], typer.Option("--admin-tools", help="Toggle admin-tools.", show_default=False), ] = None, + letsencrypt_preferred_challenge: Annotated[ + Optional[LETSENCRYPT_PREFERRED_CHALLENGE], + typer.Option(help="Select preferred letsencrypt challenge.", show_default=False), + ] = None, + letsencrypt_email: Annotated[ + Optional[str], + typer.Option(help="Specify email for letsencrypt", show_default=False), + ] = None, environment: Annotated[ Optional[FMBenchEnvType], - typer.Option("--environment", help="Switch bench environment.", show_default=False), + typer.Option("--environment", "--env", help="Switch bench environment.", show_default=False), ] = None, developer_mode: Annotated[ Optional[EnableDisableOptionsEnum], @@ -530,20 +567,38 @@ def update( new_ssl_certificate = SSLCertificate(domain=benchname, ssl_type=SUPPORTED_SSL_TYPES.none) if ssl == SUPPORTED_SSL_TYPES.le: - if fm_config_manager.le_email == 'dummy@fm.fm': - email = richprint.prompt_ask(prompt='Please enter [bold][green]email[/bold][/green] for Let\'s Encrypt') + if not letsencrypt_preferred_challenge: + if fm_config_manager.letsencrypt.exists: + if letsencrypt_preferred_challenge is None: + letsencrypt_preferred_challenge = LETSENCRYPT_PREFERRED_CHALLENGE.dns01 + + if not letsencrypt_preferred_challenge: + letsencrypt_preferred_challenge = LETSENCRYPT_PREFERRED_CHALLENGE.http01 + + if fm_config_manager.letsencrypt.email == 'dummy@fm.fm' or fm_config_manager.letsencrypt.email is None: + if not letsencrypt_email: + richprint.stop() + raise typer.BadParameter( + "No email provided, required by certbot.", param_hint='--letsencrypt-email' + ) + else: + email = letsencrypt_email + validate_email(email, check_deliverability=False) - fm_config_manager.le_email = email - fm_config_manager.export_to_toml() + else: richprint.print( - "Let's Encrypt email saved to configuration. It will be used automatically from now on." + "Defaulting to Let's Encrypt email from [blue]fm_config.toml[/blue] since [blue]'--letsencrypt-email'[/blue] is not given." ) - else: - richprint.print("Using Let's Encrypt email from configuration.") - email = fm_config_manager.le_email - fm_config_manager.export_to_toml() - - new_ssl_certificate = LetsencryptSSLCertificate(domain=benchname, ssl_type=ssl, email=email) + email = fm_config_manager.letsencrypt.email + + new_ssl_certificate = LetsencryptSSLCertificate( + domain=benchname, + ssl_type=ssl, + email=email, + preferred_challenge=letsencrypt_preferred_challenge, + api_key=fm_config_manager.letsencrypt.api_key, + api_token=fm_config_manager.letsencrypt.api_token, + ) richprint.print("Updating Certificate.") bench.update_certificate(new_ssl_certificate) diff --git a/frappe_manager/metadata_manager.py b/frappe_manager/metadata_manager.py index 4b6d92a5..0e959925 100644 --- a/frappe_manager/metadata_manager.py +++ b/frappe_manager/metadata_manager.py @@ -1,21 +1,54 @@ +from typing import Optional from pathlib import Path -from pydantic import BaseModel, EmailStr +from pydantic import BaseModel, EmailStr, Field import tomlkit from frappe_manager.migration_manager.version import Version from frappe_manager import CLI_FM_CONFIG_PATH from frappe_manager.utils.helpers import get_current_fm_version +class FMLetsencryptConfig(BaseModel): + email: Optional[EmailStr] = Field(None, description="Email used by certbot.") + api_token: Optional[str] = Field(None, description="Cloudflare API token used by Certbot.") + api_key: Optional[str] = Field(None, description="Cloudflare Global API Key used by Certbot.") + + @property + def exists(self): + if self.api_token or self.api_key: + return True + + return False + + def get_toml_doc(self): + model_dict = self.model_dump(exclude_none=True) + toml_doc = tomlkit.document() + + for key, value in model_dict.items(): + if isinstance(value, Path): + toml_doc[key] = str(value.absolute()) + else: + toml_doc[key] = value + return toml_doc + + @classmethod + def import_from_toml_doc(cls, toml_doc): + config_object = cls(**toml_doc) + return config_object + + class FMConfigManager(BaseModel): root_path: Path version: Version - le_email: EmailStr + letsencrypt: FMLetsencryptConfig = Field(default=FMLetsencryptConfig()) def export_to_toml(self, path: Path = CLI_FM_CONFIG_PATH) -> bool: exclude = {'root_path'} - if self.le_email == 'dummy@fm.fm': - exclude.add('le_email') + if not self.letsencrypt.email and not self.letsencrypt.api_key and not self.letsencrypt.api_token: + exclude.add('letsencrypt') + else: + if self.letsencrypt.email == 'dummy@fm.fm': + exclude.add('letsencrypt') if self.version < Version('0.13.0'): path = CLI_FM_CONFIG_PATH.parent / '.fm.toml' @@ -46,7 +79,7 @@ def import_from_toml(cls, path: Path = CLI_FM_CONFIG_PATH) -> "FMConfigManager": old_config_path = path.parent / '.fm.toml' input_data['version'] = Version('0.8.3') - input_data['le_email'] = 'dummy@fm.fm' + input_data['letsencrypt'] = FMLetsencryptConfig(email=None, api_key=None, api_token=None) input_data['root_path'] = str(path) if old_config_path.exists(): @@ -55,7 +88,9 @@ def import_from_toml(cls, path: Path = CLI_FM_CONFIG_PATH) -> "FMConfigManager": elif path.exists(): data = tomlkit.parse(path.read_text()) input_data['version'] = Version(data.get('version', get_current_fm_version())) - input_data['le_email'] = data.get('le_email', 'dummy@fm.fm') + input_data['letsencrypt'] = FMLetsencryptConfig( + **data.get('letsencrypt', {'email': None, 'api_key': None, 'api_token': None}) + ) fm_config_instance = cls(**input_data) return fm_config_instance diff --git a/frappe_manager/migration_manager/migrations/migrate_0_13_0.py b/frappe_manager/migration_manager/migrations/migrate_0_13_0.py index 52047f84..207fd1c6 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_13_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_13_0.py @@ -2,6 +2,7 @@ import os import copy from pathlib import Path +import tomlkit from frappe_manager.compose_manager import DockerVolumeMount from frappe_manager.compose_manager.ComposeFile import ComposeFile from frappe_manager.migration_manager.migration_base import MigrationBase @@ -14,9 +15,6 @@ from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.migration_manager.version import Version from frappe_manager import CLI_DIR, CLI_SERVICES_DIRECTORY -from frappe_manager.site_manager.bench_config import BenchConfig, FMBenchEnvType -from frappe_manager.ssl_manager import SUPPORTED_SSL_TYPES -from frappe_manager.ssl_manager.certificate import SSLCertificate from frappe_manager.utils.helpers import get_container_name_prefix from frappe_manager.docker_wrapper.DockerClient import DockerClient from frappe_manager.migration_manager.backup_manager import BackupManager @@ -80,6 +78,7 @@ def migrate_services(self): # rename main config fm_config_path = CLI_DIR / 'fm_config.toml' old_fm_config_path = CLI_DIR / '.fm.toml' + if old_fm_config_path.exists(): old_fm_config_path.rename(fm_config_path) @@ -133,9 +132,6 @@ def migrate_bench_compose(self, bench: MigrationBench): # create bench config frappe = envs.get('frappe', {}) - userid = frappe.get('USERID', os.getuid()) - usergroup = frappe.get('USERGROUP', os.getgid()) - apps_list = frappe.get('APPS_LIST', None) if apps_list: @@ -143,33 +139,18 @@ def migrate_bench_compose(self, bench: MigrationBench): else: apps_list = [] - frappe_branch = frappe.get('FRAPPE_BRANCH', 'version-15') developer_mode = frappe.get('DEVELOPER_MODE', True) - admin_pass = frappe.get('ADMIN_PASS', 'admin') name = frappe.get('SITENAME', bench.name) - mariadb_host = frappe.get('MARIADB_HOST', 'global-db') - mariadb_root_pass = frappe.get('MARIADB_ROOT_PASS', '/run/secrets/db_root_password') - environment_type = frappe.get('ENVIRONMENT', FMBenchEnvType.dev) - ssl_certificate = SSLCertificate(domain=bench.name, ssl_type=SUPPORTED_SSL_TYPES.none) - - # TODO Handle admin tools compose change - bench_config = BenchConfig( - name=name, - userid=userid, - usergroup=usergroup, - apps_list=apps_list, - frappe_branch=frappe_branch, - developer_mode=developer_mode, - admin_tools=True, - admin_pass=admin_pass, - mariadb_host=mariadb_host, - mariadb_root_pass=mariadb_root_pass, - environment_type=environment_type, - root_path=bench_config_path, - ssl=ssl_certificate, - ) - bench_config.export_to_toml(bench_config_path) + bench_config = tomlkit.document() + + bench_config['name'] = name + bench_config['developer_mode'] = developer_mode + bench_config['admin_tools'] = True + bench_config['environment_type'] = 'dev' + + with open(bench_config_path, 'w') as f: + f.write(tomlkit.dumps(bench_config)) images_info = bench.compose_project.compose_file_manager.get_all_images() diff --git a/frappe_manager/migration_manager/migrations/migrate_0_14_0.py b/frappe_manager/migration_manager/migrations/migrate_0_14_0.py new file mode 100644 index 00000000..cdd50511 --- /dev/null +++ b/frappe_manager/migration_manager/migrations/migrate_0_14_0.py @@ -0,0 +1,36 @@ +from pathlib import Path +import tomlkit +from frappe_manager.migration_manager.migration_base import MigrationBase +from frappe_manager.migration_manager.migration_helpers import MigrationBenches, MigrationServicesManager +from frappe_manager.migration_manager.version import Version +from frappe_manager.migration_manager.backup_manager import BackupManager +from frappe_manager.display_manager.DisplayManager import richprint + + +class MigrationV0140(MigrationBase): + version = Version("0.14.0") + + def init(self): + self.cli_dir: Path = Path.home() / 'frappe' + self.benches_dir = self.cli_dir / "sites" + self.backup_manager = BackupManager(str(self.version), self.benches_dir) + self.benches_manager = MigrationBenches(self.benches_dir) + self.services_manager: MigrationServicesManager = MigrationServicesManager( + services_path=self.cli_dir / 'services' + ) + + def migrate_services(self): + richprint.change_head("Migrating [blue]fm_config.toml[/blue] changes") + config_path = self.cli_dir / 'fm_config.toml' + + if config_path.exists(): + data = tomlkit.parse(config_path.read_text()) + email = data.get('le_email', None) + + if email: + data['letsencrypt'] = {'email': email} + del data['le_email'] + + with open(config_path, 'w') as f: + f.write(tomlkit.dumps(data)) + richprint.print("Migrated [blue]fm_config.toml[/blue]") diff --git a/frappe_manager/services_manager/__init__.py b/frappe_manager/services_manager/__init__.py index fc0a03ed..675874ac 100644 --- a/frappe_manager/services_manager/__init__.py +++ b/frappe_manager/services_manager/__init__.py @@ -3,3 +3,4 @@ class ServicesEnum(str, Enum): global_db= "global-db" global_nginx_proxy="global-nginx-proxy" + all="all" diff --git a/frappe_manager/services_manager/commands.py b/frappe_manager/services_manager/commands.py index f9de8422..55779af6 100644 --- a/frappe_manager/services_manager/commands.py +++ b/frappe_manager/services_manager/commands.py @@ -9,44 +9,79 @@ @services_root_command.command(no_args_is_help=True) def stop( ctx: typer.Context, - service_name: Annotated[ServicesEnum, typer.Argument(help="Name of the services_manager.")], + service_name: Annotated[ServicesEnum, typer.Argument(help="Name of the service.")], ): """Stops global services.""" services_manager: ServicesManager = ctx.obj["services"] + if service_name.value == ServicesEnum.all: + for service in ServicesEnum: + if service == ServicesEnum.all: + continue - if services_manager.compose_project.is_service_running(service_name.value): - services_manager.compose_project.stop_service(services=[service_name.value]) + if not services_manager.compose_project.is_service_running(service.value): + richprint.print(f"Skipping not running service {service.value}.") + continue + + services_manager.compose_project.stop_service(services=[service.value]) + richprint.print(f"Stopped service {service.value}.") else: - richprint.exit(f"{service_name.value} is not running.") + if services_manager.compose_project.is_service_running(service_name.value): + services_manager.compose_project.stop_service(services=[service_name.value]) + else: + richprint.print(f"Skipping already stopped service {service_name.value}.") @services_root_command.command(no_args_is_help=True) def start( ctx: typer.Context, - service_name: Annotated[ServicesEnum, typer.Argument(help="Name of the services_manager.")], + service_name: Annotated[ServicesEnum, typer.Argument(help="Name of the service.")], ): """Starts global services.""" services_manager: ServicesManager = ctx.obj["services"] - if services_manager.compose_project.is_service_running(service_name.value): - services_manager.compose_project.start_service(services=[service_name.value]) + if service_name.value == ServicesEnum.all: + + for service in ServicesEnum: + if service == ServicesEnum.all: + continue + + if services_manager.compose_project.is_service_running(service.value): + richprint.print(f"Skipping already running service {service.value}.") + continue + + services_manager.compose_project.start_service(services=[service.value]) + richprint.print(f"Started service {service.value}.") else: - richprint.exit(f"{service_name.value} is already running.") + if not services_manager.compose_project.is_service_running(service_name.value): + services_manager.compose_project.start_service(services=[service_name.value]) + else: + richprint.print(f"Skipping already running service {service_name.value}.") @services_root_command.command(no_args_is_help=True) def restart( ctx: typer.Context, - service_name: Annotated[ServicesEnum, typer.Argument(help="Name of the services_manager.")], + service_name: Annotated[ServicesEnum, typer.Argument(help="Name of the service.")], ): """Restarts global services.""" services_manager: ServicesManager = ctx.obj["services"] - services_manager.compose_project.restart_service(services=[service_name.value]) + + if service_name.value == ServicesEnum.all: + + for service in ServicesEnum: + if service == ServicesEnum.all: + continue + + services_manager.compose_project.restart_service(services=[service.value]) + richprint.print(f"Restarted service {service.value}.") + else: + services_manager.compose_project.restart_service(services=[service_name.value]) + richprint.print(f"Restarted service {service_name.value}.") @services_root_command.command(no_args_is_help=True) def shell( ctx: typer.Context, - service_name: Annotated[ServicesEnum, typer.Argument(help="Name of the services_manager.")], + service_name: Annotated[ServicesEnum, typer.Argument(help="Name of the service.")], user: Annotated[Optional[str], typer.Option(help="Connect as this user.")] = None, ): """ diff --git a/frappe_manager/services_manager/services.py b/frappe_manager/services_manager/services.py index 605962ef..ee0c0d04 100644 --- a/frappe_manager/services_manager/services.py +++ b/frappe_manager/services_manager/services.py @@ -47,29 +47,34 @@ def __init__( def entrypoint_checks(self, start=False): if not self.path.exists(): try: - richprint.print(f"Creating services", emoji_code=":construction:") + richprint.print(f"Creating global services [blue]{', '.join(self.compose_project.compose_file_manager.get_services_list())}[/blue].", emoji_code=":construction:") self.path.mkdir(parents=True, exist_ok=True) self.create(clean_install=True) + except Exception as e: - raise ServicesNotCreated(f'Error Caused: {e}') + raise ServicesNotCreated(f"Not able to create global services [blue]{', '.join(self.compose_project.compose_file_manager.get_services_list())}[/blue].") self.compose_project.pull_images() - richprint.print(f"Creating services: Done") + richprint.print(f"Created global services [blue]{', '.join(self.compose_project.compose_file_manager.get_services_list())}[/blue].") + if start: self.compose_project.start_service() if not self.compose_path.exists(): raise ServicesComposeNotExist( - f"Seems like services has taken a down. Compose file not found at -> {self.compose_path}. Please recreate services." + f"Seems like global services has taken a down. No compose file found at {self.compose_path}." ) if start: if not self.typer_context.invoked_subcommand == "service": + if not self.compose_project.running: - richprint.warning("services are not running. Starting it") + self.are_ports_free() + richprint.print(f"Started non running global services [blue]{', '.join(self.compose_project.compose_file_manager.get_services_list())}[/blue].") self.compose_project.start_service() + self.database_manager: DatabaseServiceManager = MariaDBManager( DatabaseServerServiceInfo.import_from_compose_file('global-db', self.compose_project), self.compose_project ) @@ -176,7 +181,7 @@ def create(self, backup: bool = False, clean_install: bool = True): try: temp_dir.mkdir(parents=True, exist_ok=True) except Exception as e: - richprint.exit(f"Failed to create global services bind mount directories. Error: {e}") + raise ServicesNotCreated(f"Failed to create global services required dir {temp_dir.absolute()}.") # populate secrets for db db_password_path = self.path / 'secrets' / 'db_password.txt' @@ -233,7 +238,7 @@ def generate_compose(self, inputs: dict): # TODO do something about this exception except Exception as e: - richprint.exit(f"Not able to generate global site compose. Error: {e}") + raise ServicesNotCreated(f"Not able to generate global services compose file.") def shell(self, container: str, user: str | None = None): richprint.stop() @@ -250,5 +255,8 @@ def remove_itself(self): shutil.rmtree(self.path) def are_ports_free(self): + ports = [80, 443] + richprint.change_head(f"Verifying ports {', '.join(map(str, ports))} availability.") docker_used_ports = self.compose_project.get_host_port_binds() - check_and_display_port_status([80, 443], exclude=docker_used_ports) + check_and_display_port_status(ports, exclude=docker_used_ports) + richprint.print(f"Global services will utilize ports {', '.join(map(str, ports))}.") diff --git a/frappe_manager/site_manager/bench_config.py b/frappe_manager/site_manager/bench_config.py index 60e245b6..f1373b4a 100644 --- a/frappe_manager/site_manager/bench_config.py +++ b/frappe_manager/site_manager/bench_config.py @@ -5,7 +5,8 @@ from typing import Any, List, Optional from pydantic import BaseModel, Field, model_validator, validator from frappe_manager import STABLE_APP_BRANCH_MAPPING_LIST -from frappe_manager.ssl_manager import SUPPORTED_SSL_TYPES +from frappe_manager.metadata_manager import FMConfigManager, FMLetsencryptConfig +from frappe_manager.ssl_manager import LETSENCRYPT_PREFERRED_CHALLENGE, SUPPORTED_SSL_TYPES from frappe_manager.ssl_manager.certificate import SSLCertificate from frappe_manager.ssl_manager.letsencrypt_certificate import LetsencryptSSLCertificate from frappe_manager.utils.helpers import get_container_name_prefix @@ -105,14 +106,45 @@ def import_from_toml(cls, path: Path) -> "BenchConfig": data['root_path'] = str(path) - # Extract SSL data and remove it from the main data dictionary ssl_data = data.get('ssl', None) + if ssl_data: domain: str = data.get('name', None) # Set domain from main data if necessary ssl_type = ssl_data.get('ssl_type', SUPPORTED_SSL_TYPES.none) + if ssl_type == SUPPORTED_SSL_TYPES.le: email = ssl_data.get('email', None) - ssl_instance = LetsencryptSSLCertificate(domain=domain, ssl_type=ssl_type, email=email) + + fm_config_manager = FMConfigManager.import_from_toml() + + pref_challenge_data = ssl_data.get("preferred_challenge", None) + + api_token = ssl_data.get('api_token', None) + + if not api_token: + api_token = fm_config_manager.letsencrypt.api_token + + api_key = ssl_data.get('api_key', None) + + if not api_key: + api_key = fm_config_manager.letsencrypt.api_key + + if not pref_challenge_data: + if fm_config_manager.letsencrypt.exists: + preferred_challenge = LETSENCRYPT_PREFERRED_CHALLENGE.dns01 + else: + preferred_challenge = LETSENCRYPT_PREFERRED_CHALLENGE.http01 + else: + preferred_challenge = pref_challenge_data + + ssl_instance = LetsencryptSSLCertificate( + domain=domain, + ssl_type=ssl_type, + email=email, + preferred_challenge=preferred_challenge, + api_key=api_key, + api_token=api_token, + ) else: ssl_instance = SSLCertificate(domain=domain, ssl_type=SUPPORTED_SSL_TYPES.none) else: diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 9c5e6bb4..52fab987 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -32,7 +32,7 @@ BenchWorkersSupervisorConfigurtionGenerateError, ) from frappe_manager.site_manager.workers_manager.SiteWorker import BenchWorkers -from frappe_manager.ssl_manager import SUPPORTED_SSL_TYPES +from frappe_manager.ssl_manager import LETSENCRYPT_PREFERRED_CHALLENGE, SUPPORTED_SSL_TYPES from frappe_manager.ssl_manager.certificate import SSLCertificate from frappe_manager.ssl_manager.nginxproxymanager import NginxProxyManager from frappe_manager.ssl_manager.ssl_certificate_manager import SSLCertificateManager @@ -671,13 +671,12 @@ def remove_certificate(self): def update_certificate(self, certificate: SSLCertificate, raise_error: bool = True): if certificate.ssl_type == SUPPORTED_SSL_TYPES.le: if self.has_certificate(): - if not raise_error: - return - raise BenchSSLCertificateAlreadyIssued(self.name) - - self.certificate_manager.set_certificate(certificate) - self.bench_config.ssl = certificate - self.create_certificate() + if raise_error: + raise BenchSSLCertificateAlreadyIssued(self.name) + else: + self.certificate_manager.set_certificate(certificate) + self.bench_config.ssl = certificate + self.create_certificate() elif certificate.ssl_type == SUPPORTED_SSL_TYPES.none: if self.has_certificate(): @@ -686,6 +685,7 @@ def update_certificate(self, certificate: SSLCertificate, raise_error: bool = Tr if not raise_error: return raise BenchSSLCertificateNotIssued(self.name) + return True def renew_certificate(self): @@ -717,6 +717,13 @@ def info(self): protocol = 'https' if self.has_certificate() else 'http' + ssl_service_type = f'{self.bench_config.ssl.ssl_type.value}' + + if self.bench_config.ssl.ssl_type == SUPPORTED_SSL_TYPES.le: + ssl_service_type = ( + f'[{self.bench_config.ssl.preferred_challenge.value}] {self.bench_config.ssl.ssl_type.value}' + ) + data = { "Bench Url": f"{protocol}://{self.name}", "Bench Root": f"[link=file://{self.path.absolute()}]{self.path.absolute()}[/link]", @@ -729,7 +736,7 @@ def info(self): "DB User": db_user, "DB Password": db_pass, "Environment": self.bench_config.environment_type.value, - "HTTPS": f'{self.bench_config.ssl.ssl_type.value} ({format_ssl_certificate_time_remaining(self.certificate_manager.get_certficate_expiry())})' + "HTTPS": f'{ssl_service_type.upper()} ({format_ssl_certificate_time_remaining(self.certificate_manager.get_certficate_expiry())})' if self.has_certificate() else 'Not Enabled', } diff --git a/frappe_manager/ssl_manager/__init__.py b/frappe_manager/ssl_manager/__init__.py index 489ef28a..eeba9a4f 100644 --- a/frappe_manager/ssl_manager/__init__.py +++ b/frappe_manager/ssl_manager/__init__.py @@ -4,3 +4,8 @@ class SUPPORTED_SSL_TYPES(str, Enum): le = 'letsencrypt' none = 'disable' + + +class LETSENCRYPT_PREFERRED_CHALLENGE(str, Enum): + dns01 = 'dns01' + http01 = 'http01' diff --git a/frappe_manager/ssl_manager/certificate.py b/frappe_manager/ssl_manager/certificate.py index b758172c..4996763c 100644 --- a/frappe_manager/ssl_manager/certificate.py +++ b/frappe_manager/ssl_manager/certificate.py @@ -14,16 +14,3 @@ class SSLCertificate(BaseModel): @property def has_wildcard(self) -> bool: return any(is_wildcard_fqdn(domain) for domain in self.alias_domains) - - -class LetsencryptSSLCertificate(BaseModel): - domain: str - email: str - ssl_type: SUPPORTED_SSL_TYPES - hsts: str = 'off' - alias_domains: List[str] = [] - toml_exclude: Optional[set] = {'domain', 'alias_domains', 'toml_exclude'} - - @property - def has_wildcard(self) -> bool: - return any(is_wildcard_fqdn(domain) for domain in self.alias_domains) diff --git a/frappe_manager/ssl_manager/certificate_exceptions.py b/frappe_manager/ssl_manager/certificate_exceptions.py index 0c6486ce..23cf0da8 100644 --- a/frappe_manager/ssl_manager/certificate_exceptions.py +++ b/frappe_manager/ssl_manager/certificate_exceptions.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Optional +from frappe_manager import CLI_FM_CONFIG_PATH from frappe_manager.utils.helpers import format_ssl_certificate_time_remaining @@ -13,11 +13,21 @@ def __init__(self, domain, message="No ssl certificate is issued for {}."): super().__init__(self.message) -class SSLDNSChallengeNotImplemented(Exception): - """Exception raised for dns method not implemented.""" +class SSLCertificateEmailNotFoundError(Exception): + """Exception raised when a certificate is not found.""" + + def __init__(self, domain, message="Please provide email using flag '"): + self.domain = domain + self.message = message.format(self.domain) + super().__init__(self.message) + + +class SSLDNSChallengeCredentailsNotFound(Exception): + """Exception raised for dns method required credential not found.""" - def __init__(self): - super().__init__() + def __init__(self, message: str = f"Cloudflare dns credentials not found in {CLI_FM_CONFIG_PATH}"): + self.message = message + super().__init__(message) class SSLCertificateChallengeFailed(Exception): diff --git a/frappe_manager/ssl_manager/letsencrypt_certificate.py b/frappe_manager/ssl_manager/letsencrypt_certificate.py index 5f0e0e2a..87432950 100644 --- a/frappe_manager/ssl_manager/letsencrypt_certificate.py +++ b/frappe_manager/ssl_manager/letsencrypt_certificate.py @@ -1,5 +1,41 @@ -from pydantic import EmailStr +from typing import Optional, List, Self +from pydantic import EmailStr, Field, model_validator, validator +from frappe_manager.ssl_manager import LETSENCRYPT_PREFERRED_CHALLENGE from frappe_manager.ssl_manager.certificate import SSLCertificate +from frappe_manager.ssl_manager.certificate_exceptions import SSLDNSChallengeCredentailsNotFound +from frappe_manager.display_manager.DisplayManager import richprint + class LetsencryptSSLCertificate(SSLCertificate): - email: EmailStr + preferred_challenge: LETSENCRYPT_PREFERRED_CHALLENGE + email: EmailStr = Field(..., description="Email used by certbot.") + api_token: Optional[str] = Field(None, description="Cloudflare API token used by Certbot.") + api_key: Optional[str] = Field(None, description="Cloudflare Global API Key used by Certbot.") + toml_exclude: Optional[set] = {'domain', 'alias_domains', 'toml_exclude'} + + @model_validator(mode="after") + def validate_credentials(self) -> Self: + if self.preferred_challenge == LETSENCRYPT_PREFERRED_CHALLENGE.dns01: + if self.api_key or self.api_token: + return self + else: + raise SSLDNSChallengeCredentailsNotFound() + + return self + + def get_cloudflare_dns_credentials(self) -> str: + creds: List[str] = [] + + if self.api_key: + richprint.print('Using Cloudflare GLOBAL API KEY') + creds.append(f'dns_cloudflare_email = {self.email}\n') + creds.append(f'dns_cloudflare_api_key = {self.api_key}\n') + + if self.api_token: + richprint.print('Using Cloudflare API Token') + creds.append(f'dns_cloudflare_api_token = {self.api_token}\n') + + if not creds: + raise SSLDNSChallengeCredentailsNotFound() + + return "\n".join(creds) diff --git a/frappe_manager/ssl_manager/letsencrypt_certificate_service.py b/frappe_manager/ssl_manager/letsencrypt_certificate_service.py index 7094e8aa..020e873d 100644 --- a/frappe_manager/ssl_manager/letsencrypt_certificate_service.py +++ b/frappe_manager/ssl_manager/letsencrypt_certificate_service.py @@ -1,7 +1,7 @@ import shlex from io import StringIO from pathlib import Path -from typing import List, Literal, Tuple +from typing import List, Tuple from certbot._internal.main import make_or_verify_needed_dirs from certbot._internal.plugins import disco as plugins_disco from certbot._internal import cli, storage @@ -9,7 +9,7 @@ from certbot import crypto_util from certbot.errors import AuthorizationError from frappe_manager.logger import log -from frappe_manager.ssl_manager import SUPPORTED_SSL_TYPES +from frappe_manager.ssl_manager import LETSENCRYPT_PREFERRED_CHALLENGE, SUPPORTED_SSL_TYPES from frappe_manager.ssl_manager.certificate_exceptions import ( SSLCertificateChallengeFailed, SSLCertificateGenerateFailed, @@ -25,18 +25,18 @@ def __init__( self, ssl_service_dir: Path, webroot_dir: Path, - preferred_challenge: Literal['http', 'dns'] = 'http', ): self.webroot_dir = webroot_dir - self.preferred_challenge = preferred_challenge self.root_dir = ssl_service_dir / SUPPORTED_SSL_TYPES.le.value # certbot dirs self.config_dir: Path = self.root_dir / "config" self.work_dir: Path = self.root_dir / 'work' self.logs_dir: Path = self.root_dir / 'logs' + self.dns_config_dir = self.root_dir / 'dns_configs' self.base_command = f"--work-dir {self.work_dir} --config-dir {self.config_dir} --logs-dir {self.logs_dir}" + self.logger = log.get_logger() self.console_output = StringIO() def renew_certificate(self, certificate: LetsencryptSSLCertificate): @@ -81,9 +81,25 @@ def remove_certificate(self, certificate: LetsencryptSSLCertificate): richprint.print("Removed Letsencrypt certificate") def generate_certificate(self, certificate: LetsencryptSSLCertificate): - gen_command: str = self.base_command + f" certonly --webroot -w {self.webroot_dir} " + gen_command: str = self.base_command + f" certonly " + + richprint.print(f"Using Let's Encrypt {certificate.preferred_challenge.value} challenge.") + + dns_config_path = self.dns_config_dir / f'{certificate.domain}.txt' + + if certificate.preferred_challenge == LETSENCRYPT_PREFERRED_CHALLENGE.http01: + gen_command += f' --webroot -w {self.webroot_dir}' + + elif certificate.preferred_challenge == LETSENCRYPT_PREFERRED_CHALLENGE.dns01: + self.dns_config_dir.mkdir(parents=True, exist_ok=True) + + api_creds = certificate.get_cloudflare_dns_credentials() + dns_config_path.write_text(api_creds) + dns_config_path.chmod(0o600) + + gen_command += f' --dns-cloudflare --dns-cloudflare-credentials {dns_config_path.absolute()}' + gen_command += f' --keep-until-expiring --expand' - # gen_command += ' --staging' gen_command += f' --agree-tos -m "{certificate.email}" --no-eff-email' all_domains = [f'{certificate.domain}'] + certificate.alias_domains @@ -94,21 +110,25 @@ def generate_certificate(self, certificate: LetsencryptSSLCertificate): try: richprint.change_head("Getting Letsencrypt certificate") + self.logger.debug(f'Certbot command: {gen_command}') config = self._get_le_config(shlex.split(gen_command), quiet=True) plugins = plugins_disco.PluginsRegistry.find_all() config.func(config, plugins) - richprint.stdout.print(self.console_output.getvalue().strip()) + output = '\n'.join(line for line in self.console_output.getvalue().split('\n') if not line.startswith('!!')) + richprint.stdout.print(output) except AuthorizationError as e: - logger = log.get_logger() - logger.exception(e) - richprint.stdout.print(self.console_output.getvalue().strip()) - raise SSLCertificateChallengeFailed(self.preferred_challenge) + self.logger.exception(e) + output = '\n'.join(line for line in self.console_output.getvalue().split('\n') if not line.startswith('!!')) + richprint.stdout.print(output) + dns_config_path.unlink() + raise SSLCertificateChallengeFailed(certificate.preferred_challenge) except Exception as e: - logger = log.get_logger() - logger.exception(e) - richprint.stdout.print(self.console_output.getvalue().strip()) + self.logger.exception(e) + output = '\n'.join(line for line in self.console_output.getvalue().split('\n') if not line.startswith('!!')) + richprint.stdout.print(output) + dns_config_path.unlink() raise SSLCertificateGenerateFailed() richprint.print("Acquired Letsencrypt certificate: Done") diff --git a/frappe_manager/ssl_manager/ssl_certificate_manager.py b/frappe_manager/ssl_manager/ssl_certificate_manager.py index 914f9d92..45945898 100644 --- a/frappe_manager/ssl_manager/ssl_certificate_manager.py +++ b/frappe_manager/ssl_manager/ssl_certificate_manager.py @@ -7,7 +7,6 @@ from frappe_manager.ssl_manager.certificate_exceptions import ( SSLCertificateNotDueForRenewalError, SSLCertificateNotFoundError, - SSLDNSChallengeNotImplemented, ) from frappe_manager.ssl_manager.certificate import SSLCertificate from frappe_manager.ssl_manager.nginxproxymanager import NginxProxyManager @@ -23,10 +22,6 @@ class SSLCertificateManager: proxy_manager: NginxProxyManager def __init__(self, certificate: SSLCertificate, webroot_dir: Path, proxy_manager: NginxProxyManager): - # Check if the domains and alias domains doesn't contain wildcards - if certificate.has_wildcard: - raise SSLDNSChallengeNotImplemented - self.certificate = certificate self.proxy_manager = proxy_manager self.webroot_dir = webroot_dir diff --git a/frappe_manager/templates/bench_config.toml b/frappe_manager/templates/bench_config.toml index 061216f3..9340eec6 100644 --- a/frappe_manager/templates/bench_config.toml +++ b/frappe_manager/templates/bench_config.toml @@ -1,20 +1,34 @@ -name = "example.com" +# Denotes the bench name. +# Should not be edited by user. +name = "fm.com" -# when true enables frappe developer mode +# Enables Frappe developer mode for the bench. developer_mode = false -# when true enables admin tools mode +# Enables admin tools (e.g., Mailhog and Adminer) for the bench. admin_tools = false -# "prod" -> for bench prod environment -# "dev" -> for bench dev environment +# Sets the bench environment to either "prod" (production) or "dev" (development). environment_type = "prod" -# enable ssl if defined [ssl] -# ssl_type -> "letsencrypt" / "disable" +# Sets the SSL type to be used by the bench, in this case, "letsencrypt" for Let's Encrypt. ssl_type = "letsencrypt" -# only applicable if ssl_type is "letsencrypt" + +# Controls the HSTS (HTTP Strict Transport Security) header used by the bench. +# When set to "off", the HSTS header will not be included. hsts = "off" -# only applicable if ssl_type is "letsencrypt" -email = "exampl@example.com" + +# Specifies the preferred Certbot challenge method to be used for Let's Encrypt certificate validation. +preferred_challenge = "http01" + +# The email address associated with Let's Encrypt. +# This is used for notifications, recovery, and is used with `api_key` for the Global API key of Cloudflare. +# For more information, see [this documentation](https://certbot-dns-cloudflare.readthedocs.io/en/stable/#certbot-cloudflare-key-ini). +email = "cloudflare@example.com" + +# Cloudflare Global API Key for Let's Encrypt DNS01 Challenge. +api_token = "0123456789abcdef0123456789abcdef01234567" + +# Cloudflare API token for Let's Encrypt DNS01 Challenge. +api_key = "0123456789abcdef0123456789abcdef01234" diff --git a/frappe_manager/templates/fm_config.toml b/frappe_manager/templates/fm_config.toml index bb89fccb..ebe61247 100644 --- a/frappe_manager/templates/fm_config.toml +++ b/frappe_manager/templates/fm_config.toml @@ -1,5 +1,15 @@ -# Don't change the version no this will affect the migrations. -version = "0.13.0" # set by fm to the current version +# Denotes the current version of fm. +# Should not be edited by user. +version = "0.14.0" -# Email used by Let's Encrypt if set then fm will never ask email for Let's Encrypt. -le_email = 'example@example.com' +[letsencrypt] +# The email address associated with Let's Encrypt. +# This is used for notifications, recovery, and is used with api_key for the Global API key of Cloudflare. +# For more information, see [this documentation](https://certbot-dns-cloudflare.readthedocs.io/en/stable/#certbot-cloudflare-key-ini). +email = 'cloudflare@example.co' + +# Cloudflare API token for Let's Encrypt DNS01 Challenge. +api_key = '0123456789abcdef0123456789abcdef01234' + +# Cloudflare Global API Key for Let's Encrypt DNS01 Challenge. +api_token = '0123456789abcdef0123456789abcdef01234567' diff --git a/frappe_manager/utils/helpers.py b/frappe_manager/utils/helpers.py index 28da2adb..32d30c44 100644 --- a/frappe_manager/utils/helpers.py +++ b/frappe_manager/utils/helpers.py @@ -110,7 +110,6 @@ def check_and_display_port_status(ports_to_check: list, exclude=[]): ports_to_check (list): List of ports to check. exclude (list, optional): List of ports to exclude from checking. Defaults to []. """ - richprint.change_head("Checking Ports") if exclude: # Removing elements present in remove_array from original_array ports_to_check = [x for x in exclude if x not in ports_to_check] @@ -119,10 +118,8 @@ def check_and_display_port_status(ports_to_check: list, exclude=[]): already_binded = check_ports(ports_to_check) if already_binded: richprint.exit( - f"Whoa there! Looks like the {' '.join([ str(x) for x in already_binded ])} { 'ports are' if len(already_binded) > 1 else 'port is' } having a party already! Can you do us a solid and free up those ports?" + f"Ports {', '.join(map(str, already_binded))} {'are' if len(already_binded) > 1 else 'is'} currently in use. Please free up these ports." ) - richprint.print("Ports Check : Passed") - def generate_random_text(length=50): """ diff --git a/poetry.lock b/poetry.lock index 94812d07..ec79f2cc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -35,6 +35,25 @@ files = [ {file = "annotated_types-0.6.0.tar.gz", hash = "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d"}, ] +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + [[package]] name = "certbot" version = "2.10.0" @@ -65,6 +84,27 @@ dev = ["azure-devops", "ipdb", "poetry (>=1.2.0)", "poetry-plugin-export (>=1.1. docs = ["Sphinx (>=1.2)", "sphinx-rtd-theme"] test = ["coverage", "mypy", "pip", "pylint", "pytest", "pytest-cov", "pytest-xdist", "setuptools", "tox", "types-httplib2", "types-pyOpenSSL", "types-pyRFC3339", "types-pytz", "types-pywin32", "types-requests", "types-setuptools", "types-six", "wheel"] +[[package]] +name = "certbot-dns-cloudflare" +version = "2.10.0" +description = "Cloudflare DNS Authenticator plugin for Certbot" +optional = false +python-versions = ">=3.8" +files = [ + {file = "certbot-dns-cloudflare-2.10.0.tar.gz", hash = "sha256:45b058c3e515a1853b33dcc3367c12978c0930990a563d26c879d9883a639c07"}, + {file = "certbot_dns_cloudflare-2.10.0-py3-none-any.whl", hash = "sha256:2b26ed46a1c0f152478b7372f1fc36a4fa1612aa05cde3ab21dada724b32ed40"}, +] + +[package.dependencies] +acme = ">=2.10.0" +certbot = ">=2.10.0" +cloudflare = ">=1.5.1" +setuptools = ">=41.6.0" + +[package.extras] +docs = ["Sphinx (>=1.0)", "sphinx-rtd-theme"] +test = ["pytest"] + [[package]] name = "certifi" version = "2024.2.2" @@ -253,6 +293,24 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} +[[package]] +name = "cloudflare" +version = "2.20.0" +description = "Python wrapper for the Cloudflare v4 API" +optional = false +python-versions = ">3.6.0" +files = [ + {file = "cloudflare-2.20.0.tar.gz", hash = "sha256:46aefc39dfaa2365d639b423cec2cd5350ae11153c7247d3eb3545bdcf01a68a"}, +] + +[package.dependencies] +jsonlines = "*" +pyyaml = "*" +requests = "*" + +[package.extras] +test = ["pytest"] + [[package]] name = "colorama" version = "0.4.6" @@ -439,6 +497,20 @@ pyopenssl = ">=0.13" [package.extras] docs = ["sphinx (>=4.3.0)", "sphinx-rtd-theme (>=1.0)"] +[[package]] +name = "jsonlines" +version = "4.0.0" +description = "Library with helpers for the jsonlines file format" +optional = false +python-versions = ">=3.8" +files = [ + {file = "jsonlines-4.0.0-py3-none-any.whl", hash = "sha256:185b334ff2ca5a91362993f42e83588a360cf95ce4b71a73548502bda52a7c55"}, + {file = "jsonlines-4.0.0.tar.gz", hash = "sha256:0c6d2c09117550c089995247f605ae4cf77dd1533041d366351f6f298822ea74"}, +] + +[package.dependencies] +attrs = ">=19.2.0" + [[package]] name = "markdown-it-py" version = "3.0.0" @@ -784,6 +856,66 @@ files = [ {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, ] +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + [[package]] name = "requests" version = "2.31.0" @@ -1004,4 +1136,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "bb03f1df584decf16ac142f49cfd6378a799bbf3638fbef570198c9020e16a88" +content-hash = "30700255d7d524fddd9d91dba0b9559e3f356ea14ab2d37abcceb4c27156e4a8" diff --git a/pyproject.toml b/pyproject.toml index 33e282c8..d7b274b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "frappe-manager" -version = "0.13.4" +version = "0.14.0" license = "MIT" repository = "https://github.com/rtcamp/frappe-manager" description = "A CLI tool based on Docker Compose to easily manage Frappe based projects. As of now, only suitable for development in local machines running on Mac and Linux based OS." @@ -29,6 +29,7 @@ certbot = "^2.9.0" pydantic = "^2.6.4" email-validator = "^2.1.1" jinja2 = "^3.1.3" +certbot-dns-cloudflare = "^2.10.0" [tool.pyright] reportOptionalMemberAccess = false diff --git a/test/e2e_test.sh b/test/e2e_test.sh new file mode 100755 index 00000000..c3131bb6 --- /dev/null +++ b/test/e2e_test.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +PS4='+\[\033[0;33m\](\[\033[0;36m\]${BASH_SOURCE##*/}:${LINENO}\[\033[0;33m\])\[\033[0m\] ' + +set -xe + +source ${PWD}/fm.sh +source ${PWD}/helpers.sh + +main() { + Prequisites + + CreateSite "test-site.prod.local" prod + CreateSite "test-site.dev.local" dev + + ListSites + + StopSite "test-site.prod.local" + StopSite "test-site.dev.local" + + StartSite "test-site.prod.local" + StartSite "test-site.dev.local" + + TestSiteReachability "test-site.prod.local" + TestSiteReachability "test-site.dev.local" + + GetInfoSite "test-site.prod.local" + GetInfoSite "test-site.dev.local" + + DeleteSite "test-site.prod.local" + DeleteSite "test-site.dev.local" + + RemoveDanglingDockerStuff +} + +time main diff --git a/test/fm.sh b/test/fm.sh new file mode 100755 index 00000000..03a5e230 --- /dev/null +++ b/test/fm.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +CreateSite() { + local siteName="$1" + local envN="$2" + echo "Create SiteName: $siteName, Env: $envN" + if [ ! "${envN:-}" ]; then + fm create $siteName + else + fm create $siteName --env $2 + fi + + echo "Get Request to the homepage of Site: $siteName, Env: $envN" + TestSiteReachability "$siteName" +} + +TestSiteReachability() { + local siteName="$1" + curl -f \ + --retry 18 --retry-max-time 600 \ + --head \ + -H "Host: $siteName" \ + -H "Cache-Control: no-cache,no-store" \ + http://localhost:80 +} + +MigrationToLatest() { + pip install -U frappe-manager + echo "yes" | fm list + fm --version +} + +DeleteSite() { + local siteName="$1" + echo "Delete SiteName: $siteName" + echo "yes" | fm delete $siteName +} + + +GetInfoSite() { + local siteName="$1" + echo "Info SiteName: $siteName" + fm info $siteName +} + +ListSites() { + echo "List Sites" + fm list +} + +StartSite() { + local siteName="$1" + echo "Start SiteName: $siteName" + fm start $siteName +} + +StopSite() { + local siteName="$1" + echo "Stop SiteName: $siteName" + fm stop $siteName +} diff --git a/test/helpers.sh b/test/helpers.sh new file mode 100755 index 00000000..cade6583 --- /dev/null +++ b/test/helpers.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +Prequisites() { + if [ "$(id -u)" -eq 0 ]; then + info_red "You are running as root." + exit 30 + fi + + for n in git python docker docker-compose; + do + if ! [ -x "$(command -v $n)" ]; then + echo "Error: $n is not installed." >&2 + exit 56 + fi + done +} + +RemoveDanglingDockerStuff() { + docker volume rm -f $(docker volume ls -q) || echo "Failed to delete dangling docker volume" + docker rm -f $(docker ps -aq) || echo "Failed to delete dangling docker container" + docker network rm -f $(docker network ls -q) || echo "Done deleting the networks" +} diff --git a/test/migration_test.sh b/test/migration_test.sh new file mode 100755 index 00000000..0e4c3db4 --- /dev/null +++ b/test/migration_test.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +PS4='+\[\033[0;33m\](\[\033[0;36m\]${BASH_SOURCE##*/}:${LINENO}\[\033[0;33m\])\[\033[0m\] ' + +set -xe + + +source ${PWD}/fm.sh +source ${PWD}/helpers.sh + +oldToNew() { + Prequisites + CreateSite "migration-site.localhost" + ListSites + StopSite "migration-site.localhost" + StartSite "migration-site.localhost" + GetInfoSite "migration-site.localhost" + MigrationToLatest + StartSite "migration-site.localhost" + TestSiteReachability "migration-site.localhost" + DeleteSite "migration-site.localhost" + RemoveDanglingDockerStuff +} + +semiNewToNew() { + Prequisites + CreateSite "migration-site.dev.local" dev + ListSites + StopSite "migration-site.dev.local" + StartSite "migration-site.dev.local" + GetInfoSite "migration-site.dev.local" + MigrationToLatest + StartSite "migration-site.dev.local" + TestSiteReachability "migration-site.dev.local" + DeleteSite "migration-site.dev.local" + RemoveDanglingDockerStuff +} + +time $1 +# time oldToNew +# time semiNewToNew