From fcfe085336a9775392ce6b99b715e45178bc7a79 Mon Sep 17 00:00:00 2001
From: Xieyt <alokmilenium@gmail.com>
Date: Thu, 16 May 2024 19:13:42 +0530
Subject: [PATCH 01/14] feat: cloudflare dns challenge integration

---
 frappe_manager/commands.py                    | 98 ++++++++++++++-----
 frappe_manager/metadata_manager.py            | 46 +++++++--
 frappe_manager/site_manager/bench_config.py   | 25 ++++-
 frappe_manager/ssl_manager/__init__.py        |  5 +
 frappe_manager/ssl_manager/certificate.py     | 13 ---
 .../ssl_manager/certificate_exceptions.py     | 20 +++-
 .../ssl_manager/letsencrypt_certificate.py    | 40 +++++++-
 .../letsencrypt_certificate_service.py        | 49 +++++++---
 .../ssl_manager/ssl_certificate_manager.py    |  5 -
 pyproject.toml                                |  1 +
 10 files changed, 232 insertions(+), 70 deletions(-)

diff --git a/frappe_manager/commands.py b/frappe_manager/commands.py
index 7628ff3c..8fb1418a 100644
--- a/frappe_manager/commands.py
+++ b/frappe_manager/commands.py
@@ -25,7 +25,7 @@
 from frappe_manager.migration_manager.migration_executor import MigrationExecutor
 from frappe_manager.site_manager.site import Bench
 from frappe_manager.site_manager.workers_manager.SiteWorker import BenchWorkers
-from frappe_manager.ssl_manager import SUPPORTED_SSL_TYPES
+from frappe_manager.ssl_manager import LETSENCRYPT_PREFERRED_CHALLENGE, SUPPORTED_SSL_TYPES
 from frappe_manager.ssl_manager.certificate import SSLCertificate
 from frappe_manager.ssl_manager.letsencrypt_certificate import LetsencryptSSLCertificate
 from frappe_manager.utils.callbacks import (
@@ -156,6 +156,14 @@ def create(
     environment: Annotated[
         FMBenchEnvType, typer.Option("--environment", "--env", help="Select bench environment type.")
     ] = FMBenchEnvType.dev,
+    letsencrypt_preferred_challenge: Annotated[
+        Optional[LETSENCRYPT_PREFERRED_CHALLENGE],
+        typer.Option(help="Select preferred letsencrypt challenge.", show_default=False),
+    ] = None,
+    letsencrypt_email: Annotated[
+        Optional[str],
+        typer.Option(help="Specify email for letsencrypt", show_default=False),
+    ] = None,
     developer_mode: Annotated[
         EnableDisableOptionsEnum, typer.Option(help="Toggle frappe developer mode.")
     ] = EnableDisableOptionsEnum.disable,
@@ -205,18 +213,36 @@ def create(
     bench_config_path = bench_path / CLI_BENCH_CONFIG_FILE_NAME
 
     if ssl == SUPPORTED_SSL_TYPES.le:
-        if fm_config_manager.le_email == 'dummy@fm.fm':
-            email = richprint.prompt_ask(prompt='Please enter [bold][green]email[/bold][/green] for Let\'s Encrypt')
+        if not letsencrypt_preferred_challenge:
+            if fm_config_manager.letsencrypt.exists:
+                if letsencrypt_preferred_challenge is None:
+                    letsencrypt_preferred_challenge = LETSENCRYPT_PREFERRED_CHALLENGE.dns01
+
+            if not letsencrypt_preferred_challenge:
+                letsencrypt_preferred_challenge = LETSENCRYPT_PREFERRED_CHALLENGE.http01
+
+        if fm_config_manager.letsencrypt.email == 'dummy@fm.fm' or fm_config_manager.letsencrypt.email is None:
+            if not letsencrypt_email:
+                richprint.stop()
+                raise typer.BadParameter("No email provided, required by certbot.", param_hint='--letsencrypt-email')
+            else:
+                email = letsencrypt_email
+
             validate_email(email, check_deliverability=False)
-            fm_config_manager.le_email = email
-            fm_config_manager.export_to_toml()
-            richprint.print("Let's Encrypt email saved to configuration. It will be used automatically from now on.")
         else:
-            richprint.print("Using Let's Encrypt email from configuration.")
-            email = fm_config_manager.le_email
-            fm_config_manager.export_to_toml()
-
-        ssl_certificate = LetsencryptSSLCertificate(domain=benchname, ssl_type=ssl, email=email)
+            richprint.print(
+                "Defaulting to Let's Encrypt email from [blue]fm_config.toml[/blue] since [blue]'--letsencrypt-email'[/blue] is not given."
+            )
+            email = fm_config_manager.letsencrypt.email
+
+        ssl_certificate = LetsencryptSSLCertificate(
+            domain=benchname,
+            ssl_type=ssl,
+            email=email,
+            preferred_challenge=letsencrypt_preferred_challenge,
+            api_key=fm_config_manager.letsencrypt.api_key,
+            api_token=fm_config_manager.letsencrypt.api_token,
+        )
 
     elif ssl == SUPPORTED_SSL_TYPES.none:
         ssl_certificate = SSLCertificate(domain=benchname, ssl_type=ssl)
@@ -483,9 +509,17 @@ def update(
         Optional[EnableDisableOptionsEnum],
         typer.Option("--admin-tools", help="Toggle admin-tools.", show_default=False),
     ] = None,
+    letsencrypt_preferred_challenge: Annotated[
+        Optional[LETSENCRYPT_PREFERRED_CHALLENGE],
+        typer.Option(help="Select preferred letsencrypt challenge.", show_default=False),
+    ] = None,
+    letsencrypt_email: Annotated[
+        Optional[str],
+        typer.Option(help="Specify email for letsencrypt", show_default=False),
+    ] = None,
     environment: Annotated[
         Optional[FMBenchEnvType],
-        typer.Option("--environment", help="Switch bench environment.", show_default=False),
+        typer.Option("--environment", "--env", help="Switch bench environment.", show_default=False),
     ] = None,
     developer_mode: Annotated[
         Optional[EnableDisableOptionsEnum],
@@ -530,20 +564,38 @@ def update(
         new_ssl_certificate = SSLCertificate(domain=benchname, ssl_type=SUPPORTED_SSL_TYPES.none)
 
         if ssl == SUPPORTED_SSL_TYPES.le:
-            if fm_config_manager.le_email == 'dummy@fm.fm':
-                email = richprint.prompt_ask(prompt='Please enter [bold][green]email[/bold][/green] for Let\'s Encrypt')
+            if not letsencrypt_preferred_challenge:
+                if fm_config_manager.letsencrypt.exists:
+                    if letsencrypt_preferred_challenge is None:
+                        letsencrypt_preferred_challenge = LETSENCRYPT_PREFERRED_CHALLENGE.dns01
+
+                if not letsencrypt_preferred_challenge:
+                    letsencrypt_preferred_challenge = LETSENCRYPT_PREFERRED_CHALLENGE.http01
+
+            if fm_config_manager.letsencrypt.email == 'dummy@fm.fm' or fm_config_manager.letsencrypt.email is None:
+                if not letsencrypt_email:
+                    richprint.stop()
+                    raise typer.BadParameter(
+                        "No email provided, required by certbot.", param_hint='--letsencrypt-email'
+                    )
+                else:
+                    email = letsencrypt_email
+
                 validate_email(email, check_deliverability=False)
-                fm_config_manager.le_email = email
-                fm_config_manager.export_to_toml()
+            else:
                 richprint.print(
-                    "Let's Encrypt email saved to configuration. It will be used automatically from now on."
+                    "Defaulting to Let's Encrypt email from [blue]fm_config.toml[/blue] since [blue]'--letsencrypt-email'[/blue] is not given."
                 )
-            else:
-                richprint.print("Using Let's Encrypt email from configuration.")
-                email = fm_config_manager.le_email
-                fm_config_manager.export_to_toml()
-
-            new_ssl_certificate = LetsencryptSSLCertificate(domain=benchname, ssl_type=ssl, email=email)
+                email = fm_config_manager.letsencrypt.email
+
+            new_ssl_certificate = LetsencryptSSLCertificate(
+                domain=benchname,
+                ssl_type=ssl,
+                email=email,
+                preferred_challenge=letsencrypt_preferred_challenge,
+                api_key=fm_config_manager.letsencrypt.api_key,
+                api_token=fm_config_manager.letsencrypt.api_token,
+            )
 
         richprint.print("Updating Certificate.")
         bench.update_certificate(new_ssl_certificate)
diff --git a/frappe_manager/metadata_manager.py b/frappe_manager/metadata_manager.py
index 4b6d92a5..b292f8c0 100644
--- a/frappe_manager/metadata_manager.py
+++ b/frappe_manager/metadata_manager.py
@@ -1,21 +1,53 @@
+from typing import Optional
 from pathlib import Path
-from pydantic import BaseModel, EmailStr
+from pydantic import BaseModel, EmailStr, Field
 import tomlkit
 from frappe_manager.migration_manager.version import Version
 from frappe_manager import CLI_FM_CONFIG_PATH
+from frappe_manager.ssl_manager import letsencrypt_certificate
 from frappe_manager.utils.helpers import get_current_fm_version
 
 
+class FMLetsencryptConfig(BaseModel):
+    email: Optional[EmailStr] = Field(None, description="Email used by certbot.")
+    api_token: Optional[str] = Field(None, description="Cloudflare API token used by Certbot.")
+    api_key: Optional[str] = Field(None, description="Cloudflare Global API Key used by Certbot.")
+
+    @property
+    def exists(self):
+        if self.api_token or self.api_key:
+            return True
+
+        return False
+
+    def get_toml_doc(self):
+        model_dict = self.model_dump(exclude_none=True)
+        toml_doc = tomlkit.document()
+
+        for key, value in model_dict.items():
+            if isinstance(value, Path):
+                toml_doc[key] = str(value.absolute())
+            else:
+                toml_doc[key] = value
+        return toml_doc
+
+    @classmethod
+    def import_from_toml_doc(cls, toml_doc):
+        print(toml_doc)
+        config_object = cls(**toml_doc)
+        return config_object
+
+
 class FMConfigManager(BaseModel):
     root_path: Path
     version: Version
-    le_email: EmailStr
+    letsencrypt: FMLetsencryptConfig = Field(default=FMLetsencryptConfig())
 
     def export_to_toml(self, path: Path = CLI_FM_CONFIG_PATH) -> bool:
         exclude = {'root_path'}
 
-        if self.le_email == 'dummy@fm.fm':
-            exclude.add('le_email')
+        if self.letsencrypt.email == 'dummy@fm.fm':
+            exclude.add('letsencrypt')
 
         if self.version < Version('0.13.0'):
             path = CLI_FM_CONFIG_PATH.parent / '.fm.toml'
@@ -46,7 +78,7 @@ def import_from_toml(cls, path: Path = CLI_FM_CONFIG_PATH) -> "FMConfigManager":
         old_config_path = path.parent / '.fm.toml'
 
         input_data['version'] = Version('0.8.3')
-        input_data['le_email'] = 'dummy@fm.fm'
+        input_data['letsencrypt'] = FMLetsencryptConfig(email=None, api_key=None, api_token=None)
         input_data['root_path'] = str(path)
 
         if old_config_path.exists():
@@ -55,7 +87,9 @@ def import_from_toml(cls, path: Path = CLI_FM_CONFIG_PATH) -> "FMConfigManager":
         elif path.exists():
             data = tomlkit.parse(path.read_text())
             input_data['version'] = Version(data.get('version', get_current_fm_version()))
-            input_data['le_email'] = data.get('le_email', 'dummy@fm.fm')
+            input_data['letsencrypt'] = FMLetsencryptConfig(
+                **data.get('letsencrypt', {'email': None, 'api_key': None, 'api_token': None})
+            )
 
         fm_config_instance = cls(**input_data)
         return fm_config_instance
diff --git a/frappe_manager/site_manager/bench_config.py b/frappe_manager/site_manager/bench_config.py
index 60e245b6..41e7a1ea 100644
--- a/frappe_manager/site_manager/bench_config.py
+++ b/frappe_manager/site_manager/bench_config.py
@@ -5,7 +5,8 @@
 from typing import Any, List, Optional
 from pydantic import BaseModel, Field, model_validator, validator
 from frappe_manager import STABLE_APP_BRANCH_MAPPING_LIST
-from frappe_manager.ssl_manager import SUPPORTED_SSL_TYPES
+from frappe_manager.metadata_manager import FMConfigManager, FMLetsencryptConfig
+from frappe_manager.ssl_manager import LETSENCRYPT_PREFERRED_CHALLENGE, SUPPORTED_SSL_TYPES
 from frappe_manager.ssl_manager.certificate import SSLCertificate
 from frappe_manager.ssl_manager.letsencrypt_certificate import LetsencryptSSLCertificate
 from frappe_manager.utils.helpers import get_container_name_prefix
@@ -112,7 +113,27 @@ def import_from_toml(cls, path: Path) -> "BenchConfig":
             ssl_type = ssl_data.get('ssl_type', SUPPORTED_SSL_TYPES.none)
             if ssl_type == SUPPORTED_SSL_TYPES.le:
                 email = ssl_data.get('email', None)
-                ssl_instance = LetsencryptSSLCertificate(domain=domain, ssl_type=ssl_type, email=email)
+
+                fm_config_manager = FMConfigManager.import_from_toml()
+
+                pref_challenge_data = data.get("preferred_challenge", None)
+
+                if not pref_challenge_data:
+                    if fm_config_manager.letsencrypt.exists:
+                        preferred_challenge = LETSENCRYPT_PREFERRED_CHALLENGE.dns01
+                    else:
+                        preferred_challenge = LETSENCRYPT_PREFERRED_CHALLENGE.http01
+                else:
+                    preferred_challenge = pref_challenge_data
+
+                ssl_instance = LetsencryptSSLCertificate(
+                    domain=domain,
+                    ssl_type=ssl_type,
+                    email=email,
+                    preferred_challenge=preferred_challenge,
+                    api_key=fm_config_manager.letsencrypt.api_key,
+                    api_token=fm_config_manager.letsencrypt.api_token,
+                )
             else:
                 ssl_instance = SSLCertificate(domain=domain, ssl_type=SUPPORTED_SSL_TYPES.none)
         else:
diff --git a/frappe_manager/ssl_manager/__init__.py b/frappe_manager/ssl_manager/__init__.py
index 489ef28a..eeba9a4f 100644
--- a/frappe_manager/ssl_manager/__init__.py
+++ b/frappe_manager/ssl_manager/__init__.py
@@ -4,3 +4,8 @@
 class SUPPORTED_SSL_TYPES(str, Enum):
     le = 'letsencrypt'
     none = 'disable'
+
+
+class LETSENCRYPT_PREFERRED_CHALLENGE(str, Enum):
+    dns01 = 'dns01'
+    http01 = 'http01'
diff --git a/frappe_manager/ssl_manager/certificate.py b/frappe_manager/ssl_manager/certificate.py
index b758172c..4996763c 100644
--- a/frappe_manager/ssl_manager/certificate.py
+++ b/frappe_manager/ssl_manager/certificate.py
@@ -14,16 +14,3 @@ class SSLCertificate(BaseModel):
     @property
     def has_wildcard(self) -> bool:
         return any(is_wildcard_fqdn(domain) for domain in self.alias_domains)
-
-
-class LetsencryptSSLCertificate(BaseModel):
-    domain: str
-    email: str
-    ssl_type: SUPPORTED_SSL_TYPES
-    hsts: str = 'off'
-    alias_domains: List[str] = []
-    toml_exclude: Optional[set] = {'domain', 'alias_domains', 'toml_exclude'}
-
-    @property
-    def has_wildcard(self) -> bool:
-        return any(is_wildcard_fqdn(domain) for domain in self.alias_domains)
diff --git a/frappe_manager/ssl_manager/certificate_exceptions.py b/frappe_manager/ssl_manager/certificate_exceptions.py
index 0c6486ce..23cf0da8 100644
--- a/frappe_manager/ssl_manager/certificate_exceptions.py
+++ b/frappe_manager/ssl_manager/certificate_exceptions.py
@@ -1,5 +1,5 @@
 from datetime import datetime
-from typing import Optional
+from frappe_manager import CLI_FM_CONFIG_PATH
 
 from frappe_manager.utils.helpers import format_ssl_certificate_time_remaining
 
@@ -13,11 +13,21 @@ def __init__(self, domain, message="No ssl certificate is issued for {}."):
         super().__init__(self.message)
 
 
-class SSLDNSChallengeNotImplemented(Exception):
-    """Exception raised for dns method not implemented."""
+class SSLCertificateEmailNotFoundError(Exception):
+    """Exception raised when a certificate is not found."""
+
+    def __init__(self, domain, message="Please provide email using flag '"):
+        self.domain = domain
+        self.message = message.format(self.domain)
+        super().__init__(self.message)
+
+
+class SSLDNSChallengeCredentailsNotFound(Exception):
+    """Exception raised for dns method required credential not found."""
 
-    def __init__(self):
-        super().__init__()
+    def __init__(self, message: str = f"Cloudflare dns credentials not found in {CLI_FM_CONFIG_PATH}"):
+        self.message = message
+        super().__init__(message)
 
 
 class SSLCertificateChallengeFailed(Exception):
diff --git a/frappe_manager/ssl_manager/letsencrypt_certificate.py b/frappe_manager/ssl_manager/letsencrypt_certificate.py
index 5f0e0e2a..214bec38 100644
--- a/frappe_manager/ssl_manager/letsencrypt_certificate.py
+++ b/frappe_manager/ssl_manager/letsencrypt_certificate.py
@@ -1,5 +1,41 @@
-from pydantic import EmailStr
+from typing import Optional, List, Self
+from pydantic import EmailStr, Field, model_validator, validator
+from frappe_manager.ssl_manager import LETSENCRYPT_PREFERRED_CHALLENGE
 from frappe_manager.ssl_manager.certificate import SSLCertificate
+from frappe_manager.ssl_manager.certificate_exceptions import SSLDNSChallengeCredentailsNotFound
+from frappe_manager.display_manager.DisplayManager import richprint
+
 
 class LetsencryptSSLCertificate(SSLCertificate):
-    email: EmailStr
+    preferred_challenge: LETSENCRYPT_PREFERRED_CHALLENGE
+    email: EmailStr = Field(..., description="Email used by certbot.")
+    api_token: Optional[str] = Field(None, description="Cloudflare API token used by Certbot.")
+    api_key: Optional[str] = Field(None, description="Cloudflare Global API Key used by Certbot.")
+    toml_exclude: Optional[set] = {'domain', 'alias_domains', 'toml_exclude', 'api_token', 'api_key'}
+
+    @model_validator(mode="after")
+    def validate_credentials(self) -> Self:
+        if self.preferred_challenge == LETSENCRYPT_PREFERRED_CHALLENGE.dns01:
+            if self.api_key or self.api_token:
+                return self
+            else:
+                raise SSLDNSChallengeCredentailsNotFound()
+
+        return self
+
+    def get_cloudflare_dns_credentials(self) -> str:
+        creds: List[str] = []
+
+        if self.api_key:
+            richprint.print('Using Cloudflare GLOBAL API KEY')
+            creds.append(f'dns_cloudflare_email = {self.email}\n')
+            creds.append(f'dns_cloudflare_api_key = {self.api_key}\n')
+
+        if self.api_token:
+            richprint.print('Using Cloudflare API Token')
+            creds.append(f'dns_cloudflare_api_token = {self.api_token}\n')
+
+        if not creds:
+            raise SSLDNSChallengeCredentailsNotFound()
+
+        return "\n".join(creds)
diff --git a/frappe_manager/ssl_manager/letsencrypt_certificate_service.py b/frappe_manager/ssl_manager/letsencrypt_certificate_service.py
index 7094e8aa..1c1dd23c 100644
--- a/frappe_manager/ssl_manager/letsencrypt_certificate_service.py
+++ b/frappe_manager/ssl_manager/letsencrypt_certificate_service.py
@@ -1,7 +1,7 @@
 import shlex
 from io import StringIO
 from pathlib import Path
-from typing import List, Literal, Tuple
+from typing import List, Tuple
 from certbot._internal.main import make_or_verify_needed_dirs
 from certbot._internal.plugins import disco as plugins_disco
 from certbot._internal import cli, storage
@@ -9,7 +9,7 @@
 from certbot import crypto_util
 from certbot.errors import AuthorizationError
 from frappe_manager.logger import log
-from frappe_manager.ssl_manager import SUPPORTED_SSL_TYPES
+from frappe_manager.ssl_manager import LETSENCRYPT_PREFERRED_CHALLENGE, SUPPORTED_SSL_TYPES
 from frappe_manager.ssl_manager.certificate_exceptions import (
     SSLCertificateChallengeFailed,
     SSLCertificateGenerateFailed,
@@ -25,10 +25,8 @@ def __init__(
         self,
         ssl_service_dir: Path,
         webroot_dir: Path,
-        preferred_challenge: Literal['http', 'dns'] = 'http',
     ):
         self.webroot_dir = webroot_dir
-        self.preferred_challenge = preferred_challenge
         self.root_dir = ssl_service_dir / SUPPORTED_SSL_TYPES.le.value
 
         # certbot dirs
@@ -37,6 +35,7 @@ def __init__(
         self.logs_dir: Path = self.root_dir / 'logs'
 
         self.base_command = f"--work-dir {self.work_dir} --config-dir {self.config_dir} --logs-dir {self.logs_dir}"
+        self.logger = log.get_logger()
         self.console_output = StringIO()
 
     def renew_certificate(self, certificate: LetsencryptSSLCertificate):
@@ -81,9 +80,26 @@ def remove_certificate(self, certificate: LetsencryptSSLCertificate):
         richprint.print("Removed Letsencrypt certificate")
 
     def generate_certificate(self, certificate: LetsencryptSSLCertificate):
-        gen_command: str = self.base_command + f" certonly --webroot -w {self.webroot_dir} "
+        gen_command: str = self.base_command + f" certonly "
+
+        richprint.print(f"Using Let's Encrypt {certificate.preferred_challenge.value} challenge.")
+
+        import tempfile
+
+        temp_file = tempfile.NamedTemporaryFile(delete=False)
+
+        if certificate.preferred_challenge == LETSENCRYPT_PREFERRED_CHALLENGE.http01:
+            gen_command += f' --webroot -w {self.webroot_dir}'
+
+        elif certificate.preferred_challenge == LETSENCRYPT_PREFERRED_CHALLENGE.dns01:
+            api_creds = certificate.get_cloudflare_dns_credentials()
+            temp_file.write(api_creds.encode())
+            temp_file.flush()
+
+            gen_command += f' --dns-cloudflare --dns-cloudflare-credentials {temp_file.name}'
+
         gen_command += f' --keep-until-expiring --expand'
-        # gen_command += ' --staging'
+        gen_command += ' --staging'
         gen_command += f' --agree-tos -m "{certificate.email}" --no-eff-email'
 
         all_domains = [f'{certificate.domain}'] + certificate.alias_domains
@@ -94,22 +110,27 @@ def generate_certificate(self, certificate: LetsencryptSSLCertificate):
 
         try:
             richprint.change_head("Getting Letsencrypt certificate")
+            self.logger.debug(f'Certbot command: {gen_command}')
             config = self._get_le_config(shlex.split(gen_command), quiet=True)
             plugins = plugins_disco.PluginsRegistry.find_all()
             config.func(config, plugins)
-            richprint.stdout.print(self.console_output.getvalue().strip())
+            output = '\n'.join(line for line in self.console_output.getvalue().split('\n') if not line.startswith('!!'))
+            richprint.stdout.print(output)
 
         except AuthorizationError as e:
-            logger = log.get_logger()
-            logger.exception(e)
-            richprint.stdout.print(self.console_output.getvalue().strip())
-            raise SSLCertificateChallengeFailed(self.preferred_challenge)
+            self.logger.exception(e)
+            output = '\n'.join(line for line in self.console_output.getvalue().split('\n') if not line.startswith('!!'))
+            richprint.stdout.print(output)
+            raise SSLCertificateChallengeFailed(certificate.preferred_challenge)
 
         except Exception as e:
-            logger = log.get_logger()
-            logger.exception(e)
-            richprint.stdout.print(self.console_output.getvalue().strip())
+            self.logger.exception(e)
+            output = '\n'.join(line for line in self.console_output.getvalue().split('\n') if not line.startswith('!!'))
+            richprint.stdout.print(output)
             raise SSLCertificateGenerateFailed()
 
+        finally:
+            temp_file.close()
+
         richprint.print("Acquired Letsencrypt certificate: Done")
         return self.get_certificate_paths(certificate)
diff --git a/frappe_manager/ssl_manager/ssl_certificate_manager.py b/frappe_manager/ssl_manager/ssl_certificate_manager.py
index 914f9d92..45945898 100644
--- a/frappe_manager/ssl_manager/ssl_certificate_manager.py
+++ b/frappe_manager/ssl_manager/ssl_certificate_manager.py
@@ -7,7 +7,6 @@
 from frappe_manager.ssl_manager.certificate_exceptions import (
     SSLCertificateNotDueForRenewalError,
     SSLCertificateNotFoundError,
-    SSLDNSChallengeNotImplemented,
 )
 from frappe_manager.ssl_manager.certificate import SSLCertificate
 from frappe_manager.ssl_manager.nginxproxymanager import NginxProxyManager
@@ -23,10 +22,6 @@ class SSLCertificateManager:
     proxy_manager: NginxProxyManager
 
     def __init__(self, certificate: SSLCertificate, webroot_dir: Path, proxy_manager: NginxProxyManager):
-        # Check if the domains and alias domains doesn't contain wildcards
-        if certificate.has_wildcard:
-            raise SSLDNSChallengeNotImplemented
-
         self.certificate = certificate
         self.proxy_manager = proxy_manager
         self.webroot_dir = webroot_dir
diff --git a/pyproject.toml b/pyproject.toml
index 33e282c8..dbc42bdd 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -29,6 +29,7 @@ certbot = "^2.9.0"
 pydantic = "^2.6.4"
 email-validator = "^2.1.1"
 jinja2 = "^3.1.3"
+certbot-dns-cloudflare = "^2.10.0"
 
 [tool.pyright]
 reportOptionalMemberAccess = false

From c909aa7442747056cae46d59019a65a9a3a014b7 Mon Sep 17 00:00:00 2001
From: Dipankar Das <65275144+dipankardas011@users.noreply.github.com>
Date: Mon, 20 May 2024 15:48:03 +0530
Subject: [PATCH 02/14] feat: added global service stop all

Signed-off-by: Dipankar Das <65275144+dipankardas011@users.noreply.github.com>
---
 frappe_manager/services_manager/__init__.py |  1 +
 frappe_manager/services_manager/commands.py | 17 +++++++++++++----
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/frappe_manager/services_manager/__init__.py b/frappe_manager/services_manager/__init__.py
index fc0a03ed..675874ac 100644
--- a/frappe_manager/services_manager/__init__.py
+++ b/frappe_manager/services_manager/__init__.py
@@ -3,3 +3,4 @@
 class ServicesEnum(str, Enum):
     global_db= "global-db"
     global_nginx_proxy="global-nginx-proxy"
+    all="all"
diff --git a/frappe_manager/services_manager/commands.py b/frappe_manager/services_manager/commands.py
index f9de8422..03fdc60e 100644
--- a/frappe_manager/services_manager/commands.py
+++ b/frappe_manager/services_manager/commands.py
@@ -13,11 +13,20 @@ def stop(
 ):
     """Stops global services."""
     services_manager: ServicesManager = ctx.obj["services"]
-
-    if services_manager.compose_project.is_service_running(service_name.value):
-        services_manager.compose_project.stop_service(services=[service_name.value])
+    if service_name.value == ServicesEnum.all:
+        for service in ServicesEnum:
+            if service == ServicesEnum.all:
+                continue
+            if services_manager.compose_project.is_service_running(service.value):
+                services_manager.compose_project.stop_service(services=[service.value])
+            else:
+                richprint.warning(f"{service.value} is not running.")
+        return
     else:
-        richprint.exit(f"{service_name.value} is not running.")
+        if services_manager.compose_project.is_service_running(service_name.value):
+            services_manager.compose_project.stop_service(services=[service_name.value])
+        else:
+            richprint.exit(f"{service_name.value} is not running.")
 
 
 @services_root_command.command(no_args_is_help=True)

From 9a4fd50b5414fa42504288ecabf06fb100fd2f1e Mon Sep 17 00:00:00 2001
From: Xieyt <alokmilenium@gmail.com>
Date: Wed, 22 May 2024 20:11:13 +0530
Subject: [PATCH 03/14] update migrations

---
 .../migrations/migrate_0_13_0.py              | 40 +++++--------------
 .../migrations/migrate_0_14_0.py              | 33 +++++++++++++++
 2 files changed, 43 insertions(+), 30 deletions(-)
 create mode 100644 frappe_manager/migration_manager/migrations/migrate_0_14_0.py

diff --git a/frappe_manager/migration_manager/migrations/migrate_0_13_0.py b/frappe_manager/migration_manager/migrations/migrate_0_13_0.py
index 52047f84..a0df5210 100644
--- a/frappe_manager/migration_manager/migrations/migrate_0_13_0.py
+++ b/frappe_manager/migration_manager/migrations/migrate_0_13_0.py
@@ -2,6 +2,7 @@
 import os
 import copy
 from pathlib import Path
+import tomlkit
 from frappe_manager.compose_manager import DockerVolumeMount
 from frappe_manager.compose_manager.ComposeFile import ComposeFile
 from frappe_manager.migration_manager.migration_base import MigrationBase
@@ -14,9 +15,6 @@
 from frappe_manager.display_manager.DisplayManager import richprint
 from frappe_manager.migration_manager.version import Version
 from frappe_manager import CLI_DIR, CLI_SERVICES_DIRECTORY
-from frappe_manager.site_manager.bench_config import BenchConfig, FMBenchEnvType
-from frappe_manager.ssl_manager import SUPPORTED_SSL_TYPES
-from frappe_manager.ssl_manager.certificate import SSLCertificate
 from frappe_manager.utils.helpers import get_container_name_prefix
 from frappe_manager.docker_wrapper.DockerClient import DockerClient
 from frappe_manager.migration_manager.backup_manager import BackupManager
@@ -133,9 +131,6 @@ def migrate_bench_compose(self, bench: MigrationBench):
         # create bench config
         frappe = envs.get('frappe', {})
 
-        userid = frappe.get('USERID', os.getuid())
-        usergroup = frappe.get('USERGROUP', os.getgid())
-
         apps_list = frappe.get('APPS_LIST', None)
 
         if apps_list:
@@ -143,33 +138,18 @@ def migrate_bench_compose(self, bench: MigrationBench):
         else:
             apps_list = []
 
-        frappe_branch = frappe.get('FRAPPE_BRANCH', 'version-15')
         developer_mode = frappe.get('DEVELOPER_MODE', True)
-        admin_pass = frappe.get('ADMIN_PASS', 'admin')
         name = frappe.get('SITENAME', bench.name)
-        mariadb_host = frappe.get('MARIADB_HOST', 'global-db')
-        mariadb_root_pass = frappe.get('MARIADB_ROOT_PASS', '/run/secrets/db_root_password')
-        environment_type = frappe.get('ENVIRONMENT', FMBenchEnvType.dev)
-        ssl_certificate = SSLCertificate(domain=bench.name, ssl_type=SUPPORTED_SSL_TYPES.none)
-
-        # TODO Handle admin tools compose change
-        bench_config = BenchConfig(
-            name=name,
-            userid=userid,
-            usergroup=usergroup,
-            apps_list=apps_list,
-            frappe_branch=frappe_branch,
-            developer_mode=developer_mode,
-            admin_tools=True,
-            admin_pass=admin_pass,
-            mariadb_host=mariadb_host,
-            mariadb_root_pass=mariadb_root_pass,
-            environment_type=environment_type,
-            root_path=bench_config_path,
-            ssl=ssl_certificate,
-        )
 
-        bench_config.export_to_toml(bench_config_path)
+        bench_config = tomlkit.document()
+
+        bench_config['name'] = name
+        bench_config['developer_mode'] = developer_mode
+        bench_config['admin_tools'] = True
+        bench_config['environment_type'] = 'dev'
+
+        with open(bench_config_path, 'w') as f:
+            f.write(tomlkit.dumps(bench_config))
 
         images_info = bench.compose_project.compose_file_manager.get_all_images()
 
diff --git a/frappe_manager/migration_manager/migrations/migrate_0_14_0.py b/frappe_manager/migration_manager/migrations/migrate_0_14_0.py
new file mode 100644
index 00000000..c486971f
--- /dev/null
+++ b/frappe_manager/migration_manager/migrations/migrate_0_14_0.py
@@ -0,0 +1,33 @@
+from pathlib import Path
+import tomlkit
+from frappe_manager.migration_manager.migration_base import MigrationBase
+from frappe_manager.migration_manager.migration_helpers import MigrationBenches, MigrationServicesManager
+from frappe_manager.migration_manager.version import Version
+from frappe_manager.migration_manager.backup_manager import BackupManager
+
+
+class MigrationV0140(MigrationBase):
+    version = Version("0.14.0")
+
+    def init(self):
+        self.cli_dir: Path = Path.home() / 'frappe'
+        self.benches_dir = self.cli_dir / "sites"
+        self.backup_manager = BackupManager(str(self.version), self.benches_dir)
+        self.benches_manager = MigrationBenches(self.benches_dir)
+        self.services_manager: MigrationServicesManager = MigrationServicesManager(
+            services_path=self.cli_dir / 'services'
+        )
+
+    def migrate_services(self):
+        config_path = self.cli_dir / 'fm_config.toml'
+
+        if config_path.exists():
+            data = tomlkit.parse(config_path.read_text())
+            email = data.get('le_email', None)
+            data['letsencrypt'] = {'email': email}
+
+            if email:
+                del data['le_email']
+
+            with open(config_path, 'w') as f:
+                f.write(tomlkit.dumps(data))

From 3cacbe02606b415d3a4d4e040c4cb022a112afd2 Mon Sep 17 00:00:00 2001
From: Xieyt <alokmilenium@gmail.com>
Date: Thu, 23 May 2024 06:27:35 +0530
Subject: [PATCH 04/14] Update Migrations

---
 frappe_manager/metadata_manager.py                         | 7 ++++---
 .../migration_manager/migrations/migrate_0_13_0.py         | 1 +
 .../migration_manager/migrations/migrate_0_14_0.py         | 5 ++++-
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/frappe_manager/metadata_manager.py b/frappe_manager/metadata_manager.py
index b292f8c0..0e959925 100644
--- a/frappe_manager/metadata_manager.py
+++ b/frappe_manager/metadata_manager.py
@@ -4,7 +4,6 @@
 import tomlkit
 from frappe_manager.migration_manager.version import Version
 from frappe_manager import CLI_FM_CONFIG_PATH
-from frappe_manager.ssl_manager import letsencrypt_certificate
 from frappe_manager.utils.helpers import get_current_fm_version
 
 
@@ -33,7 +32,6 @@ def get_toml_doc(self):
 
     @classmethod
     def import_from_toml_doc(cls, toml_doc):
-        print(toml_doc)
         config_object = cls(**toml_doc)
         return config_object
 
@@ -46,8 +44,11 @@ class FMConfigManager(BaseModel):
     def export_to_toml(self, path: Path = CLI_FM_CONFIG_PATH) -> bool:
         exclude = {'root_path'}
 
-        if self.letsencrypt.email == 'dummy@fm.fm':
+        if not self.letsencrypt.email and not self.letsencrypt.api_key and not self.letsencrypt.api_token:
             exclude.add('letsencrypt')
+        else:
+            if self.letsencrypt.email == 'dummy@fm.fm':
+                exclude.add('letsencrypt')
 
         if self.version < Version('0.13.0'):
             path = CLI_FM_CONFIG_PATH.parent / '.fm.toml'
diff --git a/frappe_manager/migration_manager/migrations/migrate_0_13_0.py b/frappe_manager/migration_manager/migrations/migrate_0_13_0.py
index a0df5210..207fd1c6 100644
--- a/frappe_manager/migration_manager/migrations/migrate_0_13_0.py
+++ b/frappe_manager/migration_manager/migrations/migrate_0_13_0.py
@@ -78,6 +78,7 @@ def migrate_services(self):
         # rename main config
         fm_config_path = CLI_DIR / 'fm_config.toml'
         old_fm_config_path = CLI_DIR / '.fm.toml'
+
         if old_fm_config_path.exists():
             old_fm_config_path.rename(fm_config_path)
 
diff --git a/frappe_manager/migration_manager/migrations/migrate_0_14_0.py b/frappe_manager/migration_manager/migrations/migrate_0_14_0.py
index c486971f..cdd50511 100644
--- a/frappe_manager/migration_manager/migrations/migrate_0_14_0.py
+++ b/frappe_manager/migration_manager/migrations/migrate_0_14_0.py
@@ -4,6 +4,7 @@
 from frappe_manager.migration_manager.migration_helpers import MigrationBenches, MigrationServicesManager
 from frappe_manager.migration_manager.version import Version
 from frappe_manager.migration_manager.backup_manager import BackupManager
+from frappe_manager.display_manager.DisplayManager import richprint
 
 
 class MigrationV0140(MigrationBase):
@@ -19,15 +20,17 @@ def init(self):
         )
 
     def migrate_services(self):
+        richprint.change_head("Migrating [blue]fm_config.toml[/blue] changes")
         config_path = self.cli_dir / 'fm_config.toml'
 
         if config_path.exists():
             data = tomlkit.parse(config_path.read_text())
             email = data.get('le_email', None)
-            data['letsencrypt'] = {'email': email}
 
             if email:
+                data['letsencrypt'] = {'email': email}
                 del data['le_email']
 
             with open(config_path, 'w') as f:
                 f.write(tomlkit.dumps(data))
+        richprint.print("Migrated [blue]fm_config.toml[/blue]")

From e2de1c83fb7c4da0d81ea27a819892c39a0f27b5 Mon Sep 17 00:00:00 2001
From: Xieyt <alokmilenium@gmail.com>
Date: Thu, 23 May 2024 16:25:09 +0530
Subject: [PATCH 05/14] remove staging from certbot

---
 frappe_manager/ssl_manager/letsencrypt_certificate_service.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/frappe_manager/ssl_manager/letsencrypt_certificate_service.py b/frappe_manager/ssl_manager/letsencrypt_certificate_service.py
index 1c1dd23c..f889c66b 100644
--- a/frappe_manager/ssl_manager/letsencrypt_certificate_service.py
+++ b/frappe_manager/ssl_manager/letsencrypt_certificate_service.py
@@ -99,7 +99,6 @@ def generate_certificate(self, certificate: LetsencryptSSLCertificate):
             gen_command += f' --dns-cloudflare --dns-cloudflare-credentials {temp_file.name}'
 
         gen_command += f' --keep-until-expiring --expand'
-        gen_command += ' --staging'
         gen_command += f' --agree-tos -m "{certificate.email}" --no-eff-email'
 
         all_domains = [f'{certificate.domain}'] + certificate.alias_domains

From c6b05b31ac5f19a0fa590cd78802b8d8336a8382 Mon Sep 17 00:00:00 2001
From: Xieyt <alokmilenium@gmail.com>
Date: Thu, 23 May 2024 16:26:26 +0530
Subject: [PATCH 06/14] bump v0.14.0

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index dbc42bdd..d7b274b4 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "frappe-manager"
-version = "0.13.4"
+version = "0.14.0"
 license = "MIT"
 repository = "https://github.com/rtcamp/frappe-manager"
 description = "A CLI tool based on Docker Compose to easily manage Frappe based projects. As of now, only suitable for development in local machines running on Mac and Linux based OS."

From f87996896019316fd2b30d909012aa31a26652b3 Mon Sep 17 00:00:00 2001
From: Xieyt <alokmilenium@gmail.com>
Date: Sat, 25 May 2024 01:42:29 +0530
Subject: [PATCH 07/14] Add bench specific dns certbot config

---
 frappe_manager/site_manager/bench_config.py   |  19 ++-
 .../ssl_manager/letsencrypt_certificate.py    |   2 +-
 .../letsencrypt_certificate_service.py        |  17 ++-
 poetry.lock                                   | 134 +++++++++++++++++-
 4 files changed, 157 insertions(+), 15 deletions(-)

diff --git a/frappe_manager/site_manager/bench_config.py b/frappe_manager/site_manager/bench_config.py
index 41e7a1ea..f1373b4a 100644
--- a/frappe_manager/site_manager/bench_config.py
+++ b/frappe_manager/site_manager/bench_config.py
@@ -106,17 +106,28 @@ def import_from_toml(cls, path: Path) -> "BenchConfig":
 
         data['root_path'] = str(path)
 
-        # Extract SSL data and remove it from the main data dictionary
         ssl_data = data.get('ssl', None)
+
         if ssl_data:
             domain: str = data.get('name', None)  # Set domain from main data if necessary
             ssl_type = ssl_data.get('ssl_type', SUPPORTED_SSL_TYPES.none)
+
             if ssl_type == SUPPORTED_SSL_TYPES.le:
                 email = ssl_data.get('email', None)
 
                 fm_config_manager = FMConfigManager.import_from_toml()
 
-                pref_challenge_data = data.get("preferred_challenge", None)
+                pref_challenge_data = ssl_data.get("preferred_challenge", None)
+
+                api_token = ssl_data.get('api_token', None)
+
+                if not api_token:
+                    api_token = fm_config_manager.letsencrypt.api_token
+
+                api_key = ssl_data.get('api_key', None)
+
+                if not api_key:
+                    api_key = fm_config_manager.letsencrypt.api_key
 
                 if not pref_challenge_data:
                     if fm_config_manager.letsencrypt.exists:
@@ -131,8 +142,8 @@ def import_from_toml(cls, path: Path) -> "BenchConfig":
                     ssl_type=ssl_type,
                     email=email,
                     preferred_challenge=preferred_challenge,
-                    api_key=fm_config_manager.letsencrypt.api_key,
-                    api_token=fm_config_manager.letsencrypt.api_token,
+                    api_key=api_key,
+                    api_token=api_token,
                 )
             else:
                 ssl_instance = SSLCertificate(domain=domain, ssl_type=SUPPORTED_SSL_TYPES.none)
diff --git a/frappe_manager/ssl_manager/letsencrypt_certificate.py b/frappe_manager/ssl_manager/letsencrypt_certificate.py
index 214bec38..87432950 100644
--- a/frappe_manager/ssl_manager/letsencrypt_certificate.py
+++ b/frappe_manager/ssl_manager/letsencrypt_certificate.py
@@ -11,7 +11,7 @@ class LetsencryptSSLCertificate(SSLCertificate):
     email: EmailStr = Field(..., description="Email used by certbot.")
     api_token: Optional[str] = Field(None, description="Cloudflare API token used by Certbot.")
     api_key: Optional[str] = Field(None, description="Cloudflare Global API Key used by Certbot.")
-    toml_exclude: Optional[set] = {'domain', 'alias_domains', 'toml_exclude', 'api_token', 'api_key'}
+    toml_exclude: Optional[set] = {'domain', 'alias_domains', 'toml_exclude'}
 
     @model_validator(mode="after")
     def validate_credentials(self) -> Self:
diff --git a/frappe_manager/ssl_manager/letsencrypt_certificate_service.py b/frappe_manager/ssl_manager/letsencrypt_certificate_service.py
index f889c66b..c4ebbe40 100644
--- a/frappe_manager/ssl_manager/letsencrypt_certificate_service.py
+++ b/frappe_manager/ssl_manager/letsencrypt_certificate_service.py
@@ -33,6 +33,7 @@ def __init__(
         self.config_dir: Path = self.root_dir / "config"
         self.work_dir: Path = self.root_dir / 'work'
         self.logs_dir: Path = self.root_dir / 'logs'
+        self.dns_config_dir = self.root_dir / 'dns_configs'
 
         self.base_command = f"--work-dir {self.work_dir} --config-dir {self.config_dir} --logs-dir {self.logs_dir}"
         self.logger = log.get_logger()
@@ -84,19 +85,18 @@ def generate_certificate(self, certificate: LetsencryptSSLCertificate):
 
         richprint.print(f"Using Let's Encrypt {certificate.preferred_challenge.value} challenge.")
 
-        import tempfile
-
-        temp_file = tempfile.NamedTemporaryFile(delete=False)
+        dns_config_path = self.dns_config_dir / f'{certificate.domain}.txt'
 
         if certificate.preferred_challenge == LETSENCRYPT_PREFERRED_CHALLENGE.http01:
             gen_command += f' --webroot -w {self.webroot_dir}'
 
         elif certificate.preferred_challenge == LETSENCRYPT_PREFERRED_CHALLENGE.dns01:
+            self.dns_config_dir.mkdir(parents=True, exist_ok=True)
+
             api_creds = certificate.get_cloudflare_dns_credentials()
-            temp_file.write(api_creds.encode())
-            temp_file.flush()
+            dns_config_path.write_text(api_creds)
 
-            gen_command += f' --dns-cloudflare --dns-cloudflare-credentials {temp_file.name}'
+            gen_command += f' --dns-cloudflare --dns-cloudflare-credentials {dns_config_path.absolute()}'
 
         gen_command += f' --keep-until-expiring --expand'
         gen_command += f' --agree-tos -m "{certificate.email}" --no-eff-email'
@@ -120,16 +120,15 @@ def generate_certificate(self, certificate: LetsencryptSSLCertificate):
             self.logger.exception(e)
             output = '\n'.join(line for line in self.console_output.getvalue().split('\n') if not line.startswith('!!'))
             richprint.stdout.print(output)
+            dns_config_path.unlink()
             raise SSLCertificateChallengeFailed(certificate.preferred_challenge)
 
         except Exception as e:
             self.logger.exception(e)
             output = '\n'.join(line for line in self.console_output.getvalue().split('\n') if not line.startswith('!!'))
             richprint.stdout.print(output)
+            dns_config_path.unlink()
             raise SSLCertificateGenerateFailed()
 
-        finally:
-            temp_file.close()
-
         richprint.print("Acquired Letsencrypt certificate: Done")
         return self.get_certificate_paths(certificate)
diff --git a/poetry.lock b/poetry.lock
index 94812d07..ec79f2cc 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -35,6 +35,25 @@ files = [
     {file = "annotated_types-0.6.0.tar.gz", hash = "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d"},
 ]
 
+[[package]]
+name = "attrs"
+version = "23.2.0"
+description = "Classes Without Boilerplate"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"},
+    {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"},
+]
+
+[package.extras]
+cov = ["attrs[tests]", "coverage[toml] (>=5.3)"]
+dev = ["attrs[tests]", "pre-commit"]
+docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"]
+tests = ["attrs[tests-no-zope]", "zope-interface"]
+tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"]
+tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"]
+
 [[package]]
 name = "certbot"
 version = "2.10.0"
@@ -65,6 +84,27 @@ dev = ["azure-devops", "ipdb", "poetry (>=1.2.0)", "poetry-plugin-export (>=1.1.
 docs = ["Sphinx (>=1.2)", "sphinx-rtd-theme"]
 test = ["coverage", "mypy", "pip", "pylint", "pytest", "pytest-cov", "pytest-xdist", "setuptools", "tox", "types-httplib2", "types-pyOpenSSL", "types-pyRFC3339", "types-pytz", "types-pywin32", "types-requests", "types-setuptools", "types-six", "wheel"]
 
+[[package]]
+name = "certbot-dns-cloudflare"
+version = "2.10.0"
+description = "Cloudflare DNS Authenticator plugin for Certbot"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "certbot-dns-cloudflare-2.10.0.tar.gz", hash = "sha256:45b058c3e515a1853b33dcc3367c12978c0930990a563d26c879d9883a639c07"},
+    {file = "certbot_dns_cloudflare-2.10.0-py3-none-any.whl", hash = "sha256:2b26ed46a1c0f152478b7372f1fc36a4fa1612aa05cde3ab21dada724b32ed40"},
+]
+
+[package.dependencies]
+acme = ">=2.10.0"
+certbot = ">=2.10.0"
+cloudflare = ">=1.5.1"
+setuptools = ">=41.6.0"
+
+[package.extras]
+docs = ["Sphinx (>=1.0)", "sphinx-rtd-theme"]
+test = ["pytest"]
+
 [[package]]
 name = "certifi"
 version = "2024.2.2"
@@ -253,6 +293,24 @@ files = [
 [package.dependencies]
 colorama = {version = "*", markers = "platform_system == \"Windows\""}
 
+[[package]]
+name = "cloudflare"
+version = "2.20.0"
+description = "Python wrapper for the Cloudflare v4 API"
+optional = false
+python-versions = ">3.6.0"
+files = [
+    {file = "cloudflare-2.20.0.tar.gz", hash = "sha256:46aefc39dfaa2365d639b423cec2cd5350ae11153c7247d3eb3545bdcf01a68a"},
+]
+
+[package.dependencies]
+jsonlines = "*"
+pyyaml = "*"
+requests = "*"
+
+[package.extras]
+test = ["pytest"]
+
 [[package]]
 name = "colorama"
 version = "0.4.6"
@@ -439,6 +497,20 @@ pyopenssl = ">=0.13"
 [package.extras]
 docs = ["sphinx (>=4.3.0)", "sphinx-rtd-theme (>=1.0)"]
 
+[[package]]
+name = "jsonlines"
+version = "4.0.0"
+description = "Library with helpers for the jsonlines file format"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "jsonlines-4.0.0-py3-none-any.whl", hash = "sha256:185b334ff2ca5a91362993f42e83588a360cf95ce4b71a73548502bda52a7c55"},
+    {file = "jsonlines-4.0.0.tar.gz", hash = "sha256:0c6d2c09117550c089995247f605ae4cf77dd1533041d366351f6f298822ea74"},
+]
+
+[package.dependencies]
+attrs = ">=19.2.0"
+
 [[package]]
 name = "markdown-it-py"
 version = "3.0.0"
@@ -784,6 +856,66 @@ files = [
     {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"},
 ]
 
+[[package]]
+name = "pyyaml"
+version = "6.0.1"
+description = "YAML parser and emitter for Python"
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"},
+    {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"},
+    {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"},
+    {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"},
+    {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"},
+    {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"},
+    {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"},
+    {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"},
+    {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"},
+    {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"},
+    {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"},
+    {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"},
+    {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"},
+    {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"},
+    {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"},
+    {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
+    {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
+    {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
+    {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"},
+    {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
+    {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
+    {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
+    {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"},
+    {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"},
+    {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"},
+    {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"},
+    {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"},
+    {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"},
+    {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"},
+    {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"},
+    {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"},
+    {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"},
+    {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"},
+    {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"},
+    {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"},
+    {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"},
+    {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"},
+    {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"},
+    {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"},
+    {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"},
+    {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"},
+    {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"},
+    {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"},
+    {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"},
+    {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"},
+    {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"},
+    {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"},
+    {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"},
+    {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"},
+    {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"},
+    {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"},
+]
+
 [[package]]
 name = "requests"
 version = "2.31.0"
@@ -1004,4 +1136,4 @@ zstd = ["zstandard (>=0.18.0)"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "bb03f1df584decf16ac142f49cfd6378a799bbf3638fbef570198c9020e16a88"
+content-hash = "30700255d7d524fddd9d91dba0b9559e3f356ea14ab2d37abcceb4c27156e4a8"

From 3dd7d714eec22fd15757dd2acb23dafca8565ad4 Mon Sep 17 00:00:00 2001
From: Xieyt <alokmilenium@gmail.com>
Date: Sat, 25 May 2024 01:43:17 +0530
Subject: [PATCH 08/14] update example configs

---
 frappe_manager/templates/bench_config.toml | 34 +++++++++++++++-------
 frappe_manager/templates/fm_config.toml    | 18 +++++++++---
 2 files changed, 38 insertions(+), 14 deletions(-)

diff --git a/frappe_manager/templates/bench_config.toml b/frappe_manager/templates/bench_config.toml
index 061216f3..9340eec6 100644
--- a/frappe_manager/templates/bench_config.toml
+++ b/frappe_manager/templates/bench_config.toml
@@ -1,20 +1,34 @@
-name = "example.com"
+# Denotes the bench name.
+# Should not be edited by user.
+name = "fm.com"
 
-# when true enables frappe developer mode
+# Enables Frappe developer mode for the bench.
 developer_mode = false
 
-# when true enables admin tools mode
+# Enables admin tools (e.g., Mailhog and Adminer) for the bench.
 admin_tools = false
 
-# "prod" -> for bench prod environment
-# "dev"  -> for bench dev environment
+# Sets the bench environment to either "prod" (production) or "dev" (development).
 environment_type = "prod"
 
-# enable ssl if defined
 [ssl]
-# ssl_type -> "letsencrypt" / "disable"
+# Sets the SSL type to be used by the bench, in this case, "letsencrypt" for Let's Encrypt.
 ssl_type = "letsencrypt"
-# only applicable if ssl_type is "letsencrypt"
+
+# Controls the HSTS (HTTP Strict Transport Security) header used by the bench.
+# When set to "off", the HSTS header will not be included.
 hsts = "off"
-# only applicable if ssl_type is "letsencrypt"
-email = "exampl@example.com"
+
+# Specifies the preferred Certbot challenge method to be used for Let's Encrypt certificate validation.
+preferred_challenge = "http01"
+
+# The email address associated with Let's Encrypt.
+# This is used for notifications, recovery, and is used with `api_key` for the Global API key of Cloudflare.
+# For more information, see [this documentation](https://certbot-dns-cloudflare.readthedocs.io/en/stable/#certbot-cloudflare-key-ini).
+email = "cloudflare@example.com"
+
+# Cloudflare Global API Key for Let's Encrypt DNS01 Challenge.
+api_token = "0123456789abcdef0123456789abcdef01234567"
+
+# Cloudflare API token for Let's Encrypt DNS01 Challenge.
+api_key = "0123456789abcdef0123456789abcdef01234"
diff --git a/frappe_manager/templates/fm_config.toml b/frappe_manager/templates/fm_config.toml
index bb89fccb..ebe61247 100644
--- a/frappe_manager/templates/fm_config.toml
+++ b/frappe_manager/templates/fm_config.toml
@@ -1,5 +1,15 @@
-# Don't change the version no this will affect the migrations.
-version = "0.13.0" # set by fm to the current version
+# Denotes the current version of fm.
+# Should not be edited by user.
+version = "0.14.0"
 
-# Email used by Let's Encrypt if set then fm will never ask email for Let's Encrypt.
-le_email = 'example@example.com'
+[letsencrypt]
+# The email address associated with Let's Encrypt.
+# This is used for notifications, recovery, and is used with api_key for the Global API key of Cloudflare.
+# For more information, see [this documentation](https://certbot-dns-cloudflare.readthedocs.io/en/stable/#certbot-cloudflare-key-ini).
+email = 'cloudflare@example.co'
+
+# Cloudflare API token for Let's Encrypt DNS01 Challenge.
+api_key = '0123456789abcdef0123456789abcdef01234'
+
+# Cloudflare Global API Key for Let's Encrypt DNS01 Challenge.
+api_token = '0123456789abcdef0123456789abcdef01234567'

From 7a3fffc603e1e49417ec403678b193ed33a05828 Mon Sep 17 00:00:00 2001
From: Xieyt <alokmilenium@gmail.com>
Date: Sat, 25 May 2024 02:49:28 +0530
Subject: [PATCH 09/14] Show letsencrypt preferred challenge in info command

---
 frappe_manager/site_manager/site.py | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py
index 9c5e6bb4..52fab987 100644
--- a/frappe_manager/site_manager/site.py
+++ b/frappe_manager/site_manager/site.py
@@ -32,7 +32,7 @@
     BenchWorkersSupervisorConfigurtionGenerateError,
 )
 from frappe_manager.site_manager.workers_manager.SiteWorker import BenchWorkers
-from frappe_manager.ssl_manager import SUPPORTED_SSL_TYPES
+from frappe_manager.ssl_manager import LETSENCRYPT_PREFERRED_CHALLENGE, SUPPORTED_SSL_TYPES
 from frappe_manager.ssl_manager.certificate import SSLCertificate
 from frappe_manager.ssl_manager.nginxproxymanager import NginxProxyManager
 from frappe_manager.ssl_manager.ssl_certificate_manager import SSLCertificateManager
@@ -671,13 +671,12 @@ def remove_certificate(self):
     def update_certificate(self, certificate: SSLCertificate, raise_error: bool = True):
         if certificate.ssl_type == SUPPORTED_SSL_TYPES.le:
             if self.has_certificate():
-                if not raise_error:
-                    return
-                raise BenchSSLCertificateAlreadyIssued(self.name)
-
-            self.certificate_manager.set_certificate(certificate)
-            self.bench_config.ssl = certificate
-            self.create_certificate()
+                if raise_error:
+                    raise BenchSSLCertificateAlreadyIssued(self.name)
+            else:
+                self.certificate_manager.set_certificate(certificate)
+                self.bench_config.ssl = certificate
+                self.create_certificate()
 
         elif certificate.ssl_type == SUPPORTED_SSL_TYPES.none:
             if self.has_certificate():
@@ -686,6 +685,7 @@ def update_certificate(self, certificate: SSLCertificate, raise_error: bool = Tr
                 if not raise_error:
                     return
                 raise BenchSSLCertificateNotIssued(self.name)
+
         return True
 
     def renew_certificate(self):
@@ -717,6 +717,13 @@ def info(self):
 
         protocol = 'https' if self.has_certificate() else 'http'
 
+        ssl_service_type = f'{self.bench_config.ssl.ssl_type.value}'
+
+        if self.bench_config.ssl.ssl_type == SUPPORTED_SSL_TYPES.le:
+            ssl_service_type = (
+                f'[{self.bench_config.ssl.preferred_challenge.value}] {self.bench_config.ssl.ssl_type.value}'
+            )
+
         data = {
             "Bench Url": f"{protocol}://{self.name}",
             "Bench Root": f"[link=file://{self.path.absolute()}]{self.path.absolute()}[/link]",
@@ -729,7 +736,7 @@ def info(self):
             "DB User": db_user,
             "DB Password": db_pass,
             "Environment": self.bench_config.environment_type.value,
-            "HTTPS": f'{self.bench_config.ssl.ssl_type.value} ({format_ssl_certificate_time_remaining(self.certificate_manager.get_certficate_expiry())})'
+            "HTTPS": f'{ssl_service_type.upper()} ({format_ssl_certificate_time_remaining(self.certificate_manager.get_certficate_expiry())})'
             if self.has_certificate()
             else 'Not Enabled',
         }

From fd636a2390e3509c3d4ae53adab1991f1dab2a71 Mon Sep 17 00:00:00 2001
From: Xieyt <alokmilenium@gmail.com>
Date: Mon, 27 May 2024 15:24:50 +0530
Subject: [PATCH 10/14] fix: certobt cloudflare cred file perm warning

---
 frappe_manager/ssl_manager/letsencrypt_certificate_service.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/frappe_manager/ssl_manager/letsencrypt_certificate_service.py b/frappe_manager/ssl_manager/letsencrypt_certificate_service.py
index c4ebbe40..020e873d 100644
--- a/frappe_manager/ssl_manager/letsencrypt_certificate_service.py
+++ b/frappe_manager/ssl_manager/letsencrypt_certificate_service.py
@@ -95,6 +95,7 @@ def generate_certificate(self, certificate: LetsencryptSSLCertificate):
 
             api_creds = certificate.get_cloudflare_dns_credentials()
             dns_config_path.write_text(api_creds)
+            dns_config_path.chmod(0o600)
 
             gen_command += f' --dns-cloudflare --dns-cloudflare-credentials {dns_config_path.absolute()}'
 

From 6e79882b04e73a902291cfb812e1384a3ac183b7 Mon Sep 17 00:00:00 2001
From: Dipankar Das <65275144+dipankardas011@users.noreply.github.com>
Date: Mon, 27 May 2024 18:12:31 +0530
Subject: [PATCH 11/14] feat: added the all functionality to services restart
 and start as well

Signed-off-by: Dipankar Das <65275144+dipankardas011@users.noreply.github.com>
---
 frappe_manager/services_manager/commands.py | 25 ++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/frappe_manager/services_manager/commands.py b/frappe_manager/services_manager/commands.py
index 03fdc60e..f484aafd 100644
--- a/frappe_manager/services_manager/commands.py
+++ b/frappe_manager/services_manager/commands.py
@@ -36,11 +36,20 @@ def start(
 ):
     """Starts global services."""
     services_manager: ServicesManager = ctx.obj["services"]
-
-    if services_manager.compose_project.is_service_running(service_name.value):
-        services_manager.compose_project.start_service(services=[service_name.value])
+    if service_name.value == ServicesEnum.all:
+        for service in ServicesEnum:
+            if service == ServicesEnum.all:
+                continue
+            if not services_manager.compose_project.is_service_running(service.value):
+                services_manager.compose_project.start_service(services=[service.value])
+            else:
+                richprint.exit(f"{service_name.value} is already running.")
+        return
     else:
-        richprint.exit(f"{service_name.value} is already running.")
+        if not services_manager.compose_project.is_service_running(service_name.value):
+            services_manager.compose_project.start_service(services=[service_name.value])
+        else:
+            richprint.exit(f"{service_name.value} is already running.")
 
 @services_root_command.command(no_args_is_help=True)
 def restart(
@@ -49,7 +58,13 @@ def restart(
 ):
     """Restarts global services."""
     services_manager: ServicesManager = ctx.obj["services"]
-    services_manager.compose_project.restart_service(services=[service_name.value])
+    if service_name.value == ServicesEnum.all:
+        for service in ServicesEnum:
+            if service == ServicesEnum.all:
+                continue
+            services_manager.compose_project.restart_service(services=[service_name.value])
+    else:
+        services_manager.compose_project.restart_service(services=[service_name.value])
 
 
 @services_root_command.command(no_args_is_help=True)

From d18852c07033b0e670293b200a25627c018fbe9a Mon Sep 17 00:00:00 2001
From: Xieyt <alokmilenium@gmail.com>
Date: Wed, 29 May 2024 03:25:11 +0530
Subject: [PATCH 12/14] Fix restart command

---
 frappe_manager/services_manager/commands.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/frappe_manager/services_manager/commands.py b/frappe_manager/services_manager/commands.py
index f484aafd..81a8c560 100644
--- a/frappe_manager/services_manager/commands.py
+++ b/frappe_manager/services_manager/commands.py
@@ -54,17 +54,22 @@ def start(
 @services_root_command.command(no_args_is_help=True)
 def restart(
     ctx: typer.Context,
-    service_name: Annotated[ServicesEnum, typer.Argument(help="Name of the services_manager.")],
+    service_name: Annotated[ServicesEnum, typer.Argument(help="Name of the service.")],
 ):
     """Restarts global services."""
     services_manager: ServicesManager = ctx.obj["services"]
+
     if service_name.value == ServicesEnum.all:
+
         for service in ServicesEnum:
             if service == ServicesEnum.all:
                 continue
-            services_manager.compose_project.restart_service(services=[service_name.value])
+
+            services_manager.compose_project.restart_service(services=[service.value])
+            richprint.print(f"Restarted service {service.value}.")
     else:
         services_manager.compose_project.restart_service(services=[service_name.value])
+        richprint.print(f"Restarted service {service_name.value}.")
 
 
 @services_root_command.command(no_args_is_help=True)

From ed559ff21f33ca6190337418a0087efa5e282887 Mon Sep 17 00:00:00 2001
From: Xieyt <alokmilenium@gmail.com>
Date: Wed, 29 May 2024 04:01:36 +0530
Subject: [PATCH 13/14] Update status msgs and misc fixes.

---
 frappe_manager/commands.py                  | 15 +++++----
 frappe_manager/services_manager/commands.py | 36 ++++++++++++---------
 frappe_manager/services_manager/services.py | 24 +++++++++-----
 frappe_manager/utils/helpers.py             |  5 +--
 4 files changed, 47 insertions(+), 33 deletions(-)

diff --git a/frappe_manager/commands.py b/frappe_manager/commands.py
index 7628ff3c..a9ca986e 100644
--- a/frappe_manager/commands.py
+++ b/frappe_manager/commands.py
@@ -105,35 +105,38 @@ def app_callback(
 
         fm_config_manager: FMConfigManager = FMConfigManager.import_from_toml()
 
+
         # docker pull
         if first_time_install:
             if not fm_config_manager.root_path.exists():
-                richprint.print('🔍 It seems like the first installation. Pulling images... 🖼️')
+                richprint.print("It seems like the first installation. Pulling docker images...️", "🔍")
+
                 completed_status = pull_docker_images()
                 if not completed_status:
                     shutil.rmtree(CLI_DIR)
                     richprint.exit("Aborting. [bold][blue]fm[/blue][/bold] will not be able to work without images. 🖼️")
+
                 current_version = Version(get_current_fm_version())
                 fm_config_manager.version = current_version
                 fm_config_manager.export_to_toml()
 
         migrations = MigrationExecutor(fm_config_manager)
         migration_status = migrations.execute()
+
         if not migration_status:
             richprint.exit(f"Rollbacked to previous version of fm {migrations.prev_version}.")
 
         services_manager: ServicesManager = ServicesManager(verbose=verbose)
+        services_manager.set_typer_context(ctx)
+
         services_manager.init()
+
         try:
-            services_manager.entrypoint_checks()
+            services_manager.entrypoint_checks(start=True)
         except ServicesNotCreated as e:
             services_manager.remove_itself()
             richprint.exit(f"Not able to create services. {e}")
 
-        if not services_manager.compose_project.running:
-            services_manager.are_ports_free()
-            services_manager.compose_project.start_service()
-
         ctx.obj["services"] = services_manager
         ctx.obj["verbose"] = verbose
         ctx.obj['fm_config_manager'] = fm_config_manager
diff --git a/frappe_manager/services_manager/commands.py b/frappe_manager/services_manager/commands.py
index 81a8c560..55779af6 100644
--- a/frappe_manager/services_manager/commands.py
+++ b/frappe_manager/services_manager/commands.py
@@ -9,7 +9,7 @@
 @services_root_command.command(no_args_is_help=True)
 def stop(
     ctx: typer.Context,
-    service_name: Annotated[ServicesEnum, typer.Argument(help="Name of the services_manager.")],
+    service_name: Annotated[ServicesEnum, typer.Argument(help="Name of the service.")],
 ):
     """Stops global services."""
     services_manager: ServicesManager = ctx.obj["services"]
@@ -17,39 +17,45 @@ def stop(
         for service in ServicesEnum:
             if service == ServicesEnum.all:
                 continue
-            if services_manager.compose_project.is_service_running(service.value):
-                services_manager.compose_project.stop_service(services=[service.value])
-            else:
-                richprint.warning(f"{service.value} is not running.")
-        return
+
+            if not services_manager.compose_project.is_service_running(service.value):
+                richprint.print(f"Skipping not running service {service.value}.")
+                continue
+
+            services_manager.compose_project.stop_service(services=[service.value])
+            richprint.print(f"Stopped service {service.value}.")
     else:
         if services_manager.compose_project.is_service_running(service_name.value):
             services_manager.compose_project.stop_service(services=[service_name.value])
         else:
-            richprint.exit(f"{service_name.value} is not running.")
+            richprint.print(f"Skipping already stopped service {service_name.value}.")
 
 
 @services_root_command.command(no_args_is_help=True)
 def start(
     ctx: typer.Context,
-    service_name: Annotated[ServicesEnum, typer.Argument(help="Name of the services_manager.")],
+    service_name: Annotated[ServicesEnum, typer.Argument(help="Name of the service.")],
 ):
     """Starts global services."""
     services_manager: ServicesManager = ctx.obj["services"]
+
     if service_name.value == ServicesEnum.all:
+
         for service in ServicesEnum:
             if service == ServicesEnum.all:
                 continue
-            if not services_manager.compose_project.is_service_running(service.value):
-                services_manager.compose_project.start_service(services=[service.value])
-            else:
-                richprint.exit(f"{service_name.value} is already running.")
-        return
+
+            if services_manager.compose_project.is_service_running(service.value):
+                richprint.print(f"Skipping already running service {service.value}.")
+                continue
+
+            services_manager.compose_project.start_service(services=[service.value])
+            richprint.print(f"Started service {service.value}.")
     else:
         if not services_manager.compose_project.is_service_running(service_name.value):
             services_manager.compose_project.start_service(services=[service_name.value])
         else:
-            richprint.exit(f"{service_name.value} is already running.")
+            richprint.print(f"Skipping already running service {service_name.value}.")
 
 @services_root_command.command(no_args_is_help=True)
 def restart(
@@ -75,7 +81,7 @@ def restart(
 @services_root_command.command(no_args_is_help=True)
 def shell(
     ctx: typer.Context,
-    service_name: Annotated[ServicesEnum, typer.Argument(help="Name of the services_manager.")],
+    service_name: Annotated[ServicesEnum, typer.Argument(help="Name of the service.")],
     user: Annotated[Optional[str], typer.Option(help="Connect as this user.")] = None,
 ):
     """
diff --git a/frappe_manager/services_manager/services.py b/frappe_manager/services_manager/services.py
index 605962ef..ee0c0d04 100644
--- a/frappe_manager/services_manager/services.py
+++ b/frappe_manager/services_manager/services.py
@@ -47,29 +47,34 @@ def __init__(
     def entrypoint_checks(self, start=False):
         if not self.path.exists():
             try:
-                richprint.print(f"Creating services", emoji_code=":construction:")
+                richprint.print(f"Creating global services [blue]{', '.join(self.compose_project.compose_file_manager.get_services_list())}[/blue].", emoji_code=":construction:")
                 self.path.mkdir(parents=True, exist_ok=True)
                 self.create(clean_install=True)
+
             except Exception as e:
-                raise ServicesNotCreated(f'Error Caused: {e}')
+                raise ServicesNotCreated(f"Not able to create global services [blue]{', '.join(self.compose_project.compose_file_manager.get_services_list())}[/blue].")
 
             self.compose_project.pull_images()
 
-            richprint.print(f"Creating services: Done")
+            richprint.print(f"Created global services [blue]{', '.join(self.compose_project.compose_file_manager.get_services_list())}[/blue].")
+
             if start:
                 self.compose_project.start_service()
 
         if not self.compose_path.exists():
             raise ServicesComposeNotExist(
-                f"Seems like services has taken a down. Compose file not found at -> {self.compose_path}. Please recreate services."
+                f"Seems like global services has taken a down. No compose file found at {self.compose_path}."
             )
 
         if start:
             if not self.typer_context.invoked_subcommand == "service":
+
                 if not self.compose_project.running:
-                    richprint.warning("services are not running. Starting it")
+                    self.are_ports_free()
+                    richprint.print(f"Started non running global services [blue]{', '.join(self.compose_project.compose_file_manager.get_services_list())}[/blue].")
                     self.compose_project.start_service()
 
+
         self.database_manager: DatabaseServiceManager = MariaDBManager(
             DatabaseServerServiceInfo.import_from_compose_file('global-db', self.compose_project), self.compose_project
         )
@@ -176,7 +181,7 @@ def create(self, backup: bool = False, clean_install: bool = True):
             try:
                 temp_dir.mkdir(parents=True, exist_ok=True)
             except Exception as e:
-                richprint.exit(f"Failed to create global services bind mount directories. Error: {e}")
+                raise ServicesNotCreated(f"Failed to create global services required dir {temp_dir.absolute()}.")
 
         # populate secrets for db
         db_password_path = self.path / 'secrets' / 'db_password.txt'
@@ -233,7 +238,7 @@ def generate_compose(self, inputs: dict):
 
         # TODO do something about this exception
         except Exception as e:
-            richprint.exit(f"Not able to generate global site compose. Error: {e}")
+            raise ServicesNotCreated(f"Not able to generate global services compose file.")
 
     def shell(self, container: str, user: str | None = None):
         richprint.stop()
@@ -250,5 +255,8 @@ def remove_itself(self):
         shutil.rmtree(self.path)
 
     def are_ports_free(self):
+        ports = [80, 443]
+        richprint.change_head(f"Verifying ports {', '.join(map(str, ports))} availability.")
         docker_used_ports = self.compose_project.get_host_port_binds()
-        check_and_display_port_status([80, 443], exclude=docker_used_ports)
+        check_and_display_port_status(ports, exclude=docker_used_ports)
+        richprint.print(f"Global services will utilize ports {', '.join(map(str, ports))}.")
diff --git a/frappe_manager/utils/helpers.py b/frappe_manager/utils/helpers.py
index 28da2adb..32d30c44 100644
--- a/frappe_manager/utils/helpers.py
+++ b/frappe_manager/utils/helpers.py
@@ -110,7 +110,6 @@ def check_and_display_port_status(ports_to_check: list, exclude=[]):
         ports_to_check (list): List of ports to check.
         exclude (list, optional): List of ports to exclude from checking. Defaults to [].
     """
-    richprint.change_head("Checking Ports")
     if exclude:
         # Removing elements present in remove_array from original_array
         ports_to_check = [x for x in exclude if x not in ports_to_check]
@@ -119,10 +118,8 @@ def check_and_display_port_status(ports_to_check: list, exclude=[]):
         already_binded = check_ports(ports_to_check)
         if already_binded:
             richprint.exit(
-                f"Whoa there! Looks like the {' '.join([ str(x) for x in already_binded ])} { 'ports are' if len(already_binded) > 1 else 'port is' } having a party already! Can you do us a solid and free up those ports?"
+                f"Ports {', '.join(map(str, already_binded))} {'are' if len(already_binded) > 1 else 'is'} currently in use. Please free up these ports."
             )
-    richprint.print("Ports Check : Passed")
-
 
 def generate_random_text(length=50):
     """

From c3683002ba41195966dfd61d0c5cf4554595328e Mon Sep 17 00:00:00 2001
From: Dipankar Das <65275144+dipankardas011@users.noreply.github.com>
Date: Thu, 23 May 2024 18:33:53 +0530
Subject: [PATCH 14/14] feat(e2e): added e2e script for testing

Signed-off-by: Dipankar Das <65275144+dipankardas011@users.noreply.github.com>
---
 .github/workflows/e2e-migration.yml | 98 +++++++++++++++++++++++++++++
 .github/workflows/e2e-site.yaml     | 54 ++++++++++++++++
 test/e2e_test.sh                    | 36 +++++++++++
 test/fm.sh                          | 61 ++++++++++++++++++
 test/helpers.sh                     | 22 +++++++
 test/migration_test.sh              | 41 ++++++++++++
 6 files changed, 312 insertions(+)
 create mode 100644 .github/workflows/e2e-migration.yml
 create mode 100644 .github/workflows/e2e-site.yaml
 create mode 100755 test/e2e_test.sh
 create mode 100755 test/fm.sh
 create mode 100755 test/helpers.sh
 create mode 100755 test/migration_test.sh

diff --git a/.github/workflows/e2e-migration.yml b/.github/workflows/e2e-migration.yml
new file mode 100644
index 00000000..8cc1e5d3
--- /dev/null
+++ b/.github/workflows/e2e-migration.yml
@@ -0,0 +1,98 @@
+name: E2E migration testing
+
+on:
+  push:
+    tags:
+      - v*
+    branches:
+      - main
+      - develop
+  workflow_dispatch:
+
+jobs:
+  e2e-migration-from-0_9_0-to-latest:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        python-ver: ["3.11", "3.12"]
+        os: [ubuntu-latest]
+        # os: [self-hosted-arm64, ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-ver }}
+
+      - name: header
+        run: |
+          echo -e "\e[1;33m┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\e[0m
+          \e[1;33m┃\e[0m \e[1;36m Migration from v0.9.0 to latest \e[0m \e[1;33m┃\e[0m
+          \e[1;33m┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛\e[0m"
+
+      - name: install frappe from v0.9.0
+        run: |
+          cd /tmp
+          python -m pip install -U git+https://github.com/rtCamp/Frappe-Manager.git@v0.9.0
+
+      - name: frappe version
+        run: fm --version
+
+      - name: e2e run
+        timeout-minutes: 20
+        working-directory: test
+        run: |
+          ./migration_test.sh oldToNew
+
+      - name: cleanup
+        if: always()
+        run: |
+          python -m pip uninstall -y frappe-manager
+          sudo rm -rf ~/frappe
+
+  e2e-migration-from-before_latest-to-latest:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        python-ver: ["3.11", "3.12"]
+        os: [ubuntu-latest]
+        # os: [self-hosted-arm64, ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-ver }}
+
+      - name: install jq
+        run: |
+          sudo apt upgrade -y
+          sudo apt install -y jq curl
+      - name: header
+        run: |
+          echo -e "\e[1;33m┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\e[0m
+          \e[1;33m┃\e[0m \e[1;36m Migration from latest~1 to latest \e[0m \e[1;33m┃\e[0m
+          \e[1;33m┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛\e[0m"
+
+      - name: install frappe previous the latest release
+        run: |
+          python -m pip install -U git+https://github.com/rtCamp/Frappe-Manager.git@$(curl --silent https://api.github.com/repos/rtCamp/Frappe-Manager/tags | jq -r '.[1].name')
+
+      - name: frappe version
+        run: fm --version
+
+      - name: e2e run
+        timeout-minutes: 30
+        working-directory: test
+        run: |
+          ./migration_test.sh semiNewToNew
+
+      - name: cleanup
+        if: always()
+        run: |
+          python -m pip uninstall -y frappe-manager
+          sudo rm -rf ~/frappe
diff --git a/.github/workflows/e2e-site.yaml b/.github/workflows/e2e-site.yaml
new file mode 100644
index 00000000..e00c0772
--- /dev/null
+++ b/.github/workflows/e2e-site.yaml
@@ -0,0 +1,54 @@
+name: E2E site testing
+
+on:
+  push:
+    tags:
+      - v*
+    branches:
+      - main
+      - develop
+  workflow_dispatch:
+
+jobs:
+  e2e-current:
+    name: e2e current latest branch
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        python-ver: ["3.11", "3.12"]
+        os: [ubuntu-latest]
+        # os: [self-hosted-arm64, ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-ver }}
+
+      - name: header
+        run: |
+          echo -e "\e[1;33m┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\e[0m
+          \e[1;33m┃\e[0m \e[1;36m E2E Test \e[0m \e[1;33m┃\e[0m
+          \e[1;33m┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛\e[0m"
+
+      - name: install frappe from current commit
+        run: |
+          cd ..
+          python -m pip install --upgrade ./Frappe-Manager
+
+      - name: frappe version
+        run: fm --version
+
+      - name: e2e run
+        timeout-minutes: 20
+        working-directory: test
+        run: |
+          ./e2e_test.sh
+
+      - name: cleanup
+        if: always()
+        run: |
+          python -m pip uninstall -y frappe-manager
+          sudo rm -rf ~/frappe
diff --git a/test/e2e_test.sh b/test/e2e_test.sh
new file mode 100755
index 00000000..c3131bb6
--- /dev/null
+++ b/test/e2e_test.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+PS4='+\[\033[0;33m\](\[\033[0;36m\]${BASH_SOURCE##*/}:${LINENO}\[\033[0;33m\])\[\033[0m\] '
+
+set -xe
+
+source ${PWD}/fm.sh
+source ${PWD}/helpers.sh
+
+main() {
+	Prequisites
+
+	CreateSite "test-site.prod.local" prod
+	CreateSite "test-site.dev.local" dev
+
+	ListSites
+
+	StopSite "test-site.prod.local"
+	StopSite "test-site.dev.local"
+
+	StartSite "test-site.prod.local"
+	StartSite "test-site.dev.local"
+
+	TestSiteReachability "test-site.prod.local"
+	TestSiteReachability "test-site.dev.local"
+
+	GetInfoSite "test-site.prod.local"
+	GetInfoSite "test-site.dev.local"
+
+	DeleteSite "test-site.prod.local"
+	DeleteSite "test-site.dev.local"
+
+	RemoveDanglingDockerStuff
+}
+
+time main
diff --git a/test/fm.sh b/test/fm.sh
new file mode 100755
index 00000000..03a5e230
--- /dev/null
+++ b/test/fm.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+CreateSite() {
+	local siteName="$1"
+	local envN="$2"
+	echo "Create SiteName: $siteName, Env: $envN"
+	if [ ! "${envN:-}" ]; then
+	   fm create $siteName
+	else
+	   fm create $siteName --env $2
+	fi
+
+	echo "Get Request to the homepage of Site: $siteName, Env: $envN"
+	TestSiteReachability "$siteName"
+}
+
+TestSiteReachability() {
+	local siteName="$1"
+	curl -f \
+	    --retry 18 --retry-max-time 600 \
+	    --head \
+	    -H "Host: $siteName" \
+	    -H "Cache-Control: no-cache,no-store" \
+	    http://localhost:80
+}
+
+MigrationToLatest() {
+    pip install -U frappe-manager
+    echo "yes" | fm list
+    fm --version
+}
+
+DeleteSite() {
+    local siteName="$1"
+    echo "Delete SiteName: $siteName"
+    echo "yes" | fm delete $siteName
+}
+
+
+GetInfoSite() {
+    local siteName="$1"
+    echo "Info SiteName: $siteName"
+    fm info $siteName
+}
+
+ListSites() {
+    echo "List Sites"
+    fm list
+}
+
+StartSite() {
+    local siteName="$1"
+    echo "Start SiteName: $siteName"
+    fm start $siteName
+}
+
+StopSite() {
+    local siteName="$1"
+    echo "Stop SiteName: $siteName"
+    fm stop $siteName
+}
diff --git a/test/helpers.sh b/test/helpers.sh
new file mode 100755
index 00000000..cade6583
--- /dev/null
+++ b/test/helpers.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+Prequisites() {
+    if [ "$(id -u)" -eq 0 ]; then
+        info_red "You are running as root."
+        exit 30
+    fi
+
+    for n in git python docker docker-compose;
+    do
+        if ! [ -x "$(command -v $n)" ]; then
+            echo "Error: $n is not installed." >&2
+            exit 56
+        fi
+    done
+}
+
+RemoveDanglingDockerStuff() {
+   docker volume rm -f $(docker volume ls -q) || echo "Failed to delete dangling docker volume"
+   docker rm -f $(docker ps -aq) || echo "Failed to delete dangling docker container"
+   docker network rm -f $(docker network ls -q) || echo "Done deleting the networks"
+}
diff --git a/test/migration_test.sh b/test/migration_test.sh
new file mode 100755
index 00000000..0e4c3db4
--- /dev/null
+++ b/test/migration_test.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+PS4='+\[\033[0;33m\](\[\033[0;36m\]${BASH_SOURCE##*/}:${LINENO}\[\033[0;33m\])\[\033[0m\] '
+
+set -xe
+
+
+source ${PWD}/fm.sh
+source ${PWD}/helpers.sh
+
+oldToNew() {
+	Prequisites
+	CreateSite "migration-site.localhost"
+	ListSites
+	StopSite "migration-site.localhost"
+	StartSite "migration-site.localhost"
+	GetInfoSite "migration-site.localhost"
+	MigrationToLatest
+	StartSite "migration-site.localhost"
+	TestSiteReachability "migration-site.localhost"
+	DeleteSite "migration-site.localhost"
+	RemoveDanglingDockerStuff
+}
+
+semiNewToNew() {
+	Prequisites
+	CreateSite "migration-site.dev.local" dev
+	ListSites
+	StopSite "migration-site.dev.local"
+	StartSite "migration-site.dev.local"
+	GetInfoSite "migration-site.dev.local"
+	MigrationToLatest
+	StartSite "migration-site.dev.local"
+	TestSiteReachability "migration-site.dev.local"
+	DeleteSite "migration-site.dev.local"
+	RemoveDanglingDockerStuff
+}
+
+time $1
+# time oldToNew
+# time semiNewToNew