From 30053f251fd9634eb2875e46de4e91667a36fd3d Mon Sep 17 00:00:00 2001 From: bob Date: Thu, 13 Jul 2023 09:51:38 +0000 Subject: [PATCH 1/7] add aws secretmanager support --- .gitignore | 5 ++++- requirements.txt | 3 ++- sql/engines/__init__.py | 15 +++++++++++++-- sql/models.py | 1 + src/init_sql/v1.9.3.sql | 1 + 5 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 src/init_sql/v1.9.3.sql diff --git a/.gitignore b/.gitignore index 09b51e97cd..f50115ed2e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,9 +8,12 @@ archery/settings.py.github archery/settings.py.dev archery/settings_dev.py sql/migrations/ +static/ +nohup.out +supervisord.pid venv env sonar-project.properties .scannerwork .env -local_settings.py \ No newline at end of file +local_settings.py diff --git a/requirements.txt b/requirements.txt index 8063aada18..73f053fc6a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,4 +38,5 @@ django-environ==0.8.1 alibabacloud_dysmsapi20170525==2.0.9 tencentcloud-sdk-python==3.0.656 mozilla-django-oidc==3.0.0 -django-auth-dingding==0.0.2 \ No newline at end of file +django-auth-dingding==0.0.2 +boto3==1.26.103 \ No newline at end of file diff --git a/sql/engines/__init__.py b/sql/engines/__init__.py index 3223f9492d..b2d4d3bc7e 100644 --- a/sql/engines/__init__.py +++ b/sql/engines/__init__.py @@ -1,7 +1,8 @@ """engine base库, 包含一个``EngineBase`` class和一个get_engine函数""" from sql.engines.models import ResultSet, ReviewSet from sql.utils.ssh_tunnel import SSHConnection - +import boto3 +import simplejson as json class EngineBase: """enginebase 只定义了init函数和若干方法的名字, 具体实现用mysql.py pg.py等实现""" @@ -20,7 +21,17 @@ def __init__(self, instance=None): self.password = instance.password self.db_name = instance.db_name self.mode = instance.mode - + self.awsSecretId = instance.awsSecretId + + if self.awsSecretId.strip(): + client = boto3.client('secretsmanager') + response = client.get_secret_value( + SecretId=self.awsSecretId + ) + secret = json.loads(response['SecretString']) + self.user = secret["username"] + self.password = secret["password"] + # 判断如果配置了隧道则连接隧道，只测试了MySQL if self.instance.tunnel: self.ssh = SSHConnection( diff --git a/sql/models.py b/sql/models.py index c5daa6eedd..9815371662 100755 --- a/sql/models.py +++ b/sql/models.py @@ -201,6 +201,7 @@ class Instance(models.Model): "Oracle service name", max_length=50, null=True, blank=True ) sid = models.CharField("Oracle sid", max_length=50, null=True, blank=True) + awsSecretId = models.CharField("AWS Secret Id", max_length=50, null=True, blank=True) resource_group = models.ManyToManyField( ResourceGroup, verbose_name="资源组", blank=True ) diff --git a/src/init_sql/v1.9.3.sql b/src/init_sql/v1.9.3.sql new file mode 100644 index 0000000000..776dd9029f --- /dev/null +++ b/src/init_sql/v1.9.3.sql @@ -0,0 +1 @@ +ALTER TABLE sql_instance ADD awsSecretId varchar(100) NULL; From 296726e2a3642ab6a16fac8d37d933b73d16f08b Mon Sep 17 00:00:00 2001 From: bob Date: Thu, 13 Jul 2023 13:22:31 +0000 Subject: [PATCH 2/7] fix NoneType error --- sql/engines/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/engines/__init__.py b/sql/engines/__init__.py index b2d4d3bc7e..5056bf73ef 100644 --- a/sql/engines/__init__.py +++ b/sql/engines/__init__.py @@ -23,7 +23,7 @@ def __init__(self, instance=None): self.mode = instance.mode self.awsSecretId = instance.awsSecretId - if self.awsSecretId.strip(): + if not self.awsSecretId == None and self.awsSecretId.strip(): client = boto3.client('secretsmanager') response = client.get_secret_value( SecretId=self.awsSecretId From d01a5c1e7e74c95155008b6166b157b5b5bfbd95 Mon Sep 17 00:00:00 2001 From: Bob Date: Fri, 14 Jul 2023 14:15:39 +0800 Subject: [PATCH 3/7] print log --- sql/engines/goinception.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/engines/goinception.py b/sql/engines/goinception.py index 419a866d76..2886ec24c1 100644 --- a/sql/engines/goinception.py +++ b/sql/engines/goinception.py @@ -182,6 +182,7 @@ def query_print(self, instance, db_name=None, sql=""): use `{db_name}`; {sql.rstrip(';')}; inception_magic_commit;""" + logger.info(f"执行goInception语法树打印语句：{sql}") print_info = self.query(db_name=db_name, sql=sql).to_dict()[1] if print_info.get("errmsg"): raise RuntimeError(print_info.get("errmsg")) From 3fe19bfab3933e1d2132686801e205b07904b35e Mon Sep 17 00:00:00 2001 From: Bob Date: Fri, 14 Jul 2023 15:20:44 +0800 Subject: [PATCH 4/7] fix goinception remote username and password using secret --- sql/engines/__init__.py | 10 ++++++++++ sql/engines/goinception.py | 1 - 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/sql/engines/__init__.py b/sql/engines/__init__.py index 5056bf73ef..ec876824bb 100644 --- a/sql/engines/__init__.py +++ b/sql/engines/__init__.py @@ -73,6 +73,16 @@ def remote_instance_conn(self, instance=None): self.remote_port = instance.port self.remote_user = instance.user self.remote_password = instance.password + + if not instance.awsSecretId == None and instance.awsSecretId.strip(): + client = boto3.client('secretsmanager') + response = client.get_secret_value( + SecretId=instance.awsSecretId + ) + secret = json.loads(response['SecretString']) + self.remote_user = secret["username"] + self.remote_password = secret["password"] + return ( self.remote_host, self.remote_port, diff --git a/sql/engines/goinception.py b/sql/engines/goinception.py index 2886ec24c1..419a866d76 100644 --- a/sql/engines/goinception.py +++ b/sql/engines/goinception.py @@ -182,7 +182,6 @@ def query_print(self, instance, db_name=None, sql=""): use `{db_name}`; {sql.rstrip(';')}; inception_magic_commit;""" - logger.info(f"执行goInception语法树打印语句：{sql}") print_info = self.query(db_name=db_name, sql=sql).to_dict()[1] if print_info.get("errmsg"): raise RuntimeError(print_info.get("errmsg")) From ae5ba5897197dfb0e67ee50560d445992c3d85c7 Mon Sep 17 00:00:00 2001 From: Bob Date: Fri, 14 Jul 2023 15:50:05 +0800 Subject: [PATCH 5/7] add ssl conn support to redis --- sql/engines/__init__.py | 1 + sql/engines/redis.py | 2 ++ sql/models.py | 1 + src/init_sql/v1.9.3.sql | 3 ++- 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/sql/engines/__init__.py b/sql/engines/__init__.py index ec876824bb..fb8a5c0900 100644 --- a/sql/engines/__init__.py +++ b/sql/engines/__init__.py @@ -22,6 +22,7 @@ def __init__(self, instance=None): self.db_name = instance.db_name self.mode = instance.mode self.awsSecretId = instance.awsSecretId + self.is_ssl = instance.is_ssl if not self.awsSecretId == None and self.awsSecretId.strip(): client = boto3.client('secretsmanager') diff --git a/sql/engines/redis.py b/sql/engines/redis.py index 78072208bb..e8b8d6410e 100644 --- a/sql/engines/redis.py +++ b/sql/engines/redis.py @@ -33,6 +33,7 @@ def get_connection(self, db_name=None): encoding_errors="ignore", decode_responses=True, socket_connect_timeout=10, + ssl=self.is_ssl, ) else: return redis.Redis( @@ -43,6 +44,7 @@ def get_connection(self, db_name=None): encoding_errors="ignore", decode_responses=True, socket_connect_timeout=10, + ssl=self.is_ssl, ) @property diff --git a/sql/models.py b/sql/models.py index 9815371662..f3fde57881 100755 --- a/sql/models.py +++ b/sql/models.py @@ -195,6 +195,7 @@ class Instance(models.Model): password = fields.EncryptedCharField( verbose_name="密码", max_length=300, default="", blank=True ) + is_ssl = models.BooleanField("是否启用SSL", default=False) db_name = models.CharField("数据库", max_length=64, default="", blank=True) charset = models.CharField("字符集", max_length=20, default="", blank=True) service_name = models.CharField( diff --git a/src/init_sql/v1.9.3.sql b/src/init_sql/v1.9.3.sql index 776dd9029f..73aac00ac7 100644 --- a/src/init_sql/v1.9.3.sql +++ b/src/init_sql/v1.9.3.sql @@ -1 +1,2 @@ -ALTER TABLE sql_instance ADD awsSecretId varchar(100) NULL; +ALTER TABLE sql_instance ADD awsSecretId varchar(100) DEFAULT '' COMMENT 'AWS SecretId'; +ALTER TABLE sql_instance ADD is_ssl tinyint(1) DEFAULT 0 COMMENT '是否启用SSL'; \ No newline at end of file From 94b485c245d20980c2b99089fd22fc4c0f0adab7 Mon Sep 17 00:00:00 2001 From: Bob Date: Mon, 17 Jul 2023 11:19:44 +0800 Subject: [PATCH 6/7] correct lint format --- .gitignore | 1 + sql/engines/__init__.py | 22 +++++++++------------- sql/models.py | 4 +++- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index f50115ed2e..9481b98969 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.pyc +.venv/ *.swp *.lock *.log diff --git a/sql/engines/__init__.py b/sql/engines/__init__.py index fb8a5c0900..ac8ffcc629 100644 --- a/sql/engines/__init__.py +++ b/sql/engines/__init__.py @@ -23,16 +23,14 @@ def __init__(self, instance=None): self.mode = instance.mode self.awsSecretId = instance.awsSecretId self.is_ssl = instance.is_ssl - + if not self.awsSecretId == None and self.awsSecretId.strip(): - client = boto3.client('secretsmanager') - response = client.get_secret_value( - SecretId=self.awsSecretId - ) - secret = json.loads(response['SecretString']) + client = boto3.client("secretsmanager") + response = client.get_secret_value(SecretId=instance.awsSecretId) + secret = json.loads(response["SecretString"]) self.user = secret["username"] self.password = secret["password"] - + # 判断如果配置了隧道则连接隧道，只测试了MySQL if self.instance.tunnel: self.ssh = SSHConnection( @@ -74,13 +72,11 @@ def remote_instance_conn(self, instance=None): self.remote_port = instance.port self.remote_user = instance.user self.remote_password = instance.password - + if not instance.awsSecretId == None and instance.awsSecretId.strip(): - client = boto3.client('secretsmanager') - response = client.get_secret_value( - SecretId=instance.awsSecretId - ) - secret = json.loads(response['SecretString']) + client = boto3.client("secretsmanager") + response = client.get_secret_value(SecretId=instance.awsSecretId) + secret = json.loads(response["SecretString"]) self.remote_user = secret["username"] self.remote_password = secret["password"] diff --git a/sql/models.py b/sql/models.py index f3fde57881..9ade232048 100755 --- a/sql/models.py +++ b/sql/models.py @@ -202,7 +202,9 @@ class Instance(models.Model): "Oracle service name", max_length=50, null=True, blank=True ) sid = models.CharField("Oracle sid", max_length=50, null=True, blank=True) - awsSecretId = models.CharField("AWS Secret Id", max_length=50, null=True, blank=True) + awsSecretId = models.CharField( + "AWS Secret Id", max_length=50, null=True, blank=True + ) resource_group = models.ManyToManyField( ResourceGroup, verbose_name="资源组", blank=True ) From 2829fb1fad278531fe927b74105c37bd302c0d0e Mon Sep 17 00:00:00 2001 From: Bob Date: Mon, 17 Jul 2023 11:21:38 +0800 Subject: [PATCH 7/7] correct lint format --- sql/engines/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/engines/__init__.py b/sql/engines/__init__.py index ac8ffcc629..d8b692c76a 100644 --- a/sql/engines/__init__.py +++ b/sql/engines/__init__.py @@ -4,6 +4,7 @@ import boto3 import simplejson as json + class EngineBase: """enginebase 只定义了init函数和若干方法的名字, 具体实现用mysql.py pg.py等实现"""