add generic webhook (hhyo#2333)

* add generic webhook draft

* move notify test out

* change signature, add tests

* delete auto notify usage

* setup can cover the slow query table creation

* 增加测试覆盖率

* fix notify_for_audit usage

* more info in generic webhook

* fix api response

* support setting notifier with env

.coveragerc

@@ -1,9 +1,8 @@
 [run]
-include =
-    common*
-    sql*
-    sql_api*
+source =
+    .
 omit =
     src*
     downloads*
     sql/migrations/*
+    venv*

.env.list

@@ -13,6 +13,6 @@ AUTH_LDAP_USER_ATTR_MAP=username=cn,display=displayname,email=email
 CSRF_TRUSTED_ORIGINS=http://127.0.0.1:9123
 
 # https://django-q.readthedocs.io/en/latest/configure.html#
-Q_CLUISTER_WORKERS=4
-Q_CLUISTER_TIMEOUT=60
+Q_CLUSTER_WORKERS=4
+Q_CLUSTER_TIMEOUT=60
 Q_CLUISTER_SYNC=false

.github/workflows/django.yml

@@ -78,13 +78,12 @@ jobs:
       run: |
         mysql -h127.0.0.1 -uroot -e "CREATE DATABASE archery CHARSET UTF8MB4;"
         mysql -h127.0.0.1 -uroot -e "DROP DATABASE IF EXISTS test_archery;CREATE DATABASE test_archery CHARSET UTF8MB4;"
-        mysql -h127.0.0.1 -uroot test_archery<src/init_sql/mysql_slow_query_review.sql
-      
+
     - name: Run Tests
       run: |
         python manage.py makemigrations
         python manage.py makemigrations sql
-        coverage run --source='.' manage.py test -v 3 --keepdb
+        coverage run manage.py test -v 3 --keepdb
         coverage xml
 
     - name: Upload coverage to Codecov

archery/settings.py

@@ -54,6 +54,18 @@
             "cassandra",
         ],
     ),
+    ENABLED_NOTIFIERS=(
+        list,
+        [
+            "sql.notify:DingdingWebhookNotifier",
+            "sql.notify:DingdingPersonNotifier",
+            "sql.notify:FeishuWebhookNotifier",
+            "sql.notify:FeishuPersonNotifier",
+            "sql.notify:QywxWebhookNotifier",
+            "sql.notify:MailNotifier",
+            "sql.notify:GenericWebhookNotifier",
+        ],
+    ),
 )
 
 # SECURITY WARNING: keep the secret key used in production secret!
@@ -86,6 +98,9 @@
     "phoenix": {"path": "sql.engines.phoenix:PhoenixEngine"},
     "odps": {"path": "sql.engines.odps:ODPSEngine"},
 }
+
+ENABLED_NOTIFIERS = env("ENABLED_NOTIFIERS")
+
 ENABLED_ENGINES = env("ENABLED_ENGINES")
 
 # Application definition

common/config.py

@@ -15,7 +15,6 @@
 class SysConfig(object):
     def __init__(self):
         self.sys_config = {}
-        self.get_all_config()
 
     def get_all_config(self):
         try:
@@ -34,10 +33,28 @@ def get_all_config(self):
             self.sys_config = {}
 
     def get(self, key, default_value=None):
-        value = self.sys_config.get(key, default_value)
+        value = self.sys_config.get(key)
+        if value:
+            return value
+        # 尝试去数据库里取
+        config_entry = Config.objects.filter(item=key).last()
+        if config_entry:
+            # 清洗成 python 的 bool
+            value = self.filter_bool(config_entry.value)
         # 是字符串的话, 如果是空, 或者全是空格, 返回默认值
         if isinstance(value, str) and value.strip() == "":
             return default_value
+        if value is not None:
+            self.sys_config[key] = value
+            return value
+        return default_value
+
+    @staticmethod
+    def filter_bool(value: str):
+        if value.lower() == "true":
+            return True
+        if value.lower() == "false":
+            return False
         return value
 
     def set(self, key, value):

common/templates/config.html

@@ -736,6 +736,17 @@ <h5 style="color: darkgrey"><b>工单通知</b></h5>
                                     </div>
                                 </div>
                             </div>
+                            <div class="form-group">
+                                <label for="generic_webhook_url"
+                                       class="col-sm-4 control-label">通用Webhook地址</label>
+                                <div class="col-sm-5">
+                                    <input class="form-control"
+                                           id="generic_webhook_url"
+                                           key="generic_webhook_url"
+                                           value="{{ config.generic_webhook_url }}"
+                                           placeholder="通用 webhook 地址, 每个工单的每个状态转变均会调用" />
+                                </div>
+                            </div>
                             <h5 style="color: darkgrey"><b>短信服务</b></h5>
                             <hr/>
                             <div class="form-group">

common/tests.py

@@ -509,15 +509,6 @@ def testWorkflowByUser(self):
         expected_rows = ((self.u2.display, 3), (self.u1.display, 2))
         self.assertEqual(result["rows"], expected_rows)
 
-    def testDashboard(self):
-        """Dashboard测试"""
-        # TODO 这部分测试并没有遵循单元测试, 而是某种集成测试, 直接从响应到结果, 并且只检查状态码
-        # TODO 需要具体查看pyecharst有没有被调用, 以及调用的参数
-        c = Client()
-        c.force_login(self.superuser1)
-        r = c.get("/dashboard/")
-        self.assertEqual(r.status_code, 200)
-
 
 class AuthTest(TestCase):
     def setUp(self):

sql/archiver.py

@@ -202,20 +202,23 @@ def archive_apply(request):
             )
             archive_id = archive_info.id
             # 调用工作流插入审核信息
-            audit_result = Audit.add(WorkflowDict.workflow_type["archive"], archive_id)
+            audit_result, audit_detail = Audit.add(
+                WorkflowDict.workflow_type["archive"], archive_id
+            )
     except Exception as msg:
         logger.error(traceback.format_exc())
         result["status"] = 1
         result["msg"] = str(msg)
     else:
         result = audit_result
         # 消息通知
-        audit_id = Audit.detail_by_workflow_id(
+        workflow_audit = Audit.detail_by_workflow_id(
             workflow_id=archive_id, workflow_type=WorkflowDict.workflow_type["archive"]
-        ).audit_id
+        )
         async_task(
             notify_for_audit,
-            audit_id=audit_id,
+            workflow_audit=workflow_audit,
+            workflow_audit_detail=audit_detail,
             timeout=60,
             task_name=f"archive-apply-{archive_id}",
         )
@@ -245,15 +248,17 @@ def archive_audit(request):
     # 使用事务保持数据一致性
     try:
         with transaction.atomic():
-            audit_id = Audit.detail_by_workflow_id(
+            workflow_audit = Audit.detail_by_workflow_id(
                 workflow_id=archive_id,
                 workflow_type=WorkflowDict.workflow_type["archive"],
-            ).audit_id
+            )
+            audit_id = workflow_audit.audit_id
 
             # 调用工作流插入审核信息，更新业务表审核状态
-            audit_status = Audit.audit(
+            audit_status, workflow_audit_detail = Audit.audit(
                 audit_id, audit_status, user.username, audit_remark
-            )["data"]["workflow_status"]
+            )
+            audit_status = audit_status["data"]["workflow_status"]
             ArchiveConfig(
                 id=archive_id,
                 status=audit_status,
@@ -267,10 +272,11 @@ def archive_audit(request):
         return render(request, "error.html", context)
     else:
         # 消息通知
+        workflow_audit.refresh_from_db()
         async_task(
             notify_for_audit,
-            audit_id=audit_id,
-            audit_remark=audit_remark,
+            workflow_audit=workflow_audit,
+            workflow_audit_detail=workflow_audit_detail,
             timeout=60,
             task_name=f"archive-audit-{archive_id}",
         )