From 20b03cc4c197060440a04aec03c3c140817e109c Mon Sep 17 00:00:00 2001
From: waylon <1158341873@qq.com>
Date: Tue, 28 May 2024 14:45:13 +0800
Subject: [PATCH] Merge master to dependencies update branch
---
api/collections/nodeman.py | 8 +-
.../pages/task/TaskExecute/ExecuteInfo.vue | 18 ++--
.../ExecuteInfo/ExecuteInfoForm.vue | 88 ++++++++-------
.../ExecuteInfo/JsonschemaForm.vue | 5 +-
.../NodeConfig/JsonschemaInputParams.vue | 5 +-
.../TemplateEdit/NodeConfig/NodeConfig.vue | 100 ++++++++++--------
.../TabGlobalVariables/VariableEdit.vue | 2 +-
gcloud/utils/data_handler.py | 20 ++++
pipeline_plugins/cmdb_ip_picker/query.py | 47 ++++----
.../collections/sites/open/ip_filter_base.py | 48 +++++----
10 files changed, 193 insertions(+), 148 deletions(-)
create mode 100644 gcloud/utils/data_handler.py
diff --git a/api/collections/nodeman.py b/api/collections/nodeman.py
index b97ecb678c..01fd0eec1c 100644
--- a/api/collections/nodeman.py
+++ b/api/collections/nodeman.py
@@ -126,5 +126,9 @@ def get_rsa_public_key(self, executor):
def install_channel(self):
return self._request(method="get", url=_get_nodeman_api_v2("install_channel"), data={})
- def get_ipchooser_host_details(self, params: dict):
- return self._request(method="post", url=_get_nodeman_api("core/api/ipchooser_host/details"), data=params)
+ def get_ipchooser_host_details(self, all_scope: bool, host_list: list):
+ return self._request(
+ method="post",
+ url=_get_nodeman_api("core/api/ipchooser_host/details"),
+ data={"all_scope": all_scope, "host_list": host_list},
+ )
diff --git a/frontend/desktop/src/pages/task/TaskExecute/ExecuteInfo.vue b/frontend/desktop/src/pages/task/TaskExecute/ExecuteInfo.vue
index 51914a8a77..c22f17c8eb 100644
--- a/frontend/desktop/src/pages/task/TaskExecute/ExecuteInfo.vue
+++ b/frontend/desktop/src/pages/task/TaskExecute/ExecuteInfo.vue
@@ -874,11 +874,11 @@
}
this.pluginOutputs = outputs
this.outputRenderConfig = [...storeOutputs, ...outputs]
+ // 设置host
+ const { origin } = window.location
+ const hostUrl = `${origin + window.SITE_URL}plugin_service/data_api/${this.thirdPartyNodeCode}/`
+ $.context.bk_plugin_api_host[this.thirdPartyNodeCode] = hostUrl
if (forms.renderform) {
- // 设置host
- const { origin } = window.location
- const hostUrl = `${origin + window.SITE_URL}plugin_service/data_api/${this.thirdPartyNodeCode}/`
- $.context.bk_plugin_api_host[this.thirdPartyNodeCode] = hostUrl
// 输入参数
const renderFrom = forms.renderform
/* eslint-disable-next-line */
@@ -982,11 +982,11 @@
if (!resp.result) return
// 获取参数
const { forms, inputs } = resp.data
+ // 获取host
+ const { origin } = window.location
+ const hostUrl = `${origin + window.SITE_URL}plugin_service/data_api/${plugin}/`
+ $.context.bk_plugin_api_host[plugin] = hostUrl
if (forms.renderform) {
- // 获取host
- const { origin } = window.location
- const hostUrl = `${origin + window.SITE_URL}plugin_service/data_api/${plugin}/`
- $.context.bk_plugin_api_host[plugin] = hostUrl
// 输入参数
$.atoms[plugin] = {}
const renderFrom = forms.renderform
@@ -1169,7 +1169,7 @@
jsFlowInstance && jsFlowInstance.setCanvasPosition(offsetX, offsetY, true)
}
},
-
+
toggleNodeActive (id, isActive) {
const node = document.getElementById(id)
if (!id || !node) return
diff --git a/frontend/desktop/src/pages/task/TaskExecute/ExecuteInfo/ExecuteInfoForm.vue b/frontend/desktop/src/pages/task/TaskExecute/ExecuteInfo/ExecuteInfoForm.vue
index 8fef9f58b5..38574a2651 100644
--- a/frontend/desktop/src/pages/task/TaskExecute/ExecuteInfo/ExecuteInfoForm.vue
+++ b/frontend/desktop/src/pages/task/TaskExecute/ExecuteInfo/ExecuteInfoForm.vue
@@ -314,7 +314,7 @@
getFormsHookState () {
const hooked = {}
const keys = Object.keys(this.constants)
- this.inputs.forEach(form => {
+ Array.isArray(this.inputs) && this.inputs.forEach(form => {
// 已勾选到全局变量中, 判断勾选的输入参数生成的变量及自定义全局变量source_info是否包含该节点对应表单tag_code
// 可能存在表单勾选时已存在相同key的变量，选择复用自定义变量
const isHooked = keys.some(item => {
@@ -437,46 +437,7 @@
}
// 第三方插件
if (isThird) {
- const resp = await this.loadPluginServiceDetail({
- plugin_code: plugin,
- plugin_version: version,
- with_app_detail: true
- })
- if (!resp.result) return
- // 获取参数
- const { outputs: respsOutputs, forms, inputs } = resp.data
- if (forms.renderform) {
- if (!this.isLegacySubProcess) {
- // 获取第三方插件公共输出参数
- if (!this.pluginOutput['remote_plugin']) {
- await this.loadAtomConfig({ atom: 'remote_plugin', version: '1.0.0' })
- }
- // 输出参数
- const storeOutputs = this.pluginOutput['remote_plugin']['1.0.0']
- const outputs = []
- for (const [key, val] of Object.entries(respsOutputs.properties)) {
- outputs.push({
- name: val.title,
- key,
- type: val.type,
- schema: { description: val.description || '--' }
- })
- }
- this.outputs = [...storeOutputs, ...outputs]
- }
- // 获取host
- const { origin } = window.location
- const hostUrl = `${origin + window.SITE_URL}plugin_service/data_api/${plugin}/`
- $.context.bk_plugin_api_host[plugin] = hostUrl
- // 输入参数
- $.atoms[plugin] = {}
- const renderFrom = forms.renderform
- /* eslint-disable-next-line */
- eval(renderFrom)
- } else {
- $.atoms[plugin] = inputs
- this.outputs = [] // jsonschema form输出参数
- }
+ await this.getThirdConfig(plugin, version)
} else {
await this.loadAtomConfig({ atom: plugin, version, classify, name, project_id, scope: 'task' })
this.outputs = this.pluginOutput[plugin][version]
@@ -487,6 +448,51 @@
console.log(e)
}
},
+ // 第三方插件输入输出配置
+ async getThirdConfig (plugin, version) {
+ try {
+ const resp = await this.loadPluginServiceDetail({
+ plugin_code: plugin,
+ plugin_version: version,
+ with_app_detail: true
+ })
+ if (!resp.result) return
+ // 获取参数
+ const { outputs: respOutputs, forms, inputs } = resp.data
+ // 获取host
+ const { origin } = window.location
+ const hostUrl = `${origin + window.SITE_URL}plugin_service/data_api/${plugin}/`
+ $.context.bk_plugin_api_host[plugin] = hostUrl
+ if (forms.renderform) {
+ // 输入参数
+ $.atoms[plugin] = {}
+ const renderFrom = forms.renderform
+ /* eslint-disable-next-line */
+ eval(renderFrom)
+ } else {
+ $.atoms[plugin] = inputs
+ }
+
+ // 输出参数
+ const outputs = []
+ // 获取第三方插件公共输出参数
+ if (!this.pluginOutput['remote_plugin']) {
+ await this.loadAtomConfig({ atom: 'remote_plugin', version: '1.0.0' })
+ }
+ const storeOutputs = this.pluginOutput['remote_plugin']['1.0.0']
+ for (const [key, val] of Object.entries(respOutputs.properties)) {
+ outputs.push({
+ name: val.title || key,
+ key,
+ type: val.type,
+ schema: { description: val.description }
+ })
+ }
+ this.outputs = [...storeOutputs, ...outputs]
+ } catch (error) {
+ console.warn(error)
+ }
+ },
/**
* 获取子流程任务节点输入参数值
*/
diff --git a/frontend/desktop/src/pages/task/TaskExecute/ExecuteInfo/JsonschemaForm.vue b/frontend/desktop/src/pages/task/TaskExecute/ExecuteInfo/JsonschemaForm.vue
index c4a028d2e5..615549c1d8 100644
--- a/frontend/desktop/src/pages/task/TaskExecute/ExecuteInfo/JsonschemaForm.vue
+++ b/frontend/desktop/src/pages/task/TaskExecute/ExecuteInfo/JsonschemaForm.vue
@@ -35,10 +35,7 @@
data () {
return {
formData: tools.deepClone(this.value),
- context: {
- site_url: $.context.site_url,
- project_id: $.context.project?.id
- }
+ context: { ...$.context }
}
},
watch: {
diff --git a/frontend/desktop/src/pages/template/TemplateEdit/NodeConfig/JsonschemaInputParams.vue b/frontend/desktop/src/pages/template/TemplateEdit/NodeConfig/JsonschemaInputParams.vue
index af6b41329c..ca13736680 100644
--- a/frontend/desktop/src/pages/template/TemplateEdit/NodeConfig/JsonschemaInputParams.vue
+++ b/frontend/desktop/src/pages/template/TemplateEdit/NodeConfig/JsonschemaInputParams.vue
@@ -38,10 +38,7 @@
data () {
return {
inputFormData: tools.deepClone(this.value),
- context: {
- site_url: $.context.site_url,
- project_id: $.context.project?.id
- }
+ context: { ...$.context }
}
},
watch: {
diff --git a/frontend/desktop/src/pages/template/TemplateEdit/NodeConfig/NodeConfig.vue b/frontend/desktop/src/pages/template/TemplateEdit/NodeConfig/NodeConfig.vue
index cc5304f1d5..d5c2c70665 100644
--- a/frontend/desktop/src/pages/template/TemplateEdit/NodeConfig/NodeConfig.vue
+++ b/frontend/desktop/src/pages/template/TemplateEdit/NodeConfig/NodeConfig.vue
@@ -625,53 +625,7 @@
}
// 第三方插件
if (isThird) {
- const resp = await this.loadPluginServiceDetail({
- plugin_code: plugin,
- plugin_version: version,
- with_app_detail: true
- })
- if (!resp.result) return
- // 获取参数
- const { outputs: respsOutputs, forms, inputs } = resp.data
- // 获取不同版本的描述
- let desc = resp.data.desc || ''
- if (desc && desc.includes('\n')) {
- const descList = desc.split('\n')
- desc = descList.join('
')
- }
- this.updateBasicInfo({ desc })
- if (forms.renderform) {
- if (!this.isSubflow) {
- // 获取第三方插件公共输出参数
- if (!this.pluginOutput['remote_plugin']) {
- await this.loadAtomConfig({ atom: 'remote_plugin', version: '1.0.0' })
- }
- // 输出参数
- const storeOutputs = this.pluginOutput['remote_plugin']['1.0.0']
- const outputs = []
- for (const [key, val] of Object.entries(respsOutputs.properties)) {
- outputs.push({
- name: val.title,
- key,
- type: val.type,
- schema: { description: val.description || '--' }
- })
- }
- this.outputs = [...storeOutputs, ...outputs]
- }
- // 获取host
- const { origin } = window.location
- const hostUrl = `${origin + window.SITE_URL}plugin_service/data_api/${plugin}/`
- $.context.bk_plugin_api_host[plugin] = hostUrl
- // 输入参数
- $.atoms[plugin] = {}
- const renderFrom = forms.renderform
- /* eslint-disable-next-line */
- eval(renderFrom)
- } else {
- $.atoms[plugin] = inputs
- this.outputs = [] // jsonschema form输出参数
- }
+ await this.getThirdConfig(plugin, version)
} else {
await this.loadAtomConfig({ atom: plugin, version, classify, name, project_id })
}
@@ -681,6 +635,58 @@
console.log(e)
}
},
+ // 第三方插件输入输出配置
+ async getThirdConfig (plugin, version) {
+ try {
+ const resp = await this.loadPluginServiceDetail({
+ plugin_code: plugin,
+ plugin_version: version,
+ with_app_detail: true
+ })
+ if (!resp.result) return
+ // 获取参数
+ const { outputs: respOutputs, forms, inputs } = resp.data
+ // 获取不同版本的描述
+ let desc = resp.data.desc || ''
+ if (desc && desc.includes('\n')) {
+ const descList = desc.split('\n')
+ desc = descList.join('
')
+ }
+ this.updateBasicInfo({ desc })
+ // 获取host
+ const { origin } = window.location
+ const hostUrl = `${origin + window.SITE_URL}plugin_service/data_api/${plugin}/`
+ $.context.bk_plugin_api_host[plugin] = hostUrl
+ if (forms.renderform) {
+ // 输入参数
+ $.atoms[plugin] = {}
+ const renderFrom = forms.renderform
+ /* eslint-disable-next-line */
+ eval(renderFrom)
+ } else {
+ $.atoms[plugin] = inputs
+ }
+
+ // 输出参数
+ const outputs = []
+ // 获取第三方插件公共输出参数
+ if (!this.pluginOutput['remote_plugin']) {
+ await this.loadAtomConfig({ atom: 'remote_plugin', version: '1.0.0' })
+ }
+ const storeOutputs = this.pluginOutput['remote_plugin']['1.0.0']
+ for (const [key, val] of Object.entries(respOutputs.properties)) {
+ outputs.push({
+ name: val.title || key,
+ key,
+ type: val.type,
+ schema: { description: val.description }
+ })
+ }
+ this.outputs = [...storeOutputs, ...outputs]
+ } catch (error) {
+ console.warn(error)
+ }
+ },
/**
* 加载子流程任务节点输入、输出、版本配置项
*/
diff --git a/frontend/desktop/src/pages/template/TemplateEdit/TemplateSetting/TabGlobalVariables/VariableEdit.vue b/frontend/desktop/src/pages/template/TemplateEdit/TemplateSetting/TabGlobalVariables/VariableEdit.vue
index e93c8370b3..65cdf9e1d4 100644
--- a/frontend/desktop/src/pages/template/TemplateEdit/TemplateSetting/TabGlobalVariables/VariableEdit.vue
+++ b/frontend/desktop/src/pages/template/TemplateEdit/TemplateSetting/TabGlobalVariables/VariableEdit.vue
@@ -1045,7 +1045,7 @@
margin-right: 10px;
padding: 0 25px;
}
-
+
}
.textarea-wrap {
height: 88px;
diff --git a/gcloud/utils/data_handler.py b/gcloud/utils/data_handler.py
new file mode 100644
index 0000000000..d24133eae1
--- /dev/null
+++ b/gcloud/utils/data_handler.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community
+Edition) available.
+Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+
+def _default_func(data, *args, **kwargs):
+ return data
+
+
+def chunk_data(data, chunk_size, func=None, *args, **kwargs):
+ return [(func or _default_func)(data[i : i + chunk_size], *args, **kwargs) for i in range(0, len(data), chunk_size)]
diff --git a/pipeline_plugins/cmdb_ip_picker/query.py b/pipeline_plugins/cmdb_ip_picker/query.py
index 666d776a2d..a09275b650 100644
--- a/pipeline_plugins/cmdb_ip_picker/query.py
+++ b/pipeline_plugins/cmdb_ip_picker/query.py
@@ -23,8 +23,10 @@
from api.utils.request import batch_request
from gcloud.conf import settings
from gcloud.utils import cmdb
+from gcloud.utils.data_handler import chunk_data
from gcloud.utils.handlers import handle_api_error
from gcloud.utils.ip import format_sundry_ip
+from pipeline_plugins.components.utils.common import batch_execute_func
from .constants import ERROR_CODES, NO_ERROR
from .utils import (
@@ -40,6 +42,17 @@
get_client_by_user = settings.ESB_GET_CLIENT_BY_USER
+def format_agent_ip(data, *args, **kwargs):
+ bk_biz_id = kwargs["bk_biz_id"]
+ return [
+ {
+ "host_id": host["bk_host_id"],
+ "meta": {"bk_biz_id": bk_biz_id, "scope_type": "biz", "scope_id": bk_biz_id},
+ }
+ for host in data
+ ]
+
+
def cmdb_search_topo_tree(request, bk_biz_id, bk_supplier_account=""):
"""
@summary: 获取 CMDB 上业务的拓扑树，包含空闲机和故障机模块，根节点是业务
@@ -162,27 +175,21 @@ def cmdb_search_host(request, bk_biz_id, bk_supplier_account="", bk_supplier_id=
host["agent"] = agent_id_status_map.get(bk_agent_id, -1)
else:
client = BKNodeManClient(username=request.user.username)
- agent_kwargs = {
- "all_scope": True,
- "host_list": [
- {
- "cloud_id": host["bk_cloud_id"],
- "ip": host["bk_host_innerip"],
- "meta": {"bk_biz_id": bk_biz_id, "scope_type": "biz", "scope_id": bk_biz_id},
- }
- for host in data
- if host["bk_host_innerip"] != ""
- ],
- }
- agent_result = client.get_ipchooser_host_details(agent_kwargs)
- if not agent_result["result"]:
- message = handle_api_error(
- _("节点管理(nodeman)"), "nodeman.get_ipchooser_host_details", agent_kwargs, agent_result
- )
- result = {"result": False, "code": ERROR_CODES.API_GSE_ERROR, "message": message}
- return JsonResponse(result)
+ host_list = chunk_data(data, 1000, format_agent_ip, bk_biz_id=bk_biz_id)
+ agent_kwargs = [{"all_scope": True, "host_list": host} for host in host_list]
+ results = batch_execute_func(client.get_ipchooser_host_details, agent_kwargs, interval_enabled=True)
- agent_data = format_agent_data(agent_result["data"])
+ agent_data = []
+ for result in results:
+ agent_result = result["result"]
+ if not agent_result["result"]:
+ message = handle_api_error(
+ _("节点管理(nodeman)"), "nodeman.get_ipchooser_host_details", agent_kwargs, agent_result
+ )
+ result = {"result": False, "code": ERROR_CODES.API_GSE_ERROR, "message": message}
+ return JsonResponse(result)
+ agent_data.extend(agent_result["data"])
+ agent_data = format_agent_data(agent_data)
for host in data:
# agent在线状态，0为不在线，1为在线，-1为未知
agent_info = agent_data.get(
diff --git a/pipeline_plugins/variables/collections/sites/open/ip_filter_base.py b/pipeline_plugins/variables/collections/sites/open/ip_filter_base.py
index 83cbd96e39..d1d954b3fd 100644
--- a/pipeline_plugins/variables/collections/sites/open/ip_filter_base.py
+++ b/pipeline_plugins/variables/collections/sites/open/ip_filter_base.py
@@ -22,11 +22,13 @@
from gcloud.core.models import Project
from gcloud.exceptions import ApiRequestError
from gcloud.utils import cmdb
+from gcloud.utils.data_handler import chunk_data
from gcloud.utils.handlers import handle_api_error
from gcloud.utils.ip import IpRegexType, extract_ip_from_ip_str, get_ip_by_regex_type
from pipeline_plugins.base.utils.inject import supplier_account_for_business, supplier_id_for_project
from pipeline_plugins.cmdb_ip_picker.utils import format_agent_data, get_gse_agent_status_ipv6
from pipeline_plugins.components.collections.sites.open.cc.base import cc_get_host_by_innerip_with_ipv6
+from pipeline_plugins.components.utils.common import batch_execute_func
logger = logging.getLogger("root")
get_client_by_user = gcloud_settings.ESB_GET_CLIENT_BY_USER
@@ -94,30 +96,36 @@ def match_ges_v2(self, gse_agent_status, username, bk_biz_id, bk_supplier_id, or
return match_ip
+ @staticmethod
+ def format_origin_ip(data, *args, **kwargs):
+ bk_biz_id = kwargs["bk_biz_id"]
+ return [
+ {
+ "cloud_id": host["bk_cloud_id"],
+ "ip": host["ip"],
+ "meta": {"bk_biz_id": bk_biz_id, "scope_type": "biz", "scope_id": bk_biz_id},
+ }
+ for host in data
+ ]
+
def match_gse_v1(self, gse_agent_status, username, bk_biz_id, bk_supplier_id, origin_ip_list):
match_ip = origin_ip_list
if gse_agent_status in [GseAgentStatus.ONlINE.value, GseAgentStatus.OFFLINE.value]:
+
client = get_nodeman_client_by_user(username=username)
- agent_kwargs = {
- "all_scope": True,
- "host_list": [
- {
- "cloud_id": host["bk_cloud_id"],
- "ip": host["ip"],
- "meta": {"bk_biz_id": bk_biz_id, "scope_type": "biz", "scope_id": bk_biz_id},
- }
- for host in origin_ip_list
- if host["ip"] != ""
- ],
- }
- agent_result = client.get_ipchooser_host_details(agent_kwargs)
-
- if not agent_result["result"]:
- message = handle_api_error(
- _("节点管理(nodeman)"), "nodeman.get_ipchooser_host_details", agent_kwargs, agent_result
- )
- raise ApiRequestError(f"ERROR:{message}")
- agent_data = format_agent_data(agent_result["data"])
+ host_list = chunk_data(origin_ip_list, 1000, self.format_origin_ip, bk_biz_id=bk_biz_id)
+ agent_kwargs = [{"all_scope": True, "host_list": host} for host in host_list]
+ results = batch_execute_func(client.get_ipchooser_host_details, agent_kwargs, interval_enabled=True)
+ agent_data = []
+ for result in results:
+ agent_result = result["result"]
+ if not agent_result["result"]:
+ message = handle_api_error(
+ _("节点管理(nodeman)"), "nodeman.get_ipchooser_host_details", agent_kwargs, agent_result
+ )
+ raise ApiRequestError(f"ERROR:{message}")
+ agent_data.extend(agent_result["data"])
+ agent_data = format_agent_data(agent_data)
agent_online_ip_list = []
agent_offline_ip_list = []
for plat_ip, info in agent_data.items():