Merge pull request #187 from WeBankFinTech/branch-1.0.0

Branch 1.0.0

README.md

@@ -1,13 +1,11 @@
 [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)  
 
-English | [中文](/README_zh.md)  
+English | [中文](docs/zh_CN/ch1/README.md)  
 
 ## Overview
 Qualitis is a data quality management platform that supports quality verification, notification, and management for various datasource. It is used to solve various data quality problems caused by data processing.
 
-Based on Spring Boot, Qualitis submits quality model task to [Linkis](https://github.com/apache/incubator-linkis) platform. It provides functions such as data quality model construction, data quality model execution, data quality verification, reports of data quality generation and so on.
-
-At the same time, Qualitis provides enterprise-level features of financial-level resource isolation, management and access control. It is also guaranteed working well under high-concurrency, high-performance and high-availability scenarios.
+Based on Spring Boot, Qualitis submits quality model task to [Linkis](https://github.com/WeBankFinTech/Linkis) platform. It provides functions such as data quality model construction, data quality model execution, data quality verification, reports of data quality generation and so on. At the same time, Qualitis provides enterprise-level features of financial-level resource isolation, management and access control. It is also guaranteed working well under high-concurrency, high-performance and high-availability scenarios.
 
 ## Features
 - **Define Data Quality Model**  
@@ -34,14 +32,13 @@ Supports workflow
 Workflow needs [DataSphereStudio](https://github.com/WeBankFinTech/DataSphereStudio).
 
 - **Administrator Console**  
-Administrator console provided.  
-And it also supports personnel management, access control management, privilege control management, metadata management and so on.  
+Administrator console provided. And it also supports personnel management, access control management, privilege control management, metadata management and so on.  
 
 ## Compared with similar systems
 ![](images/en_US/ch1/CompareSimilarSystem.png)
 
 ## Documents
-[Quick Deploy](docs/en_US/ch1/QuickDeploy.md)  
+[Quick Deploy](docs/en_US/ch1/Quick%20Deploy%20Standalone.md)  
 [User Manual](docs/en_US/ch1/User%20Manual.md)  
 [Architecture Design](docs/en_US/ch1/Architecture%20Design.md)  
 <br/>
@@ -67,12 +64,15 @@ Supports generating data quality reports with optional latitude.
 ### 4. Support intelligent discovery of data quality problems
 <br/>
 
+**If you have any needs, please send us an issue and we will reply to you in time.**
+
+## Contributing
+Community partners are very welcome to contribute new engines and codes to us!
+
 ## Communication
 If you desire immediate response, please kindly raise issues to us or scan the below QR code by WeChat and QQ to join our group:   
 ![](images/en_US/ch1/ContractUs.png)
 
 ## License
 **Qualitis is under the Apache 2.0 license. See the [LICENSE](/LICENSE) file for details.**
 
-## Tips
-The front-end code of Qualitis adopts the front-end framework FES self-developed by WeBank. The FES framework is currently open source, and the source code can be downloaded through the ui folder in the root directory. Of course, it can also be used directly by downloading the release. The default front-end compilation package is installed in Qualitis-x.x.x/conf/static.

appconn/pom.xml

@@ -79,7 +79,7 @@
       <groupId>org.apache.linkis</groupId>
       <artifactId>linkis-cs-common</artifactId>
       <version>${linkis.version}</version>
-      <scope>compile</scope>
+      <scope>provided</scope>
     </dependency>
     <dependency>
       <artifactId>linkis-bml-client</artifactId>
@@ -99,6 +99,7 @@
       <groupId>org.apache.linkis</groupId>
       <artifactId>linkis-httpclient</artifactId>
       <version>${linkis.version}</version>
+      <scope>provided</scope>
       <exclusions>
         <exclusion>
           <artifactId>linkis-common</artifactId>
@@ -130,9 +131,13 @@
       <version>${dss.version}</version>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+      <version>2.14.1</version>
+    </dependency>
   </dependencies>
 
-
   <build>
     <plugins>
       <plugin>

appconn/src/main/icons/TableRules.icon

@@ -0,0 +1 @@
+<svg t="1572510532353" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="22979" xmlns:xlink="http://www.w3.org/1999/xlink" width="30" height="30"> <path d="M512 11.324L82.824 154.323V583.56A429.116 429.116 0 0 0 512 1012.616a429.116 429.116 0 0 0 429.176-429.177V154.443L512 11.264zM860.762 583.56a348.762 348.762 0 0 1-697.524 0V214.679L512 91.799l348.762 122.88v368.82zM362.616 470.679a40.117 40.117 0 0 0-56.862 0 40.117 40.117 0 0 0 0 56.922L450.32 672.166l2.41 2.41a37.948 37.948 0 0 0 53.73 0l249.795-249.797a37.948 37.948 0 0 0 0-53.79l-3.132-3.132a37.948 37.948 0 0 0-53.73 0L479.533 587.595 362.677 470.74z" p-id="22980" fill="#339999"/> </svg>

appconn/src/main/icons/checkalert.icon

@@ -0,0 +1 @@
+<svg t="1572510532353" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="22979" xmlns:xlink="http://www.w3.org/1999/xlink" width="30" height="30"> <path d="M512 11.324L82.824 154.323V583.56A429.116 429.116 0 0 0 512 1012.616a429.116 429.116 0 0 0 429.176-429.177V154.443L512 11.264zM860.762 583.56a348.762 348.762 0 0 1-697.524 0V214.679L512 91.799l348.762 122.88v368.82zM362.616 470.679a40.117 40.117 0 0 0-56.862 0 40.117 40.117 0 0 0 0 56.922L450.32 672.166l2.41 2.41a37.948 37.948 0 0 0 53.73 0l249.795-249.797a37.948 37.948 0 0 0 0-53.79l-3.132-3.132a37.948 37.948 0 0 0-53.73 0L479.533 587.595 362.677 470.74z" p-id="22980" fill="#339999"/> </svg>

appconn/src/main/java/com/webank/wedatasphere/dss/appconn/qualitis/execution/QualitisRefExecutionOperation.java

@@ -25,7 +25,6 @@
 import com.webank.wedatasphere.dss.standard.app.development.listener.core.LongTermRefExecutionOperation;
 import com.webank.wedatasphere.dss.standard.app.development.listener.core.Killable;
 import com.webank.wedatasphere.dss.standard.app.development.listener.core.Procedure;
-import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService;
 import com.webank.wedatasphere.dss.standard.app.development.listener.ref.ExecutionResponseRef;
 import com.webank.wedatasphere.dss.standard.app.development.listener.ref.RefExecutionRequestRef;
 import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException;
@@ -53,23 +52,25 @@
  */
 public class QualitisRefExecutionOperation extends LongTermRefExecutionOperation<RefExecutionRequestRef.RefExecutionProjectWithContextRequestRef>
     implements Killable, Procedure {
-    private static final String SUBMIT_TASK_PATH = "qualitis/outer/api/v1/execution";
-    private static final String GET_TASK_STATUS_PATH = "qualitis/outer/api/v1/application/{applicationId}/status/";
-    private static final String GET_TASK_RESULT_PATH = "qualitis/outer/api/v1/application/{applicationId}/result/";
-    private static final String KILL_TASK_PATH = "qualitis/outer/api/v1/execution/application/kill/{applicationId}/{executionUser}";
+
+    private static Logger LOGGER = LoggerFactory.getLogger(QualitisRefExecutionOperation.class);
+
+    private static final String FILTER = "filter";
     private static final String NODE_NAME_KEY = "nodeName";
     private static final String RULEGROUPID = "ruleGroupId";
     private static final String RULE_GROUP_ID = "rule_group_id";
     private static final String EXECUTION_USER_KEY = "executionUser";
     private static final String WDS_SUBMIT_USER_KEY = "wds.dss.workflow.submit.user";
+    private static final String SUBMIT_TASK_PATH = "qualitis/outer/api/v1/execution";
+    private static final String GET_TASK_LOG_PATH = "qualitis/outer/api/v1/application/{applicationId}/log";
+    private static final String GET_TASK_STATUS_PATH = "qualitis/outer/api/v1/application/{applicationId}/status/";
+    private static final String GET_TASK_RESULT_PATH = "qualitis/outer/api/v1/application/{applicationId}/result/";
+    private static final String KILL_TASK_PATH = "qualitis/outer/api/v1/execution/application/kill/{applicationId}/{executionUser}";
 
-    private static Logger LOGGER = LoggerFactory.getLogger(QualitisRefExecutionOperation.class);
 
     private String appId = "linkis_id";
     private String appToken = "***REMOVED***";
 
-    private static final String FILTER = "filter";
-
     @Override
     protected String getAppConnName() {
         return QualitisAppConn.QUALITIS_APPCONN_NAME;
@@ -254,7 +255,7 @@ public RefExecutionState state(RefExecutionAction action) {
         }
         LOGGER.info("Start to check job. url: {}, method: {}, body: {}", url, HttpMethod.GET, entity);
         Map<String, Object> response = restTemplate.getForEntity(url, Map.class).getBody();
-        String finishLog = String.format("Succeed to submit job to qualitis. response: %s", response);
+        String finishLog = String.format("Succeed to check job. response: %s", response);
         LOGGER.info(finishLog);
 
         if (response == null) {
@@ -265,7 +266,7 @@ public RefExecutionState state(RefExecutionAction action) {
 
         if (! checkResponse(response)) {
             String message = (String) response.get("message");
-            String errorMsg = String.format("Error! Can not submit job, exception: %s", message);
+            String errorMsg = String.format("Error! Can not check job, exception: %s", message);
             LOGGER.error(errorMsg);
             return null;
         }
@@ -362,7 +363,7 @@ public ExecutionResponseRef result(RefExecutionAction action) {
         }
         LOGGER.info("Start to get job result. url: {}, method: {}, body: {}", url, HttpMethod.GET, entity);
         Map<String, Object> response = restTemplate.getForEntity(url, Map.class).getBody();
-        String finishLog = String.format("Succeed to submit job to qualitis. response: %s", response);
+        String finishLog = String.format("Succeed to get job result. response: %s", response);
         LOGGER.info(finishLog);
 
         if (response == null) {
@@ -373,7 +374,7 @@ public ExecutionResponseRef result(RefExecutionAction action) {
 
         if (! checkResponse(response)) {
             String message = (String) response.get("message");
-            String errorMsg = String.format("Error! Can not submit job, exception: %s", message);
+            String errorMsg = String.format("Error! Can not get job result, exception: %s", message);
             LOGGER.error(errorMsg);
             return null;
         }
@@ -388,6 +389,13 @@ public ExecutionResponseRef result(RefExecutionAction action) {
         LOGGER.info(taskMsg);
         LOGGER.info(resultMessage);
 
+        try {
+            action.getExecutionRequestRefContext().appendLog(this.log(action));
+        } catch (Exception e) {
+            LOGGER.error("Get qualitis log failed.");
+            LOGGER.error(e.getMessage(), e);
+        }
+
         if (failedNum != 0) {
             return ExecutionResponseRef.newBuilder().error();
         } else {
@@ -403,6 +411,50 @@ public float progress(RefExecutionAction action) {
 
     @Override
     public String log(RefExecutionAction action) {
-        return "";
+        if (null == action) {
+            return "";
+        }
+
+        QualitisRefExecutionAction qualitisRefExecutionAction = (QualitisRefExecutionAction) action;
+        String applicationId = qualitisRefExecutionAction.getApplicationId();
+        String executionUser = qualitisRefExecutionAction.getExecutionUser();
+        LOGGER.info("Qualitis application ID: {}", applicationId);
+        LOGGER.info("Qualitis execution user: {}", executionUser);
+        if (StringUtils.isEmpty(applicationId) || StringUtils.isEmpty(executionUser)) {
+            return "";
+        }
+        HttpHeaders headers = new HttpHeaders();
+        headers.setContentType(MediaType.APPLICATION_JSON);
+
+        HttpEntity<Object> entity = new HttpEntity<>(headers);
+
+        RestTemplate restTemplate = new RestTemplate();
+        String url = null;
+        try {
+            url = HttpUtils.buildUrI(getBaseUrl(), GET_TASK_LOG_PATH.replace("{applicationId}", applicationId), appId, appToken, RandomStringUtils.randomNumeric(5), String.valueOf(System.currentTimeMillis())).toString();
+        } catch (NoSuchAlgorithmException e) {
+            LOGGER.info("Qualitis no signature algor.", e);
+        } catch (URISyntaxException e) {
+            LOGGER.error("Qualitis uri syntax exception.", e);
+        }
+        LOGGER.info("Start to get job log. url: {}, method: {}, body: {}", url, HttpMethod.GET, entity);
+        Map<String, Object> response = restTemplate.getForEntity(url, Map.class).getBody();
+        String finishLog = String.format("Succeed to get job log.");
+        LOGGER.info(finishLog);
+
+        if (response == null) {
+            String errorMsg = "Error! Can not get job log, response is null";
+            LOGGER.error(errorMsg);
+            return "";
+        }
+
+        if (! checkResponse(response)) {
+            String message = (String) response.get("message");
+            String errorMsg = String.format("Error! Can not get job log, exception: %s", message);
+            LOGGER.error(errorMsg);
+            return "";
+        }
+
+         return (String) response.get("data");
     }
 }

appconn/src/main/java/com/webank/wedatasphere/dss/appconn/qualitis/project/QualitisCheckTemplateSearchOperation.java

@@ -5,26 +5,28 @@
 import com.webank.wedatasphere.dss.appconn.qualitis.ref.entity.QualitisTemplate;
 import com.webank.wedatasphere.dss.appconn.qualitis.utils.HttpUtils;
 import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRef;
-import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRefImpl;
 import com.webank.wedatasphere.dss.standard.app.structure.optional.AbstractOptionalOperation;
-import com.webank.wedatasphere.dss.standard.app.structure.project.ref.ProjectResponseRef;
-import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRefImpl;
 import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef;
 import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefImpl;
 import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException;
-import org.apache.commons.collections.CollectionUtils;
+import java.net.URISyntaxException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import org.apache.commons.lang.RandomStringUtils;
 import org.apache.linkis.common.conf.CommonVars;
 import org.apache.linkis.server.BDPJettyServerHelper;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.http.*;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
 import org.springframework.web.client.RestTemplate;
 
-import java.net.URISyntaxException;
-import java.security.NoSuchAlgorithmException;
-import java.util.*;
-
 /**
  * @author leebai
  * @date 2022/5/19 17:00