Add new subprojects: z-desensitization for data anonymization and pri…

…vate-knowledge for vector knowledge management (#863)
XiaoMi · Jul 17, 2024 · 1cd3923 · 1cd3923
2 parents 539bdfc + ed52ad7
commit 1cd3923
Show file tree

Hide file tree

Showing 137 changed files with 18,639 additions and 0 deletions.
diff --git a/m78-all/private-knowledge/.gitignore b/m78-all/private-knowledge/.gitignore
@@ -0,0 +1,26 @@
+*.bak
+build_info.properties
+.classpath
+dependency-reduced-pom.xml
+*.diff
+.DS_Store
+
+# IDEA
+.idea/
+*.iml
+logs/
+log/
+
+# eclipse (Scala IDE)
+.project
+.settings/
+.tags*
+target/
+test-output/
+nohup*
+*.log
+*.swp
+*.pyc
+.sbtserver*
+/.java-version
+/private-knowledge-server/src/main/resources/athena/athena_module.md
diff --git a/m78-all/private-knowledge/Dockerfile b/m78-all/private-knowledge/Dockerfile
@@ -0,0 +1,13 @@
+FROM openjdk:8
+
+ENV APP_HOME /opt/app
+
+RUN mkdir -p ${APP_HOME}
+
+RUN echo 'Asia/Shanghai' >/etc/timezone
+
+COPY private-knowledge-server/target/private-knowledge-server-*.jar ${APP_HOME}/private-knowledge-server.jar
+
+WORKDIR ${APP_HOME}
+
+ENTRYPOINT ["sh", "-c", "java $JAVA_OPTS -jar private-knowledge-server.jar"]
diff --git a/m78-all/private-knowledge/mybatis-flex.config b/m78-all/private-knowledge/mybatis-flex.config
@@ -0,0 +1 @@
+processor.enable=true
diff --git a/m78-all/private-knowledge/pom.xml b/m78-all/private-knowledge/pom.xml
@@ -0,0 +1,146 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>run.mone.privateknowledge</groupId>
+    <artifactId>private-knowledge</artifactId>
+    <version>1.0-SNAPSHOT-SNAPSHOT</version>
+    <packaging>pom</packaging>
+
+    <modules>
+        <module>private-knowledge-api</module>
+        <module>private-knowledge-common</module>
+        <module>private-knowledge-service</module>
+        <module>private-knowledge-server</module>
+    </modules>
+
+    <parent>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-parent</artifactId>
+        <version>2.3.12.RELEASE</version>
+    </parent>
+
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>run.mone</groupId>
+                <artifactId>infra-result</artifactId>
+                <version>1.4-SNAPSHOT</version>
+            </dependency>
+            <dependency>
+                <groupId>com.xiaomi.youpin</groupId>
+                <artifactId>aop</artifactId>
+                <version>1.4-SNAPSHOT</version>
+            </dependency>
+            <dependency>
+                <groupId>com.xiaomi.youpin</groupId>
+                <artifactId>common</artifactId>
+                <version>1.7-SNAPSHOT</version>
+            </dependency>
+            <dependency>
+                <groupId>ch.qos.logback</groupId>
+                <artifactId>logback-classic</artifactId>
+                <version>1.1.2</version>
+            </dependency>
+            <dependency>
+                <groupId>ch.qos.logback</groupId>
+                <artifactId>logback-core</artifactId>
+                <version>1.1.2</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.dubbo</groupId>
+                <artifactId>dubbo</artifactId>
+                <version>2.7.12-mone-v20-SNAPSHOT</version>
+            </dependency>
+            <dependency>
+                <groupId>com.xiaomi.mone</groupId>
+                <artifactId>dubbo-auth</artifactId>
+                <version>2.7.12-SNAPSHOT</version>
+            </dependency>
+            <dependency>
+                <groupId>com.xiaomi.mone</groupId>
+                <artifactId>dubbo-trace</artifactId>
+                <version>2.7.12-SNAPSHOT</version>
+            </dependency>
+            <dependency>
+                <groupId>com.alibaba</groupId>
+                <artifactId>dubbo-registry-nacos</artifactId>
+                <version>1.2.1-mone-SNAPSHOT</version>
+            </dependency>
+            <dependency>
+                <groupId>com.alibaba.spring</groupId>
+                <artifactId>spring-context-support</artifactId>
+                <version>1.0.10</version>
+            </dependency>
+            <dependency>
+                <groupId>com.alibaba.nacos</groupId>
+                <artifactId>nacos-spring-context</artifactId>
+                <version>0.3.6-mone-SNAPSHOT</version>
+            </dependency>
+            <dependency>
+                <groupId>com.alibaba.nacos</groupId>
+                <artifactId>nacos-client</artifactId>
+                <version>1.2.1-mone-v7-SNAPSHOT</version>
+            </dependency>
+            <dependency>
+                <groupId>com.alibaba.nacos</groupId>
+                <artifactId>nacos-api</artifactId>
+                <version>1.2.1-mone-v3-SNAPSHOT</version>
+            </dependency>
+            <dependency>
+                <groupId>com.mybatis-flex</groupId>
+                <artifactId>mybatis-flex-spring-boot-starter</artifactId>
+                <version>1.7.3</version>
+            </dependency>
+            <dependency>
+                <groupId>com.mybatis-flex</groupId>
+                <artifactId>mybatis-flex-codegen</artifactId>
+                <version>1.7.8</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.commons</groupId>
+                <artifactId>commons-math3</artifactId>
+                <version>3.6.1</version>
+            </dependency>
+            <dependency>
+                <groupId>redis.clients</groupId>
+                <artifactId>jedis</artifactId>
+                <version>2.9.0</version>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <fork>true</fork>
+                    <verbose>true</verbose>
+                    <encoding>UTF-8</encoding>
+                    <compilerArguments>
+                        <sourcepath>
+                            ${project.basedir}/src/main/java
+                        </sourcepath>
+                    </compilerArguments>
+                </configuration>
+            </plugin>
+
+        </plugins>
+    </build>
+
+    <distributionManagement>
+        <snapshotRepository>
+            <id>ossrh</id>
+            <url>https://s01.oss.sonatype.org/content/repositories/snapshots</url>
+        </snapshotRepository>
+        <repository>
+            <id>ossrh</id>
+            <url>https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/</url>
+        </repository>
+    </distributionManagement>
+
+</project>
diff --git a/m78-all/private-knowledge/private-knowledge-api/pom.xml b/m78-all/private-knowledge/private-knowledge-api/pom.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <parent>
+        <artifactId>private-knowledge</artifactId>
+        <groupId>run.mone.privateknowledge</groupId>
+        <version>1.0-SNAPSHOT-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>private-knowledge-api</artifactId>
+
+
+    <dependencies>
+
+        <dependency>
+            <groupId>run.mone</groupId>
+            <artifactId>infra-result</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.xiaomi.youpin</groupId>
+            <artifactId>common</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <version>1.18.20</version>
+        </dependency>
+
+    </dependencies>
+
+
+
+</project>
diff --git a/...te-knowledge/private-knowledge-api/src/main/java/run/mone/knowledge/api/DemoProvider.java b/...te-knowledge/private-knowledge-api/src/main/java/run/mone/knowledge/api/DemoProvider.java
@@ -0,0 +1,11 @@
+package run.mone.knowledge.api;
+
+import com.xiaomi.data.push.common.Health;
+import run.mone.knowledge.api.dto.DemoReqDto;
+import run.mone.knowledge.api.dto.DemoResDto;
+import com.xiaomi.youpin.infra.rpc.Result;
+
+public interface DemoProvider {
+
+    Result<DemoResDto> query(DemoReqDto reqDto);
+}
diff --git a/...ledge/private-knowledge-api/src/main/java/run/mone/knowledge/api/DubboHealthProvider.java b/...ledge/private-knowledge-api/src/main/java/run/mone/knowledge/api/DubboHealthProvider.java
@@ -0,0 +1,9 @@
+package run.mone.knowledge.api;
+
+import com.xiaomi.youpin.infra.rpc.Result;
+import com.xiaomi.data.push.common.Health;
+
+public interface DubboHealthProvider {
+
+    Result<Health> health();
+}
diff --git a/.../private-knowledge-api/src/main/java/run/mone/knowledge/api/IKnowledgeVectorProvider.java b/.../private-knowledge-api/src/main/java/run/mone/knowledge/api/IKnowledgeVectorProvider.java
@@ -0,0 +1,27 @@
+package run.mone.knowledge.api;
+
+import com.xiaomi.youpin.infra.rpc.Result;
+import run.mone.knowledge.api.dto.KnowledgeVectorDto;
+import run.mone.knowledge.api.dto.SimilarKnowledgeVectorQry;
+import run.mone.knowledge.api.dto.SimilarKnowledgeVectorRsp;
+
+import java.util.List;
+
+/**
+ * @author wmin
+ * @date 2024/2/5
+ */
+public interface IKnowledgeVectorProvider {
+
+    Result<Boolean> insertOrUpdateKnowledgeVector(KnowledgeVectorDto param);
+
+    Result<Boolean> deleteKnowledgeVector(KnowledgeVectorDto param);
+
+    /**
+     * 查询相似的topN条向量数据
+     * @param qry
+     * @return
+     */
+    Result<List<SimilarKnowledgeVectorRsp>> qrySimilarKnowledgeVector(SimilarKnowledgeVectorQry qry);
+
+}
diff --git a/...e/private-knowledge-api/src/main/java/run/mone/knowledge/api/constant/CommonConstant.java b/...e/private-knowledge-api/src/main/java/run/mone/knowledge/api/constant/CommonConstant.java
@@ -0,0 +1,11 @@
+package run.mone.knowledge.api.constant;
+
+/**
+ * @author wmin
+ * @date 2024/2/5
+ */
+public class CommonConstant {
+
+    public static final String PRIVATE_KNOWLEDGE_BASE_REDIS_PREFIX = "private_knowledge_base_";
+
+}
diff --git a/...-knowledge/private-knowledge-api/src/main/java/run/mone/knowledge/api/dto/DemoReqDto.java b/...-knowledge/private-knowledge-api/src/main/java/run/mone/knowledge/api/dto/DemoReqDto.java
@@ -0,0 +1,10 @@
+package run.mone.knowledge.api.dto;
+
+import lombok.Data;
+import lombok.ToString;
+
+@ToString
+@Data
+public class DemoReqDto {
+    private Long id;
+}
diff --git a/...-knowledge/private-knowledge-api/src/main/java/run/mone/knowledge/api/dto/DemoResDto.java b/...-knowledge/private-knowledge-api/src/main/java/run/mone/knowledge/api/dto/DemoResDto.java
@@ -0,0 +1,11 @@
+package run.mone.knowledge.api.dto;
+
+import lombok.Data;
+import lombok.ToString;
+
+@ToString
+@Data
+public class DemoResDto {
+    private String test1;
+    private String test2;
+}
diff --git a/...edge/private-knowledge-api/src/main/java/run/mone/knowledge/api/dto/KnowledgeTagInfo.java b/...edge/private-knowledge-api/src/main/java/run/mone/knowledge/api/dto/KnowledgeTagInfo.java
@@ -0,0 +1,13 @@
+package run.mone.knowledge.api.dto;
+
+import lombok.Data;
+
+/**
+ * @author wmin
+ * @date 2024/2/5
+ */
+@Data
+public class KnowledgeTagInfo {
+    private Integer tagIndex;
+    private String tagValue;
+}
diff --git a/...vate-knowledge-api/src/main/java/run/mone/knowledge/api/dto/KnowledgeVectorDetailDto.java b/...vate-knowledge-api/src/main/java/run/mone/knowledge/api/dto/KnowledgeVectorDetailDto.java
@@ -0,0 +1,37 @@
+package run.mone.knowledge.api.dto;
+
+import lombok.Data;
+import run.mone.knowledge.api.enums.KnowledgeTypeEnum;
+
+
+/**
+ * @author wmin
+ * @date 2024/2/5
+ */
+@Data
+public class KnowledgeVectorDetailDto {
+
+    /**
+     * @see KnowledgeTypeEnum
+     */
+    private String type;
+
+    private String tag1;
+
+    private String tag2;
+
+    private String tag3;
+
+    private String tag4;
+
+    private String tag5;
+
+    private String tag6;
+
+    private String tag7;
+
+    private String content;
+
+    private double[] vector;
+
+}