From d5f6724ec9a27a63be885960e4e96eab6c39b36f Mon Sep 17 00:00:00 2001 From: wodiwudi <40229449+wodiwudi@users.noreply.github.com> Date: Mon, 19 Aug 2024 15:01:20 +0800 Subject: [PATCH 01/11] Refactor codegen unit test (#878) * update * refactor codegen * change pom * change infra pom --- jcommon/codegen/pom.xml | 152 ++++++++++-------- .../run/mone/ai/codegen/CodeGenerator.java | 78 ++++++++- .../mone/ai/codegen/util/TemplateUtils.java | 35 ++++ .../src/main/resources/tlp/testSpring.java | 29 ++++ .../xiaomi/youpin/codegen/test/GenTest.java | 17 +- 5 files changed, 231 insertions(+), 80 deletions(-) create mode 100644 jcommon/codegen/src/main/resources/tlp/testSpring.java diff --git a/jcommon/codegen/pom.xml b/jcommon/codegen/pom.xml index cfc3d4421..01f03b86c 100644 --- a/jcommon/codegen/pom.xml +++ b/jcommon/codegen/pom.xml @@ -2,77 +2,87 @@ - 4.0.0 - - run.mone - jcommon + 4.0.0 + + run.mone + jcommon + 1.6.0-jdk21-SNAPSHOT + + codegen 1.6.0-jdk21-SNAPSHOT - - codegen - 1.6.0-jdk21-SNAPSHOT - - - commons-io - commons-io - 2.6 - compile - - - run.mone - codegen-temlates - 1.0-SNAPSHOT - compile - - - com.ibeetl - beetl - 2.7.14 - - - com.mybatis-flex - mybatis-flex-codegen - 1.7.8 - - - org.springframework - spring-jdbc - 5.3.29 - true - - - com.zaxxer - HikariCP - 4.0.3 - true - - - org.apache.commons - commons-lang3 - 3.8.1 - compile - - - run.mone - infra-result - 1.6.0-jdk21-SNAPSHOT - - - thrift - org.apache.thrift - - - - - org.mybatis.generator - mybatis-generator-core - 1.4.0 - - - mysql - mysql-connector-java - 8.0.26 - + + + commons-io + commons-io + 2.6 + compile + + + run.mone + codegen-temlates + 1.0-SNAPSHOT + compile + + + com.ibeetl + beetl + 3.15.4.RELEASE + true + + + com.mybatis-flex + mybatis-flex-codegen + 1.7.8 + + + org.springframework + spring-jdbc + 5.3.29 + true + + + com.zaxxer + HikariCP + 4.0.3 + true + + + org.apache.commons + commons-lang3 + 3.8.1 + compile + + + run.mone + infra-result + 1.6.0-jdk21-SNAPSHOT + + + thrift + org.apache.thrift + + + + + org.mybatis.generator + mybatis-generator-core + 1.4.0 + + + mysql + mysql-connector-java + 8.0.26 + - - + + org.mockito + mockito-core + 3.9.0 + + + org.mockito + mockito-inline + 3.9.0 + + diff --git a/jcommon/codegen/src/main/java/run/mone/ai/codegen/CodeGenerator.java b/jcommon/codegen/src/main/java/run/mone/ai/codegen/CodeGenerator.java index 57483329f..dc2ddbb8b 100644 --- a/jcommon/codegen/src/main/java/run/mone/ai/codegen/CodeGenerator.java +++ b/jcommon/codegen/src/main/java/run/mone/ai/codegen/CodeGenerator.java @@ -24,7 +24,17 @@ public class CodeGenerator { private static String serviceName = "s"; - private static String basePath = "/Users/wodiwudi/java/nr-car-account"; + private static String testPackageName = "TP"; + + private static String testPath = ""; + + private static String mainClass = ""; + + private static String author = ""; + + private static String testType = ""; + + private static String basePath = "you base path"; private static boolean createPojo = false; @@ -38,6 +48,8 @@ public class CodeGenerator { private static boolean createController = false; + private static final String SPRING_BOOT_TEST_TYPE = "springboot"; + /** * 主方法 * 判断输入参数长度是否大于 0,调用方法处理参数并提取数据 @@ -53,6 +65,11 @@ public static void main(String[] args) { className = map.get("pojoName"); testName = map.get("testName"); serviceName = map.get("serviceName"); + testPackageName = map.get("testPackageName"); + testPath = map.get("testPath"); + mainClass = map.get("mainClass"); + author = map.get("author"); + testType = map.get("testType"); } if (StringUtils.isEmpty(className)) { return; @@ -61,6 +78,11 @@ public static void main(String[] args) { data.put("className", className); data.put("author", "goodjava@qq.com"); data.put("serviceName", serviceName); + data.put("testPackageName", testPackageName); + data.put("testPath", testPath); + data.put("mainClass", mainClass); + data.put("testAuthor", author); + data.put("testType", testType); Optional first = Arrays.stream(testName.split("\\.")).findFirst(); data.put("testName", first.get()); // 调用方法并获取结果 @@ -86,10 +108,14 @@ public static void main(String[] args) { if (createTest) { String cn = testName; - System.out.println("create test:" + cn); - String test = TemplateUtils.renderTemplateFromFile("tlp/test.java", data); - TemplateUtils.writeStringToFile(test, basePath + "/your project/src/test/java/com/xiaomi/nr/car/account/constant/" + cn); - + System.out.println("create test cn :" + cn + "path:" + testPath); + String test; + if (SPRING_BOOT_TEST_TYPE.equals(testType)) { + test = TemplateUtils.renderTemplateFromFile("tlp/testSpring.java", data); + } else { + test = TemplateUtils.renderTemplateFromFile("tlp/test.java", data); + } + TemplateUtils.writeStringToFile(test, testPath + cn); } if (createController) { @@ -98,6 +124,48 @@ public static void main(String[] args) { } } + public static void createAllUnitTest(Map map) { + if (map.isEmpty()) { + return; + } + //方便ai调用的时候,设置表名 + className = map.get("pojoName"); + testName = map.get("testName"); + serviceName = map.get("serviceName"); + testPackageName = map.get("testPackageName"); + testPath = map.get("testPath"); + mainClass = map.get("mainClass"); + author = map.get("author"); + testType = map.get("testType"); + + if (StringUtils.isEmpty(className)) { + return; + } + Map data = new HashMap<>(); + data.put("className", className); + data.put("author", "goodjava@qq.com"); + data.put("serviceName", serviceName); + data.put("testPackageName", testPackageName); + data.put("testPath", testPath); + data.put("mainClass", mainClass); + data.put("testAuthor", author); + data.put("testType", testType); + Optional first = Arrays.stream(testName.split("\\.")).findFirst(); + data.put("testName", first.get()); + + if (createTest) { + String cn = testName; + System.out.println("create test cn :" + cn + "path:" + testPath); + String test; + if (SPRING_BOOT_TEST_TYPE.equals(testType)) { + test = TemplateUtils.renderTemplateFromFileV2("tlp/testSpring.java", data); + } else { + test = TemplateUtils.renderTemplateFromFileV2("tlp/test.java", data); + } + TemplateUtils.writeStringToFile(test, testPath + cn); + } + } + private static Map parseArgsAndExtractData(String[] args) { String jsonStr = args[0]; diff --git a/jcommon/codegen/src/main/java/run/mone/ai/codegen/util/TemplateUtils.java b/jcommon/codegen/src/main/java/run/mone/ai/codegen/util/TemplateUtils.java index 4d2805cbe..1c3aaac36 100644 --- a/jcommon/codegen/src/main/java/run/mone/ai/codegen/util/TemplateUtils.java +++ b/jcommon/codegen/src/main/java/run/mone/ai/codegen/util/TemplateUtils.java @@ -14,6 +14,8 @@ import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; @@ -62,6 +64,39 @@ public static String renderTemplateFromFile(String templateFileName, Map m) { + try { + InputStream is = TemplateUtils.class.getClassLoader().getResourceAsStream(templateFileName); + if (is == null) { + throw new IOException("Resource not found: " + templateFileName); + } + + // 使用Channels和ByteBuffer读取所有字节 + ByteBuffer byteBuffer = ByteBuffer.allocate(1024); + int bytesRead; + while ((bytesRead = Channels.newChannel(is).read(byteBuffer)) != -1) { + if (byteBuffer.remaining() == 0) { + ByteBuffer newBuffer = ByteBuffer.allocate(byteBuffer.capacity() * 2); + byteBuffer.flip(); + newBuffer.put(byteBuffer); + byteBuffer = newBuffer; + } + } + byteBuffer.flip(); + byte[] bytes = new byte[byteBuffer.limit()]; + byteBuffer.get(bytes); + + // 将字节数组转换为字符串 + String template = new String(bytes, StandardCharsets.UTF_8); + + return renderTemplate(template, m); + } catch (IOException ex) { + log.error("Error reading template file", ex); + } + return ""; + } + //把String写到指定文件中(class) public static void writeStringToFile(String content, String filePath) { try (BufferedWriter writer = new BufferedWriter(new FileWriter(filePath))) { diff --git a/jcommon/codegen/src/main/resources/tlp/testSpring.java b/jcommon/codegen/src/main/resources/tlp/testSpring.java new file mode 100644 index 000000000..3e639ed7d --- /dev/null +++ b/jcommon/codegen/src/main/resources/tlp/testSpring.java @@ -0,0 +1,29 @@ +package ${testPackageName}; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import javax.annotation.Resource; +import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; +import static org.mockito.ArgumentMatchers.*; + +/** + * @author ${testAuthor} + */ +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = ${mainClass}.class) +public class ${testName} { + + +@Resource +private ${serviceName} ${strutil.toLowerCase(serviceName)}Service; + + +} \ No newline at end of file diff --git a/jcommon/codegen/src/test/java/com/xiaomi/youpin/codegen/test/GenTest.java b/jcommon/codegen/src/test/java/com/xiaomi/youpin/codegen/test/GenTest.java index 281b2c8ee..8f34f10f4 100644 --- a/jcommon/codegen/src/test/java/com/xiaomi/youpin/codegen/test/GenTest.java +++ b/jcommon/codegen/src/test/java/com/xiaomi/youpin/codegen/test/GenTest.java @@ -16,19 +16,27 @@ package com.xiaomi.youpin.codegen.test; +import com.google.common.collect.Lists; import com.xiaomi.youpin.codegen.*; import com.xiaomi.youpin.codegen.bo.ApiHeaderBo; import com.xiaomi.youpin.codegen.bo.Dependency; import com.xiaomi.youpin.infra.rpc.Result; +import org.apache.commons.lang3.tuple.Pair; import org.junit.Test; +import run.mone.ai.codegen.util.TemplateUtils; +import javax.annotation.Resource; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; public class GenTest { + @Resource + private TemplateUtils templateUtils; + @Test public void testFilter() throws Exception { FilterGen filterGen = new FilterGen(); @@ -38,8 +46,8 @@ public void testFilter() throws Exception { @Test public void testHttpRequestGen() throws Exception { HttpRequestGen httpRequestGen = new HttpRequestGen(); - httpRequestGen.generateJavaReq("getUserInfo",2,13,"[{\"paramNotNull\":\"0\",\"paramType\":\"13\",\"paramName\":\"obj\",\"paramKey\":\"subObj\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[{\"paramNotNull\":\"0\",\"paramType\":\"0\",\"paramName\":\"sub_dzx\",\"paramKey\":\"subName\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]},{\"paramNotNull\":\"0\",\"paramType\":\"3\",\"paramName\":\"666\",\"paramKey\":\"subID\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]}]}]"); - httpRequestGen.generateJavaReq("getUserInfo",2,13,"[{\"paramNotNull\":\"0\",\"paramType\":\"13\",\"paramName\":\"obj\",\"paramKey\":\"subObj\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[{\"paramNotNull\":\"0\",\"paramType\":\"0\",\"paramName\":\"sub_dzx\",\"paramKey\":\"subName\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]},{\"paramNotNull\":\"0\",\"paramType\":\"3\",\"paramName\":\"666\",\"paramKey\":\"subID\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]}]},{\"paramNotNull\":\"0\",\"paramType\":\"0\",\"paramName\":\"dzx\",\"paramKey\":\"userName\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]}]"); + httpRequestGen.generateJavaReq("getUserInfo", 2, 13, "[{\"paramNotNull\":\"0\",\"paramType\":\"13\",\"paramName\":\"obj\",\"paramKey\":\"subObj\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[{\"paramNotNull\":\"0\",\"paramType\":\"0\",\"paramName\":\"sub_dzx\",\"paramKey\":\"subName\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]},{\"paramNotNull\":\"0\",\"paramType\":\"3\",\"paramName\":\"666\",\"paramKey\":\"subID\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]}]}]"); + httpRequestGen.generateJavaReq("getUserInfo", 2, 13, "[{\"paramNotNull\":\"0\",\"paramType\":\"13\",\"paramName\":\"obj\",\"paramKey\":\"subObj\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[{\"paramNotNull\":\"0\",\"paramType\":\"0\",\"paramName\":\"sub_dzx\",\"paramKey\":\"subName\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]},{\"paramNotNull\":\"0\",\"paramType\":\"3\",\"paramName\":\"666\",\"paramKey\":\"subID\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]}]},{\"paramNotNull\":\"0\",\"paramType\":\"0\",\"paramName\":\"dzx\",\"paramKey\":\"userName\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]}]"); List headers = new ArrayList<>(); ApiHeaderBo header1 = new ApiHeaderBo(); @@ -50,7 +58,7 @@ public void testHttpRequestGen() throws Exception { header2.setHeaderValue("true"); headers.add(header1); headers.add(header2); - Result result = httpRequestGen.generateCurlReq(0,"/Api/getUserInfo",1,"[{\"paramNotNull\":\"0\",\"paramType\":\"13\",\"paramName\":\"obj\",\"paramKey\":\"subObj\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[{\"paramNotNull\":\"0\",\"paramType\":\"0\",\"paramName\":\"sub_dzx\",\"paramKey\":\"subName\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]},{\"paramNotNull\":\"0\",\"paramType\":\"3\",\"paramName\":\"666\",\"paramKey\":\"subID\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]}]}]",headers); + Result result = httpRequestGen.generateCurlReq(0, "/Api/getUserInfo", 1, "[{\"paramNotNull\":\"0\",\"paramType\":\"13\",\"paramName\":\"obj\",\"paramKey\":\"subObj\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[{\"paramNotNull\":\"0\",\"paramType\":\"0\",\"paramName\":\"sub_dzx\",\"paramKey\":\"subName\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]},{\"paramNotNull\":\"0\",\"paramType\":\"3\",\"paramName\":\"666\",\"paramKey\":\"subID\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]}]}]", headers); System.out.print(result.getData()); } @@ -137,9 +145,10 @@ public void testDDD() throws Exception { new DDDProGen().generateAndZip("/tmp/work", "abcd", "com.xiaomi.youpin", "com.xiaomi.youpin.abcdefg", "dfz", "1.0.0", dep); //new CNSalesCrmGen().generateAndZip("/home/work", "testt", "com.xiaomi.test", "com.xiaomi.test.testt", "dfz", "1.0.0", dep); } + @Test public void testFaas() throws Exception { FaasGen gen = new FaasGen(); - gen.generateAndZip("/tmp/work", "project-faas", "com.xiaomi.youpin", "com.xiaomi.youpin.projectmimimi", "dfz", "1.0.0","modulex","functiony"); + gen.generateAndZip("/tmp/work", "project-faas", "com.xiaomi.youpin", "com.xiaomi.youpin.projectmimimi", "dfz", "1.0.0", "modulex", "functiony"); } } From f3cdec08040029e389b1da44413ed99934e71c87 Mon Sep 17 00:00:00 2001 From: EricDing <128116675+sadadw1@users.noreply.github.com> Date: Mon, 19 Aug 2024 17:08:39 +0800 Subject: [PATCH 02/11] fix: datasource close 20240819 (#879) * feat: add prometheus-trace-etl release * feat: add gitlab log release jdk21 * feat: update athena * feat: update check code file path * feat: add code generator and mybatis generator * feat: add mybtais generator * fix: update test error * fix: add HikariDataSource close --- .../run/mone/ai/codegen/FeatureGenerator.java | 45 +++++++++---------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/jcommon/codegen/src/main/java/run/mone/ai/codegen/FeatureGenerator.java b/jcommon/codegen/src/main/java/run/mone/ai/codegen/FeatureGenerator.java index 5ddc8b3df..b5f946410 100644 --- a/jcommon/codegen/src/main/java/run/mone/ai/codegen/FeatureGenerator.java +++ b/jcommon/codegen/src/main/java/run/mone/ai/codegen/FeatureGenerator.java @@ -8,15 +8,11 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.jdbc.core.JdbcTemplate; -import run.mone.ai.codegen.bo.FeatureGenerateType; import run.mone.ai.codegen.bo.FeatureGenerateBo; +import run.mone.ai.codegen.bo.FeatureGenerateType; import run.mone.ai.codegen.util.TemplateUtils; import java.lang.reflect.Type; -import run.mone.ai.codegen.bo.FeatureGeneratType; -import run.mone.ai.codegen.bo.FeatureGenerateBo; -import run.mone.ai.codegen.util.TemplateUtils; - import java.util.Base64; import java.util.HashMap; import java.util.Map; @@ -141,26 +137,29 @@ public static void generateWithGenerator(FeatureGenerateBo featureGenerateBo) { } //配置数据源 - HikariDataSource dataSource = new HikariDataSource(); - dataSource.setDriverClassName("com.mysql.cj.jdbc.Driver"); - dataSource.setJdbcUrl(featureGenerateBo.getJdbcUrl()); - dataSource.setUsername(featureGenerateBo.getUserName()); - dataSource.setPassword(featureGenerateBo.getPassword()); - - //创建mapper相关代码 - - if (FeatureGenerateType.CODE_WITH_TEMPLATE == featureGenType) { - GlobalConfig globalConfig = createGlobalConfigUseStyle(featureGenerateBo); - Generator generator = new Generator(dataSource, globalConfig); - generator.generate(); - return; - } + try (HikariDataSource dataSource = new HikariDataSource()) { + dataSource.setDriverClassName("com.mysql.cj.jdbc.Driver"); + dataSource.setJdbcUrl(featureGenerateBo.getJdbcUrl()); + dataSource.setUsername(featureGenerateBo.getUserName()); + dataSource.setPassword(featureGenerateBo.getPassword()); + + //创建mapper相关代码 + + if (FeatureGenerateType.CODE_WITH_TEMPLATE == featureGenType) { + GlobalConfig globalConfig = createGlobalConfigUseStyle(featureGenerateBo); + Generator generator = new Generator(dataSource, globalConfig); + generator.generate(); + return; + } - //创建table + //创建table - if (FeatureGenerateType.TABLE == featureGenType) { - JdbcTemplate jt = new JdbcTemplate(dataSource); - jt.update(featureGenerateBo.getSql()); + if (FeatureGenerateType.TABLE == featureGenType) { + JdbcTemplate jt = new JdbcTemplate(dataSource); + jt.update(featureGenerateBo.getSql()); + } + } catch (Exception e) { + log.error("generate with template error", e); } } From 8a3985f9bf398a732c724ed7795b9d6de5048af3 Mon Sep 17 00:00:00 2001 From: caobaoyu Date: Tue, 20 Aug 2024 11:20:21 +0800 Subject: [PATCH 03/11] feat: update Gitlab.java with change clone from private to public --- .../gitlab/src/main/java/com/xiaomi/youpin/gitlab/Gitlab.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jcommon/gitlab/src/main/java/com/xiaomi/youpin/gitlab/Gitlab.java b/jcommon/gitlab/src/main/java/com/xiaomi/youpin/gitlab/Gitlab.java index 718043bc2..418d060bd 100644 --- a/jcommon/gitlab/src/main/java/com/xiaomi/youpin/gitlab/Gitlab.java +++ b/jcommon/gitlab/src/main/java/com/xiaomi/youpin/gitlab/Gitlab.java @@ -516,7 +516,7 @@ private static boolean modifyFile(String changedBody, String filePathToReplace, return true; } - private static boolean clone(String gitUrl, String branch, String username, String token, String gitPath) { + public static boolean clone(String gitUrl, String branch, String username, String token, String gitPath) { clearIfPresent(gitPath); CloneCommand cloneCommand = Git.cloneRepository().setURI(gitUrl) .setCredentialsProvider(new UsernamePasswordCredentialsProvider(username, token)) From fa5254ed6543a3f7488699b894e5b204994471c8 Mon Sep 17 00:00:00 2001 From: zhangzhiyong Date: Tue, 20 Aug 2024 14:29:18 +0800 Subject: [PATCH 04/11] chore: update test files and MoneCodeParser.java with no specific changes --- .../java/run/mone/neo4j/MoneCodeParser.java | 112 +++++++++++++++++- .../mone/neo4j/test/MoneCodeParserTest.java | 8 +- .../run/mone/neo4j/test/anno/Resource.java | 8 ++ .../mone/neo4j/test/anno/RestController.java | 8 ++ .../java/run/mone/neo4j/test/anno/Table.java | 8 ++ .../test/java/run/mone/neo4j/test/m/Cat.java | 21 ++++ .../run/mone/neo4j/test/m/CatService.java | 25 ++++ .../test/java/run/mone/neo4j/test/m/Dog.java | 11 ++ .../java/run/mone/neo4j/test/m/Person.java | 11 ++ .../test/{A.java => m/PersonService.java} | 12 +- 10 files changed, 216 insertions(+), 8 deletions(-) create mode 100644 jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Resource.java create mode 100644 jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/RestController.java create mode 100644 jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Table.java create mode 100644 jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Cat.java create mode 100644 jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java create mode 100644 jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Dog.java create mode 100644 jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Person.java rename jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/{A.java => m/PersonService.java} (72%) diff --git a/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java index 328dc825b..db7acf099 100644 --- a/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java +++ b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java @@ -3,9 +3,12 @@ import com.github.javaparser.JavaParser; import com.github.javaparser.ast.CompilationUnit; import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration; +import com.github.javaparser.ast.body.FieldDeclaration; import com.github.javaparser.ast.body.MethodDeclaration; import com.github.javaparser.ast.comments.Comment; import com.github.javaparser.ast.comments.JavadocComment; +import com.github.javaparser.ast.expr.AnnotationExpr; +import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonObject; @@ -16,6 +19,7 @@ import okhttp3.*; import org.neo4j.driver.Record; import org.neo4j.driver.*; +import org.neo4j.driver.types.Node; import java.io.File; import java.io.IOException; @@ -35,6 +39,29 @@ public class MoneCodeParser { private String embeddingUrl = ""; + public void queryEntityClasses() { + try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, NEO4J_PASSWORD)); + Session session = driver.session()) { + // 查询 type 为 'entity' 的所有 Class 节点 + String query = "MATCH (c:Class {type: 'entity'}) RETURN c"; + Result result = session.run(query); + while (result.hasNext()) { + Record record = result.next(); + Node classNode = record.get("c").asNode(); + String name = classNode.get("name").asString(); + String fullName = classNode.get("full_name").asString(); + String type = classNode.get("type").asString(); + + // 输出或处理查询结果 + System.out.println("Class Name: " + name); + System.out.println("Full Name: " + fullName); + System.out.println("Type: " + type); + System.out.println(classNode.get("code").asString()); + } + } + } + + //查询所有Comment的信息(使用neo4j),返回是个List(class) public List> getAllComments() { try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, NEO4J_PASSWORD)); @@ -52,13 +79,25 @@ public List> getAllComments() { } + //给你ClassOrInterfaceDeclaration,帮我过滤掉所有method中的body,返回这个class的String内容(class) + public static String filterMethodBodies(ClassOrInterfaceDeclaration classOrInterfaceDeclaration) { + classOrInterfaceDeclaration.getMethods().forEach(method -> method.setBody(null)); + return classOrInterfaceDeclaration.toString(); + } + + //给你ClassOrInterfaceDeclaration,帮我删除掉所有method,返回这个class的String内容(class) + public static String removeAllMethods(ClassOrInterfaceDeclaration classOrInterfaceDeclaration) { + classOrInterfaceDeclaration.getMethods().forEach(MethodDeclaration::remove); + return classOrInterfaceDeclaration.toString(); + } + /** * 根据文本向量查询评论 * * @param text 输入的文本,用于生成查询向量 * @return 查询结果的列表,包含评论节点和相似度分数 */ - @SneakyThrows + @SneakyThrows public List> queryCommentsByTextVector(String text) { // 替换为你的查询向量 double[] queryVector = getTextVectorFromHttp(text); @@ -118,7 +157,7 @@ public double[] convertListToFloatArray(List floatList) { * @param vectorB 第二个向量 * @return 两个向量的余弦相似度 */ - private double calculateCosineSimilarity(double[] vectorA, double[] vectorB) { + private double calculateCosineSimilarity(double[] vectorA, double[] vectorB) { double dotProduct = 0.0; double normA = 0.0; double normB = 0.0; @@ -166,7 +205,7 @@ public void updateCommentsInNeo4j(List> comments) { private static Gson gson = new Gson(); - private double[] getTextVectorFromHttp(String text) throws IOException { + private double[] getTextVectorFromHttp(String text) throws IOException { JsonObject jsonRequest = new JsonObject(); jsonRequest.addProperty("text", text); @@ -243,7 +282,7 @@ public void deleteAllNodes() { * * @param filePath Java文件的路径 */ - @SneakyThrows + @SneakyThrows private void writeToNeo4j(String filePath) { //写入到neo4j中 // 替换成你的 Java 文件路径 @@ -272,6 +311,7 @@ private static void createProjectNode(Session session, String projectName) { session.run("MERGE (p:Project {name: $name})", projectParams); } + private static void createFileNode(Session session, String projectName, String filePath) { Map fileParams = new HashMap<>(); fileParams.put("name", filePath); @@ -308,7 +348,27 @@ public void visit(ClassOrInterfaceDeclaration n, Void arg) { classParams.put("name", n.getNameAsString()); classParams.put("fullName", n.getFullyQualifiedName().orElse("")); - session.run("MERGE (c:Class {name: $name, full_name: $fullName})", classParams); + //class 的类型 + String type = getControllerType(n); + classParams.put("type", type); + + String code = ""; + + if (type.equals("entity")) { + code = removeAllMethods(n); + } + + classParams.put("code", code); + + System.out.println(classParams); + + + session.run( + "MERGE (c:Class {name: $name}) " + + "ON CREATE SET c.full_name = $fullName, c.type = $type, c.code = $code " + + "ON MATCH SET c.full_name = $fullName, c.type = $type, c.code = $code", + classParams + ); // 创建 CONTAINS 关系 (File -[:CONTAINS]-> Class) Map containsParams = new HashMap<>(); @@ -319,6 +379,33 @@ public void visit(ClassOrInterfaceDeclaration n, Void arg) { "MERGE (f)-[:CONTAINS]->(c)", containsParams); + + // 处理字段声明,查找 @Resource 注解 + n.findAll(FieldDeclaration.class).forEach(field -> { + field.getAnnotations().forEach(annotation -> { + if (annotation.getNameAsString().equals("Resource")) { + String fieldName = field.getVariables().get(0).getNameAsString(); + String fieldType = field.getElementType().asString(); + + // 创建 DEPENDS_ON 关系 (Class -[:DEPENDS_ON]-> Service) + Map dependsOnParams = new HashMap<>(); + dependsOnParams.put("className", n.getNameAsString()); + dependsOnParams.put("serviceName", fieldType); + dependsOnParams.put("fieldName", fieldName); + + + session.run("MERGE (c:Class {name: $name})", ImmutableMap.of("name",fieldType)); + + session.run("MATCH (c:Class {name: $className}) " + + "MATCH (s:Class {name: $serviceName}) " + + "MERGE (c)-[:DEPENDS_ON {field: $fieldName}]->(s)", + dependsOnParams); + } + }); + }); + + + super.visit(n, arg); } @@ -387,5 +474,20 @@ private void createCommentNode(Comment comment, MethodDeclaration n) { } + private static String getControllerType(ClassOrInterfaceDeclaration n) { + String type = ""; + Optional optional = n.getAnnotationByName("RestController"); + if (optional.isPresent()) { + type = "controller"; + } + + optional = n.getAnnotationByName("Table"); + if (optional.isPresent()) { + type = "entity"; + } + + return type; + } + } diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java index 0c83cb788..057a02e35 100644 --- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java @@ -12,9 +12,15 @@ public class MoneCodeParserTest { @Test public void testWriteCatServiceToNeo4j() { - new MoneCodeParser().writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/ai/m78/m78-service/src/main/java/run/mone/m78/service"); +// new MoneCodeParser().writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/ai/m78/m78-service/src/main/java/run/mone/m78/service"); // MoneCodeParser.writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/ai/m78/m78-service/src/main/java/run/mone/m78/service/database"); // MoneCodeParser.writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/ai/m78/m78-service/src/main/java/run/mone/m78/service/database/SqlParseUtil.java"); // new MoneCodeParser().writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/A.java"); + new MoneCodeParser().writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m"); + } + + @Test + public void test1() { + new MoneCodeParser().queryEntityClasses(); } } diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Resource.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Resource.java new file mode 100644 index 000000000..2366ed8d8 --- /dev/null +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Resource.java @@ -0,0 +1,8 @@ +package run.mone.neo4j.test.anno; + +/** + * @author goodjava@qq.com + * @date 2024/8/19 18:20 + */ +public @interface Resource { +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/RestController.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/RestController.java new file mode 100644 index 000000000..770f21261 --- /dev/null +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/RestController.java @@ -0,0 +1,8 @@ +package run.mone.neo4j.test.anno; + +/** + * @author goodjava@qq.com + * @date 2024/8/19 16:51 + */ +public @interface RestController { +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Table.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Table.java new file mode 100644 index 000000000..e009de88d --- /dev/null +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Table.java @@ -0,0 +1,8 @@ +package run.mone.neo4j.test.anno; + +/** + * @author goodjava@qq.com + * @date 2024/8/19 16:50 + */ +public @interface Table { +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Cat.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Cat.java new file mode 100644 index 000000000..5e3c87714 --- /dev/null +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Cat.java @@ -0,0 +1,21 @@ +package run.mone.neo4j.test.m; + +import run.mone.neo4j.test.anno.Table; + +/** + * @author goodjava@qq.com + * @date 2024/8/19 17:03 + */ +@Table +public class Cat { + + private int id; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java new file mode 100644 index 000000000..92ed980cb --- /dev/null +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java @@ -0,0 +1,25 @@ +package run.mone.neo4j.test.m; + +import run.mone.neo4j.test.anno.RestController; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author goodjava@qq.com + * @date 2024/8/19 18:21 + */ +@RestController +public class CatService { + + private Map data = new HashMap<>(); + + + //获取小猫的数量 + //获取小猫的数量 + public int getCatCount() { + return data.size(); + } + + +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Dog.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Dog.java new file mode 100644 index 000000000..2811cc06d --- /dev/null +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Dog.java @@ -0,0 +1,11 @@ +package run.mone.neo4j.test.m; + +import run.mone.neo4j.test.anno.Table; + +/** + * @author goodjava@qq.com + * @date 2024/8/19 17:03 + */ +@Table +public class Dog { +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Person.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Person.java new file mode 100644 index 000000000..1c67046be --- /dev/null +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Person.java @@ -0,0 +1,11 @@ +package run.mone.neo4j.test.m; + +import run.mone.neo4j.test.anno.Table; + +/** + * @author goodjava@qq.com + * @date 2024/8/19 16:51 + */ +@Table +public class Person { +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/A.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java similarity index 72% rename from jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/A.java rename to jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java index 8281e28ce..65794057e 100644 --- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/A.java +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java @@ -1,11 +1,19 @@ -package run.mone.neo4j.test; +package run.mone.neo4j.test.m; + +import run.mone.neo4j.test.anno.Resource; +import run.mone.neo4j.test.anno.RestController; /** * @author goodjava@qq.com * @date 2024/8/16 10:16 * AAA */ -public class A { +@RestController +public class PersonService { + + + @Resource + private CatService catService; /** From 8229cdc0655b24290520849dcc25bd5ed7c1ff9c Mon Sep 17 00:00:00 2001 From: zhangzhiyong Date: Wed, 21 Aug 2024 18:20:02 +0800 Subject: [PATCH 05/11] chore: update test resources and service classes --- .../src/main/java/run/mone/neo4j/BotCall.java | 64 ++++++++++++++ .../java/run/mone/neo4j/MoneCodeParser.java | 84 +++++++++++++++---- .../mone/neo4j/test/MoneCodeParserTest.java | 78 ++++++++++++++++- .../run/mone/neo4j/test/anno/Service.java | 8 ++ .../run/mone/neo4j/test/m/CatService.java | 3 +- .../test/java/run/mone/neo4j/test/m/Dog.java | 3 + .../run/mone/neo4j/test/m/PersonService.java | 4 +- jcommon/ai/neo4j/src/test/resources/entity.md | 15 ++++ .../ai/neo4j/src/test/resources/service.md | 16 ++++ 9 files changed, 257 insertions(+), 18 deletions(-) create mode 100644 jcommon/ai/neo4j/src/main/java/run/mone/neo4j/BotCall.java create mode 100644 jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Service.java create mode 100644 jcommon/ai/neo4j/src/test/resources/entity.md create mode 100644 jcommon/ai/neo4j/src/test/resources/service.md diff --git a/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/BotCall.java b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/BotCall.java new file mode 100644 index 000000000..0930633ac --- /dev/null +++ b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/BotCall.java @@ -0,0 +1,64 @@ +package run.mone.neo4j; + +import com.google.gson.JsonObject; +import lombok.Setter; +import okhttp3.*; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +/** + * @author goodjava@qq.com + * @date 2024/8/21 13:50 + */ +public class BotCall { + + //调用bot的地址 + @Setter + private static String url = ""; + + + public static String call(String desc, String input) { + OkHttpClient client = new OkHttpClient.Builder() + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(30, TimeUnit.SECONDS) + .writeTimeout(30, TimeUnit.SECONDS) + .build(); + + // 使用 Gson 的 JsonObject 构建请求体 + JsonObject mainObject = new JsonObject(); + mainObject.addProperty("userName", ""); + mainObject.addProperty("botId", ""); + mainObject.addProperty("input", ""); + + JsonObject paramsObject = new JsonObject(); + paramsObject.addProperty("desc", desc); + paramsObject.addProperty("input", input); + mainObject.add("params", paramsObject); + + // 将 JsonObject 转换为字符串 + String jsonBody = mainObject.toString(); + + RequestBody body = RequestBody.create(MediaType.parse("application/json"), jsonBody); + + Request request = new Request.Builder() + .url(url) + .post(body) + .addHeader("Accept", "application/json, text/plain, */*") + .addHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8") + .addHeader("Cache-Control", "no-cache") + .addHeader("Connection", "keep-alive") + .addHeader("Content-Type", "application/json") + .addHeader("Authorization", "") + .build(); + + try { + Response response = client.newCall(request).execute(); + return response.body().string(); + } catch (IOException e) { + e.printStackTrace(); + } + return ""; + } + +} diff --git a/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java index db7acf099..aa83443de 100644 --- a/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java +++ b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java @@ -8,6 +8,7 @@ import com.github.javaparser.ast.comments.Comment; import com.github.javaparser.ast.comments.JavadocComment; import com.github.javaparser.ast.expr.AnnotationExpr; +import com.github.javaparser.ast.nodeTypes.NodeWithName; import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -21,9 +22,10 @@ import org.neo4j.driver.*; import org.neo4j.driver.types.Node; -import java.io.File; -import java.io.IOException; +import java.io.*; +import java.nio.charset.StandardCharsets; import java.util.*; +import java.util.stream.Collectors; @Slf4j @Data @@ -34,13 +36,18 @@ public class MoneCodeParser { private String NEO4J_USER = "neo4j"; - private String NEO4J_PASSWORD = ""; + private String password = ""; private String embeddingUrl = ""; + public MoneCodeParser setPassword(String password) { + this.password = password; + return this; + } + public void queryEntityClasses() { - try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, NEO4J_PASSWORD)); + try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, password)); Session session = driver.session()) { // 查询 type 为 'entity' 的所有 Class 节点 String query = "MATCH (c:Class {type: 'entity'}) RETURN c"; @@ -61,10 +68,15 @@ public void queryEntityClasses() { } } + //获取session(class) + public Session getSession() { + return GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, password)).session(); + } + //查询所有Comment的信息(使用neo4j),返回是个List(class) public List> getAllComments() { - try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, NEO4J_PASSWORD)); + try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, password)); Session session = driver.session()) { List> comments = new ArrayList<>(); Result result = session.run("MATCH (comment:Comment) RETURN comment, id(comment) as commentId"); @@ -102,7 +114,7 @@ public List> queryCommentsByTextVector(String text) { // 替换为你的查询向量 double[] queryVector = getTextVectorFromHttp(text); - try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, NEO4J_PASSWORD)); + try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, password)); Session session = driver.session()) { // 执行查询 @@ -186,7 +198,7 @@ public void fillCommentTextVectors() { //把Comment的修改,刷新回neo4j(class) public void updateCommentsInNeo4j(List> comments) { - try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, NEO4J_PASSWORD)); + try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, password)); Session session = driver.session()) { for (Map comment : comments) { Long commentId = (Long) comment.get("commentId"); @@ -271,7 +283,7 @@ public static List getJavaFilesInDirectory(String directoryPath) { //删除所有节点(class) public void deleteAllNodes() { - try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, NEO4J_PASSWORD)); + try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, password)); Session session = driver.session()) { session.run("MATCH (n) DETACH DELETE n"); } @@ -288,7 +300,7 @@ private void writeToNeo4j(String filePath) { // 替换成你的 Java 文件路径 String projectName = "MyProject"; - try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, NEO4J_PASSWORD)); + try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, password)); Session session = driver.session()) { // 解析 Java 文件 @@ -311,6 +323,31 @@ private static void createProjectNode(Session session, String projectName) { session.run("MERGE (p:Project {name: $name})", projectParams); } + /** + * 查找具有指定注解的类 + * + * @param session 数据库会话 + * @param annotationToFind 要查找的注解 + * @return 具有指定注解的类的列表,每个类以Map形式表示 + */ + public List> findClassesWithAnnotation(Session session, String annotationToFind) { + Map params = new HashMap<>(); + params.put("annotation", annotationToFind); + Result result = session.run( + "MATCH (c:Class) " + + "WHERE ANY(anno IN c.anno WHERE anno = $annotation) " + + "RETURN c", + params + ); + List> list = new ArrayList<>(); + while (result.hasNext()) { + Record record = result.next(); + System.out.println(record.get("c").asMap()); + list.add(record.get("c").asMap()); + } + return list; + } + private static void createFileNode(Session session, String projectName, String filePath) { Map fileParams = new HashMap<>(); @@ -341,6 +378,7 @@ public Visitor(Session session, String projectName, String filePath) { this.filePath = filePath; } + @Override public void visit(ClassOrInterfaceDeclaration n, Void arg) { // 创建 Class/Interface 节点 @@ -360,13 +398,20 @@ public void visit(ClassOrInterfaceDeclaration n, Void arg) { classParams.put("code", code); - System.out.println(classParams); + //获取ClassOrInterfaceDeclaration中的注解 + List annoList = n.getAnnotations().stream().peek(annotation -> { + System.out.println("Annotation: " + annotation.getNameAsString()); + }).map(NodeWithName::getNameAsString).toList(); + //注解 + classParams.put("annotations", annoList); + + System.out.println(classParams); session.run( "MERGE (c:Class {name: $name}) " + - "ON CREATE SET c.full_name = $fullName, c.type = $type, c.code = $code " + - "ON MATCH SET c.full_name = $fullName, c.type = $type, c.code = $code", + "ON CREATE SET c.full_name = $fullName, c.type = $type, c.code = $code, c.anno = $annotations " + + "ON MATCH SET c.full_name = $fullName, c.type = $type, c.code = $code, c.anno = $annotations", classParams ); @@ -394,7 +439,7 @@ public void visit(ClassOrInterfaceDeclaration n, Void arg) { dependsOnParams.put("fieldName", fieldName); - session.run("MERGE (c:Class {name: $name})", ImmutableMap.of("name",fieldType)); + session.run("MERGE (c:Class {name: $name})", ImmutableMap.of("name", fieldType)); session.run("MATCH (c:Class {name: $className}) " + "MATCH (s:Class {name: $serviceName}) " + @@ -405,7 +450,6 @@ public void visit(ClassOrInterfaceDeclaration n, Void arg) { }); - super.visit(n, arg); } @@ -489,5 +533,17 @@ private static String getControllerType(ClassOrInterfaceDeclaration n) { return type; } + //读取resource下某个文件的文本内容(class) + public String readResourceFileContent(String fileName) { + try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream(fileName); + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + return reader.lines().collect(Collectors.joining(System.lineSeparator())); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + } diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java index 057a02e35..64577800a 100644 --- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java @@ -1,7 +1,32 @@ package run.mone.neo4j.test; +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; import org.junit.Test; +import org.neo4j.driver.Record; +import org.neo4j.driver.Result; +import org.neo4j.driver.Session; +import org.neo4j.driver.internal.InternalRecord; +import org.neo4j.driver.internal.InternalResult; +import org.neo4j.driver.internal.InternalSession; +import run.mone.neo4j.BotCall; import run.mone.neo4j.MoneCodeParser; +import run.mone.neo4j.test.MoneCodeParserTest; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + /** * @author goodjava@qq.com @@ -10,17 +35,68 @@ public class MoneCodeParserTest { + @Test public void testWriteCatServiceToNeo4j() { // new MoneCodeParser().writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/ai/m78/m78-service/src/main/java/run/mone/m78/service"); // MoneCodeParser.writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/ai/m78/m78-service/src/main/java/run/mone/m78/service/database"); // MoneCodeParser.writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/ai/m78/m78-service/src/main/java/run/mone/m78/service/database/SqlParseUtil.java"); // new MoneCodeParser().writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/A.java"); - new MoneCodeParser().writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m"); + new MoneCodeParser().setPassword(System.getenv("password")).writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m"); } @Test public void test1() { new MoneCodeParser().queryEntityClasses(); } + + + + + @Test + public void testFindClassesWithAnnotation() { + MoneCodeParser moneCodeParser = new MoneCodeParser().setPassword(System.getenv("password")); + List> actual = moneCodeParser.findClassesWithAnnotation(moneCodeParser.getSession(), "Table"); + System.out.println(actual); + +// String xuqiu = "获取狗的主人"; + String xuqiu = "管理宠物鸟"; + String res = BotCall.call(moneCodeParser.readResourceFileContent("entity.md") + xuqiu, new Gson().toJson(actual)); + System.out.println(res); + } + + + + @Test + public void testFindClassesWithAnnotation2() { + MoneCodeParser moneCodeParser = new MoneCodeParser().setPassword(System.getenv("password")); + List> actual = moneCodeParser.findClassesWithAnnotation(moneCodeParser.getSession(), "RestController"); + System.out.println(actual); + + String xuqiu = "获取狗的主人"; +// String xuqiu = "管理宠物鸟"; + String res = BotCall.call(moneCodeParser.readResourceFileContent("service.md") + xuqiu, new Gson().toJson(actual)); + System.out.println(res); + } + + + @Test + public void testJson() { + Map m = ImmutableMap.of("input", "a+b=?"); + System.out.println(new Gson().toJson(m)); + } + + + @Test + public void testReadResourceFileContent() { + String fileName = "entity.md"; + // Assuming the test resource file is already placed in the resources directory + String actualContent = new MoneCodeParser().readResourceFileContent(fileName); + + assertNotNull(actualContent); + } + + + + } diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Service.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Service.java new file mode 100644 index 000000000..b76563edf --- /dev/null +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Service.java @@ -0,0 +1,8 @@ +package run.mone.neo4j.test.anno; + +/** + * @author goodjava@qq.com + * @date 2024/8/21 16:06 + */ +public @interface Service { +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java index 92ed980cb..623cf6578 100644 --- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java @@ -1,6 +1,7 @@ package run.mone.neo4j.test.m; import run.mone.neo4j.test.anno.RestController; +import run.mone.neo4j.test.anno.Service; import java.util.HashMap; import java.util.Map; @@ -9,7 +10,7 @@ * @author goodjava@qq.com * @date 2024/8/19 18:21 */ -@RestController +@Service public class CatService { private Map data = new HashMap<>(); diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Dog.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Dog.java index 2811cc06d..4b5dc12c3 100644 --- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Dog.java +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Dog.java @@ -8,4 +8,7 @@ */ @Table public class Dog { + + private int id; + } diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java index 65794057e..a94ae0dfa 100644 --- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java @@ -2,13 +2,13 @@ import run.mone.neo4j.test.anno.Resource; import run.mone.neo4j.test.anno.RestController; +import run.mone.neo4j.test.anno.Service; /** * @author goodjava@qq.com * @date 2024/8/16 10:16 - * AAA */ -@RestController +@Service public class PersonService { diff --git a/jcommon/ai/neo4j/src/test/resources/entity.md b/jcommon/ai/neo4j/src/test/resources/entity.md new file mode 100644 index 000000000..0b8529b60 --- /dev/null +++ b/jcommon/ai/neo4j/src/test/resources/entity.md @@ -0,0 +1,15 @@ +我有这些表映射类,你帮我分析.我给你一个新功能,你看看能不能满足. +不满足的原因: +1.缺少相应的字段 +2.就没有这个类 + +如果不满足,你需要按如下规定返回: +1.缺少字段 satisfies:false reason:missing_fields code:返回缺少字段的代码 (example: private String name;) +2.缺少某个类 satisfies:false reason:missing_class code:返回缺少的类的代码 + +一些规则: +1.缺少字段的时候,尽量不要添加类字段,尽量引用那个类的id即可. +2.如果缺少类,你要帮我尽量补全这个类中缺少的字段. +3.生成的字段或者类尽量添加合适的中文注释 + +需求: \ No newline at end of file diff --git a/jcommon/ai/neo4j/src/test/resources/service.md b/jcommon/ai/neo4j/src/test/resources/service.md new file mode 100644 index 000000000..0670fbd6e --- /dev/null +++ b/jcommon/ai/neo4j/src/test/resources/service.md @@ -0,0 +1,16 @@ +我有这些表映射类和service类的信息,然后我给你一个新功能,你分析下看看能不能满足. + +不满足的原因: +1.缺少相应的方法 +2.缺少某个service类 + +如果不满足,你需要按如下规定返回: +1.缺少字段 satisfies:false reason:missing_methods code:返回缺少方法的代码 (example: private String name;) +2.缺少某个service类 satisfies:false reason:missing_class code:返回缺少的类的代码 + +一些规则: +1.缺少method的时候,生成的方法只需要给方法定义即可. +2.如果缺少类,你要帮我生成这个类,类中的方法也只需要生成接口定义. +3.生成的方法或者类尽量添加合适的中文注释 + +需求: \ No newline at end of file From b5f078b66fdc4432b673070cf89d30aab2bdb1e1 Mon Sep 17 00:00:00 2001 From: dingpei Date: Thu, 22 Aug 2024 22:31:20 +0800 Subject: [PATCH 06/11] chore: update RequestPayload.java with no content changes --- .../src/main/java/run/mone/ai/google/bo/RequestPayload.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jcommon/ai/google/src/main/java/run/mone/ai/google/bo/RequestPayload.java b/jcommon/ai/google/src/main/java/run/mone/ai/google/bo/RequestPayload.java index b649a1331..ac55968a9 100644 --- a/jcommon/ai/google/src/main/java/run/mone/ai/google/bo/RequestPayload.java +++ b/jcommon/ai/google/src/main/java/run/mone/ai/google/bo/RequestPayload.java @@ -14,6 +14,8 @@ @Builder public class RequestPayload { + @SerializedName("system") + private String system; @SerializedName("anthropic_version") private String anthropicVersion; From 5303918a15a5300f3d6cde1d4d6e8553fc2e14fc Mon Sep 17 00:00:00 2001 From: zhangzhiyong Date: Tue, 27 Aug 2024 14:44:44 +0800 Subject: [PATCH 07/11] Code Map Update Test --- .../java/run/mone/neo4j/MoneCodeParser.java | 171 ++++++++++++++++-- .../mone/neo4j/test/MoneCodeParserTest.java | 2 + .../java/run/mone/neo4j/test/anno/Test.java | 8 + .../run/mone/neo4j/test/m/PersonService.java | 12 +- .../mone/neo4j/test/m/PersonServiceTest.java | 22 +++ .../neo4j/test/m/{ => cat}/CatService.java | 10 +- 6 files changed, 209 insertions(+), 16 deletions(-) create mode 100644 jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Test.java create mode 100644 jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonServiceTest.java rename jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/{ => cat}/CatService.java (60%) diff --git a/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java index aa83443de..df1647094 100644 --- a/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java +++ b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java @@ -2,13 +2,16 @@ import com.github.javaparser.JavaParser; import com.github.javaparser.ast.CompilationUnit; +import com.github.javaparser.ast.ImportDeclaration; import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration; import com.github.javaparser.ast.body.FieldDeclaration; import com.github.javaparser.ast.body.MethodDeclaration; import com.github.javaparser.ast.comments.Comment; import com.github.javaparser.ast.comments.JavadocComment; -import com.github.javaparser.ast.expr.AnnotationExpr; +import com.github.javaparser.ast.expr.*; import com.github.javaparser.ast.nodeTypes.NodeWithName; +import com.github.javaparser.ast.type.ClassOrInterfaceType; +import com.github.javaparser.resolution.declarations.ResolvedMethodDeclaration; import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -326,11 +329,11 @@ private static void createProjectNode(Session session, String projectName) { /** * 查找具有指定注解的类 * - * @param session 数据库会话 + * @param session 数据库会话 * @param annotationToFind 要查找的注解 * @return 具有指定注解的类的列表,每个类以Map形式表示 */ - public List> findClassesWithAnnotation(Session session, String annotationToFind) { + public List> findClassesWithAnnotation(Session session, String annotationToFind) { Map params = new HashMap<>(); params.put("annotation", annotationToFind); Result result = session.run( @@ -339,7 +342,7 @@ public List> findClassesWithAnnotation(Session session, Stri "RETURN c", params ); - List> list = new ArrayList<>(); + List> list = new ArrayList<>(); while (result.hasNext()) { Record record = result.next(); System.out.println(record.get("c").asMap()); @@ -406,8 +409,6 @@ public void visit(ClassOrInterfaceDeclaration n, Void arg) { //注解 classParams.put("annotations", annoList); - System.out.println(classParams); - session.run( "MERGE (c:Class {name: $name}) " + "ON CREATE SET c.full_name = $fullName, c.type = $type, c.code = $code, c.anno = $annotations " + @@ -454,13 +455,135 @@ public void visit(ClassOrInterfaceDeclaration n, Void arg) { } + + private String getFullMethodName(MethodDeclaration method) { + String packageName = method.findCompilationUnit() + .flatMap(cu -> cu.getPackageDeclaration()) + .map(pd -> pd.getNameAsString()) + .orElse(""); + String className = method.findAncestor(com.github.javaparser.ast.body.ClassOrInterfaceDeclaration.class) + .map(c -> c.getNameAsString()) + .orElse(""); + String methodName = method.getNameAsString(); + return packageName + "." + className + "." + methodName; + } + + /** + * 获取方法调用的完整路径,包括包名、类名和方法名 + * + * @param methodCall 方法调用表达式 + * @return 方法调用的完整路径 + */ + public String getFullMethodPath(MethodCallExpr methodCall) { + StringBuilder fullPath = new StringBuilder(); + + // 获取包名 + Optional cu = methodCall.findCompilationUnit(); + if (cu.isPresent()) { + cu.get().getPackageDeclaration().ifPresent(pkg -> + fullPath.append(pkg.getNameAsString()).append(".") + ); + } + + // 获取类名 + String className = methodCall.findAncestor(ClassOrInterfaceDeclaration.class) + .map(ClassOrInterfaceDeclaration::getNameAsString) + .orElse(""); + + // 获取方法调用的对象 + String objectName = methodCall.getScope() + .map(scope -> scope.toString()) + .orElse(""); + + + //静态调用 + if (methodCall.getScope().isPresent() && methodCall.getScope().get() instanceof FieldAccessExpr) { + return objectName + "." + methodCall.getNameAsString(); + } + + //lombok 的log + if (isLogCall(methodCall)) { + return objectName + "." + methodCall.getNameAsString(); + } + + // 如果对象名不为空,尝试找到它的类型 + if (!objectName.isEmpty()) { + Optional field = methodCall.findAncestor(ClassOrInterfaceDeclaration.class) + .flatMap(classDecl -> classDecl.getFieldByName(objectName)); + + if (field.isPresent()) { + ClassOrInterfaceType type = field.get().getVariable(0).getType().asClassOrInterfaceType(); + String v = resolveTypePath(type); + return v + "." + methodCall.getNameAsString(); + } + } + + + // 构建完整路径 + fullPath.append(className).append("."); + fullPath.append(methodCall.getNameAsString()); + + return fullPath.toString(); + } + + public static String resolveTypePath(ClassOrInterfaceType type) { + String typeName = type.getNameAsString(); + + Optional cu = type.findAncestor(CompilationUnit.class); + if (cu.isPresent()) { + // 尝试从导入声明中查找匹配 + Optional importedPath = findMatchingImport(cu.get(), typeName); + if (importedPath.isPresent()) { + return importedPath.get(); + } + + // 如果没有找到匹配的导入,检查是否在同一包中 + Optional currentPackage = getCurrentPackage(cu.get()); + if (currentPackage.isPresent()) { + return currentPackage.get() + "." + typeName; + } + } + + // 如果无法解析,返回原始类型名称 + return typeName; + } + + private static Optional findMatchingImport(CompilationUnit cu, String typeName) { + return cu.getImports().stream() + .filter(importDecl -> !importDecl.isAsterisk() && importDecl.getNameAsString().endsWith("." + typeName)) + .map(ImportDeclaration::getNameAsString) + .findFirst(); + } + + private static Optional getCurrentPackage(CompilationUnit cu) { + return cu.getPackageDeclaration().map(pd -> pd.getNameAsString()); + } + + /** + * 判断方法调用是否为日志调用 + * + * @param n 方法调用表达式 + * @return 如果方法调用是日志调用则返回true,否则返回false + */ + private boolean isLogCall(MethodCallExpr n) { + if (!n.getScope().isPresent()) { + return false; + } + String scope = n.getScope().get().toString(); + String method = n.getNameAsString(); + return scope.equals("log") && + (method.equals("trace") || method.equals("debug") || method.equals("info") || + method.equals("warn") || method.equals("error")); + } + + @Override public void visit(MethodDeclaration n, Void arg) { super.visit(n, arg); // 创建 Method 节点 Map methodParams = new HashMap<>(); - methodParams.put("name", n.getNameAsString()); + methodParams.put("name", getFullMethodName(n)); methodParams.put("signature", n.getSignature().asString()); methodParams.put("code_vector", new float[]{}); // 替换为实际的代码向量 @@ -474,7 +597,7 @@ public void visit(MethodDeclaration n, Void arg) { } declaresParams.put("className", n.findAncestor(ClassOrInterfaceDeclaration.class).get().getNameAsString()); - declaresParams.put("methodName", n.getNameAsString()); + declaresParams.put("methodName", getFullMethodName(n)); session.run("MATCH (c:Class {name: $className}) " + "MATCH (m:Method {name: $methodName}) " + @@ -482,6 +605,34 @@ public void visit(MethodDeclaration n, Void arg) { declaresParams); // 处理注释 + processComments(n); + + // 处理方法调用 + processMethodCalls(n); + + } + + // 处理方法调用 + private void processMethodCalls(MethodDeclaration n) { + n.findAll(MethodCallExpr.class).forEach(methodCall -> { + Map callParams = new HashMap<>(); + callParams.put("callerName", getFullMethodName(n)); + callParams.put("calleeName", getFullMethodPath(methodCall)); + + // 创建目标方法节点(如果不存在) + session.run("MERGE (callee:Method {name: $calleeName})", callParams); + + // 创建 CALLS 关系 + session.run("MATCH (caller:Method {name: $callerName}) " + + "MATCH (callee:Method {name: $calleeName}) " + + "MERGE (caller)-[:CALLS]->(callee)", + callParams); + }); + } + + + // 处理注释 + private void processComments(MethodDeclaration n) { for (Comment comment : n.getAllContainedComments()) { createCommentNode(comment, n); } @@ -495,7 +646,6 @@ public void visit(MethodDeclaration n, Void arg) { if (commentOptional.isPresent()) { createCommentNode(commentOptional.get(), n); } - } private void createCommentNode(Comment comment, MethodDeclaration n) { @@ -508,7 +658,7 @@ private void createCommentNode(Comment comment, MethodDeclaration n) { // 创建 DOCUMENTS 关系 (Comment -[:DOCUMENTS]-> Method) Map documentsParams = new HashMap<>(); documentsParams.put("commentText", comment.getContent()); - documentsParams.put("methodName", n.getNameAsString()); + documentsParams.put("methodName", getFullMethodName(n)); documentsParams.put("methodSignature", n.getSignature().asString()); session.run("MATCH (comment:Comment {text: $commentText}) " + "MATCH (m:Method {name: $methodName, signature: $methodSignature}) " + @@ -545,5 +695,4 @@ public String readResourceFileContent(String fileName) { } - } diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java index 64577800a..440a1be06 100644 --- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java @@ -43,6 +43,8 @@ public void testWriteCatServiceToNeo4j() { // MoneCodeParser.writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/ai/m78/m78-service/src/main/java/run/mone/m78/service/database/SqlParseUtil.java"); // new MoneCodeParser().writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/A.java"); new MoneCodeParser().setPassword(System.getenv("password")).writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m"); +// new MoneCodeParser().setPassword(System.getenv("password")).writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java"); +// new MoneCodeParser().setPassword(System.getenv("password")).writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java"); } @Test diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Test.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Test.java new file mode 100644 index 000000000..a28757ccb --- /dev/null +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Test.java @@ -0,0 +1,8 @@ +package run.mone.neo4j.test.anno; + +/** + * @author goodjava@qq.com + * @date 2024/8/21 16:06 + */ +public @interface Test { +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java index a94ae0dfa..3201894ad 100644 --- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java @@ -1,14 +1,16 @@ package run.mone.neo4j.test.m; +import lombok.extern.slf4j.Slf4j; import run.mone.neo4j.test.anno.Resource; -import run.mone.neo4j.test.anno.RestController; import run.mone.neo4j.test.anno.Service; +import run.mone.neo4j.test.m.cat.CatService; /** * @author goodjava@qq.com * @date 2024/8/16 10:16 */ @Service +@Slf4j public class PersonService { @@ -16,6 +18,14 @@ public class PersonService { private CatService catService; + //获取猫的数量 + public int getCatCount() { + log.info("info"); + System.out.println("abc"); + return catService.getCatCount(); + } + + /** * 计算两数之和 * diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonServiceTest.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonServiceTest.java new file mode 100644 index 000000000..a1ebd24a3 --- /dev/null +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonServiceTest.java @@ -0,0 +1,22 @@ +package run.mone.neo4j.test.m; + +import run.mone.neo4j.test.anno.Resource; +import run.mone.neo4j.test.anno.Service; +import run.mone.neo4j.test.anno.Test; + +/** + * @author goodjava@qq.com + * @date 2024/8/26 18:23 + */ +@Service +public class PersonServiceTest { + + @Resource + private PersonService personService; + + @Test + public void test1() { + int res = personService.getCatCount(); + System.out.println(res); + } +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/cat/CatService.java similarity index 60% rename from jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java rename to jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/cat/CatService.java index 623cf6578..494c6776c 100644 --- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/cat/CatService.java @@ -1,24 +1,26 @@ -package run.mone.neo4j.test.m; +package run.mone.neo4j.test.m.cat; -import run.mone.neo4j.test.anno.RestController; +import lombok.extern.slf4j.Slf4j; import run.mone.neo4j.test.anno.Service; import java.util.HashMap; -import java.util.Map; /** * @author goodjava@qq.com * @date 2024/8/19 18:21 */ @Service +@Slf4j public class CatService { - private Map data = new HashMap<>(); + private HashMap data = new HashMap<>(); //获取小猫的数量 //获取小猫的数量 public int getCatCount() { + log.info("abc"); + System.out.println("123"); return data.size(); } From 05ec8e90e9b2574a0702664a7173b4e1954ed329 Mon Sep 17 00:00:00 2001 From: dingpei Date: Wed, 28 Aug 2024 14:07:41 +0800 Subject: [PATCH 08/11] chore: update multiple files in gpt module without content changes --- jcommon/ai/gpt/pom.xml | 54 ++++++++++++ .../main/java/run/mone/ai/gpt/GptClient.java | 54 ++++++++++++ .../run/mone/ai/gpt/bo/ResponsePayload.java | 87 +++++++++++++++++++ .../gpt/bo/multiModal/GptVisionContent.java | 22 +++++ .../ai/gpt/bo/multiModal/GptVisionMsg.java | 23 +++++ .../gpt/bo/multiModal/GptVisionRequest.java | 46 ++++++++++ jcommon/ai/pom.xml | 1 + 7 files changed, 287 insertions(+) create mode 100644 jcommon/ai/gpt/pom.xml create mode 100644 jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/GptClient.java create mode 100644 jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/ResponsePayload.java create mode 100644 jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionContent.java create mode 100644 jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionMsg.java create mode 100644 jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionRequest.java diff --git a/jcommon/ai/gpt/pom.xml b/jcommon/ai/gpt/pom.xml new file mode 100644 index 000000000..1e751705d --- /dev/null +++ b/jcommon/ai/gpt/pom.xml @@ -0,0 +1,54 @@ + + + 4.0.0 + + run.mone + ai + 1.4-jdk20-SNAPSHOT + + + gpt + 1.5-jdk8-SNAPSHOT + + + 8 + 8 + UTF-8 + + + + + + com.squareup.okhttp3 + okhttp + 4.10.0 + + + + + + + + + + maven-compiler-plugin + 3.11.0 + + 8 + 8 + true + UTF-8 + + ${project.basedir}/src/main/java + + + + + + + + + + \ No newline at end of file diff --git a/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/GptClient.java b/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/GptClient.java new file mode 100644 index 000000000..49eb76c8a --- /dev/null +++ b/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/GptClient.java @@ -0,0 +1,54 @@ +package run.mone.ai.gpt; + +import com.google.gson.Gson; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import okhttp3.*; +import run.mone.ai.gpt.bo.ResponsePayload; +import run.mone.ai.gpt.bo.multiModal.GptVisionRequest; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +@Data +@Slf4j +public class GptClient { + + private static Gson gson = new Gson(); + + public ResponsePayload visionCall(String url, String token, GptVisionRequest gptVisionRequest) { + return baseCall(url, token, gson.toJson(gptVisionRequest)); + } + + private ResponsePayload baseCall(String url, String token, String bodyStr) { + OkHttpClient client = new OkHttpClient.Builder().readTimeout(5, TimeUnit.MINUTES).build(); + MediaType mediaType = MediaType.parse("application/json; charset=utf-8"); + RequestBody body = RequestBody.create(mediaType, bodyStr); + Request request = new Request.Builder() + .url(url) + .post(body) + .addHeader("api-key", token) + .addHeader("Content-Type", "application/json; charset=utf-8") + .build(); + + try (Response response = client.newCall(request).execute()) { + if (response.code() == 429) { + ResponsePayload res = new ResponsePayload(); + ResponsePayload.Error error = new ResponsePayload.Error(); + error.setCode("429"); + error.setMessage("被gpt限流了"); + res.setError(error); + return res; + } + if (!response.isSuccessful()) throw new IOException("Unexpected code " + response); + // Handle the response + String res = response.body().string(); + log.info("claude3 res:{}", res); + return new Gson().fromJson(res, ResponsePayload.class); + } catch (Throwable e) { + log.error(e.getMessage(), e); + } + return null; + } + +} diff --git a/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/ResponsePayload.java b/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/ResponsePayload.java new file mode 100644 index 000000000..1aabe2c5d --- /dev/null +++ b/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/ResponsePayload.java @@ -0,0 +1,87 @@ +package run.mone.ai.gpt.bo; + +import lombok.Data; + +import java.util.List; + +@Data +public class ResponsePayload { + private List choices; + private long created; + private String id; + private String model; + private String object; + private List prompt_filter_results; + private String system_fingerprint; + private Usage usage; + private Error error; + + @Data + public static class Error { + private String code; + private String message; + } + + @Data + public static class Choice { + private ContentFilterResults content_filter_results; + private String finish_reason; + private int index; + private Message message; + + + @Data + public static class ContentFilterResults { + private FilterResult hate; + private FilterResult self_harm; + private FilterResult sexual; + private FilterResult violence; + + } + + @Data + public static class FilterResult { + private boolean filtered; + private String severity; + + } + + @Data + public static class Message { + private String content; + private String role; + + } + } + + @Data + public static class PromptFilterResult { + private int prompt_index; + private ContentFilterResult content_filter_result; + + + @Data + public static class ContentFilterResult { + private JailbreakResult jailbreak; + private Choice.FilterResult sexual; + private Choice.FilterResult violence; + private Choice.FilterResult hate; + private Choice.FilterResult self_harm; + + + @Data + public static class JailbreakResult { + private boolean filtered; + private boolean detected; + + } + } + } + + @Data + public static class Usage { + private int completion_tokens; + private int prompt_tokens; + private int total_tokens; + } +} diff --git a/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionContent.java b/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionContent.java new file mode 100644 index 000000000..36343a787 --- /dev/null +++ b/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionContent.java @@ -0,0 +1,22 @@ +package run.mone.ai.gpt.bo.multiModal; + +import com.google.gson.annotations.SerializedName; +import lombok.Builder; +import lombok.Data; + +import java.util.Map; + +@Data +@Builder +public class GptVisionContent { + + @SerializedName("type") + private String type; + + @SerializedName("text") + private String text; + + @SerializedName("image_url") + private Map imageUrl; + +} diff --git a/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionMsg.java b/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionMsg.java new file mode 100644 index 000000000..6787b3146 --- /dev/null +++ b/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionMsg.java @@ -0,0 +1,23 @@ +package run.mone.ai.gpt.bo.multiModal; + +import com.google.gson.annotations.SerializedName; +import lombok.Builder; +import lombok.Data; + +import java.io.Serializable; +import java.util.List; + +/** + * @author goodjava@qq.com + * @date 2023/5/25 14:16 + */ +@Data +@Builder +public class GptVisionMsg implements Serializable { + + @SerializedName("role") + private String role; + + @SerializedName("content") + private List content; +} diff --git a/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionRequest.java b/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionRequest.java new file mode 100644 index 000000000..fab363962 --- /dev/null +++ b/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionRequest.java @@ -0,0 +1,46 @@ +package run.mone.ai.gpt.bo.multiModal; + +import com.google.gson.annotations.SerializedName; +import lombok.Builder; +import lombok.Data; + +import java.util.List; + +/** + * @author goodjava@qq.com + * @date 2024/4/9 16:36 + */ +@Data +@Builder +public class GptVisionRequest { + + @SerializedName("model") + private String model; + + @SerializedName("temperature") + private double temperature; + + @SerializedName("n") + @Builder.Default + private int n = 1; + + @SerializedName("stream") + private boolean stream; + + @SerializedName("top_p") + private double topP; + + @SerializedName("max_tokens") + private int maxTokens; + + @SerializedName("presence_penalty") + private double presencePenalty; + + @SerializedName("frequency_penalty") + private double frequencyPenalty; + + @SerializedName("messages") + private List messages; + + +} diff --git a/jcommon/ai/pom.xml b/jcommon/ai/pom.xml index 41a73b29f..748017515 100644 --- a/jcommon/ai/pom.xml +++ b/jcommon/ai/pom.xml @@ -18,6 +18,7 @@ aws minimax bytedance + gpt From 766ce45680947a69b85b6d62eba9cd74545b5a86 Mon Sep 17 00:00:00 2001 From: zhangzhiyong Date: Thu, 29 Aug 2024 13:45:09 +0800 Subject: [PATCH 09/11] chore: update test and main classes without code changes --- .../java/run/mone/neo4j/MoneCodeParser.java | 27 +- .../run/mone/neo4j/MoneMethodCodeParser.java | 299 ++++++++++++++++++ .../mone/neo4j/test/MoneCodeParserTest.java | 24 +- .../run/mone/neo4j/test/m/cat/CatService.java | 1 - 4 files changed, 310 insertions(+), 41 deletions(-) create mode 100644 jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneMethodCodeParser.java diff --git a/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java index df1647094..b9521ab5e 100644 --- a/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java +++ b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java @@ -8,10 +8,11 @@ import com.github.javaparser.ast.body.MethodDeclaration; import com.github.javaparser.ast.comments.Comment; import com.github.javaparser.ast.comments.JavadocComment; -import com.github.javaparser.ast.expr.*; +import com.github.javaparser.ast.expr.AnnotationExpr; +import com.github.javaparser.ast.expr.FieldAccessExpr; +import com.github.javaparser.ast.expr.MethodCallExpr; import com.github.javaparser.ast.nodeTypes.NodeWithName; import com.github.javaparser.ast.type.ClassOrInterfaceType; -import com.github.javaparser.resolution.declarations.ResolvedMethodDeclaration; import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -607,30 +608,8 @@ public void visit(MethodDeclaration n, Void arg) { // 处理注释 processComments(n); - // 处理方法调用 - processMethodCalls(n); - } - // 处理方法调用 - private void processMethodCalls(MethodDeclaration n) { - n.findAll(MethodCallExpr.class).forEach(methodCall -> { - Map callParams = new HashMap<>(); - callParams.put("callerName", getFullMethodName(n)); - callParams.put("calleeName", getFullMethodPath(methodCall)); - - // 创建目标方法节点(如果不存在) - session.run("MERGE (callee:Method {name: $calleeName})", callParams); - - // 创建 CALLS 关系 - session.run("MATCH (caller:Method {name: $callerName}) " + - "MATCH (callee:Method {name: $calleeName}) " + - "MERGE (caller)-[:CALLS]->(callee)", - callParams); - }); - } - - // 处理注释 private void processComments(MethodDeclaration n) { for (Comment comment : n.getAllContainedComments()) { diff --git a/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneMethodCodeParser.java b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneMethodCodeParser.java new file mode 100644 index 000000000..05833ffde --- /dev/null +++ b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneMethodCodeParser.java @@ -0,0 +1,299 @@ +package run.mone.neo4j; + +import com.github.javaparser.JavaParser; +import com.github.javaparser.ast.CompilationUnit; +import com.github.javaparser.ast.ImportDeclaration; +import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration; +import com.github.javaparser.ast.body.FieldDeclaration; +import com.github.javaparser.ast.body.MethodDeclaration; +import com.github.javaparser.ast.comments.Comment; +import com.github.javaparser.ast.comments.JavadocComment; +import com.github.javaparser.ast.expr.FieldAccessExpr; +import com.github.javaparser.ast.expr.MethodCallExpr; +import com.github.javaparser.ast.type.ClassOrInterfaceType; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.neo4j.driver.*; + +import java.io.File; +import java.util.*; + +/** + * @author goodjava@qq.com + *

+ * 建立方法调用之间的关系(call),且只建立业务调用的关系 + */ +@Slf4j +public class MoneMethodCodeParser { + + private String NEO4J_URI = "bolt://localhost:7687"; + + private String NEO4J_USER = "neo4j"; + + private String password = ""; + + public MoneMethodCodeParser setPassword(String password) { + this.password = password; + return this; + } + + + public void writeJavaFilesToNeo4j(String directoryPath) { + if (new File(directoryPath).isFile()) { + writeToNeo4j(directoryPath); + return; + } + + getJavaFilesInDirectory(directoryPath).forEach(it -> { + log.info("parse it:{}", it); + writeToNeo4j(it); + }); + } + + //给一个文件夹,获取里边是.java文件的列表,注意你需要递归获取(class) + public static List getJavaFilesInDirectory(String directoryPath) { + List javaFiles = new ArrayList<>(); + File directory = new File(directoryPath); + + if (directory.exists() && directory.isDirectory()) { + File[] files = directory.listFiles(); + if (files != null) { + for (File file : files) { + if (file.isDirectory()) { + javaFiles.addAll(getJavaFilesInDirectory(file.getAbsolutePath())); + } else if (file.getName().endsWith(".java")) { + javaFiles.add(file.getAbsolutePath()); + } + } + } + } + return javaFiles; + } + + + @SneakyThrows + private void writeToNeo4j(String filePath) { + //写入到neo4j中 + // 替换成你的 Java 文件路径 + String projectName = "MyProject"; + + try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, password)); + Session session = driver.session()) { + // 解析 Java 文件 + CompilationUnit cu = new JavaParser().parse(new File(filePath)).getResult().get(); + + // 遍历类、接口、方法等 + cu.accept(new MoneMethodCodeParser.Visitor(session, projectName, filePath), null); + } + } + + + private static class Visitor extends com.github.javaparser.ast.visitor.VoidVisitorAdapter { + + private final Session session; + + public Visitor(Session session, String projectName, String filePath) { + this.session = session; + } + + + @Override + public void visit(ClassOrInterfaceDeclaration n, Void arg) { + super.visit(n, arg); + } + + + private String getFullMethodName(MethodDeclaration method) { + String packageName = method.findCompilationUnit() + .flatMap(cu -> cu.getPackageDeclaration()) + .map(pd -> pd.getNameAsString()) + .orElse(""); + String className = method.findAncestor(com.github.javaparser.ast.body.ClassOrInterfaceDeclaration.class) + .map(c -> c.getNameAsString()) + .orElse(""); + String methodName = method.getNameAsString(); + return packageName + "." + className + "." + methodName; + } + + /** + * 获取方法调用的完整路径,包括包名、类名和方法名 + * + * @param methodCall 方法调用表达式 + * @return 方法调用的完整路径 + */ + public String getFullMethodPath(MethodCallExpr methodCall) { + StringBuilder fullPath = new StringBuilder(); + + // 获取包名 + Optional cu = methodCall.findCompilationUnit(); + if (cu.isPresent()) { + cu.get().getPackageDeclaration().ifPresent(pkg -> + fullPath.append(pkg.getNameAsString()).append(".") + ); + } + + // 获取类名 + String className = methodCall.findAncestor(ClassOrInterfaceDeclaration.class) + .map(ClassOrInterfaceDeclaration::getNameAsString) + .orElse(""); + + // 获取方法调用的对象 + String objectName = methodCall.getScope() + .map(scope -> scope.toString()) + .orElse(""); + + + //静态调用 + if (methodCall.getScope().isPresent() && methodCall.getScope().get() instanceof FieldAccessExpr) { + return objectName + "." + methodCall.getNameAsString(); + } + + //lombok 的log + if (isLogCall(methodCall)) { + return objectName + "." + methodCall.getNameAsString(); + } + + // 如果对象名不为空,尝试找到它的类型 + if (!objectName.isEmpty()) { + Optional field = methodCall.findAncestor(ClassOrInterfaceDeclaration.class) + .flatMap(classDecl -> classDecl.getFieldByName(objectName)); + + if (field.isPresent()) { + ClassOrInterfaceType type = field.get().getVariable(0).getType().asClassOrInterfaceType(); + String v = resolveTypePath(type); + return v + "." + methodCall.getNameAsString(); + } + } + + + // 构建完整路径 + fullPath.append(className).append("."); + fullPath.append(methodCall.getNameAsString()); + + return fullPath.toString(); + } + + public static String resolveTypePath(ClassOrInterfaceType type) { + String typeName = type.getNameAsString(); + + Optional cu = type.findAncestor(CompilationUnit.class); + if (cu.isPresent()) { + // 尝试从导入声明中查找匹配 + Optional importedPath = findMatchingImport(cu.get(), typeName); + if (importedPath.isPresent()) { + return importedPath.get(); + } + + // 如果没有找到匹配的导入,检查是否在同一包中 + Optional currentPackage = getCurrentPackage(cu.get()); + if (currentPackage.isPresent()) { + return currentPackage.get() + "." + typeName; + } + } + + // 如果无法解析,返回原始类型名称 + return typeName; + } + + private static Optional findMatchingImport(CompilationUnit cu, String typeName) { + return cu.getImports().stream() + .filter(importDecl -> !importDecl.isAsterisk() && importDecl.getNameAsString().endsWith("." + typeName)) + .map(ImportDeclaration::getNameAsString) + .findFirst(); + } + + private static Optional getCurrentPackage(CompilationUnit cu) { + return cu.getPackageDeclaration().map(pd -> pd.getNameAsString()); + } + + /** + * 判断方法调用是否为日志调用 + * + * @param n 方法调用表达式 + * @return 如果方法调用是日志调用则返回true,否则返回false + */ + private boolean isLogCall(MethodCallExpr n) { + if (!n.getScope().isPresent()) { + return false; + } + String scope = n.getScope().get().toString(); + String method = n.getNameAsString(); + return scope.equals("log") && + (method.equals("trace") || method.equals("debug") || method.equals("info") || + method.equals("warn") || method.equals("error")); + } + + + @Override + public void visit(MethodDeclaration n, Void arg) { + super.visit(n, arg); + if (n.findAncestor(ClassOrInterfaceDeclaration.class).isEmpty()) { + return; + } + // 处理方法调用 + processMethodCalls(n); + + } + + + // 处理方法调用 + private void processMethodCalls(MethodDeclaration n) { + n.findAll(MethodCallExpr.class).forEach(methodCall -> { + Map callParams = new HashMap<>(); + callParams.put("callerName", getFullMethodName(n)); + callParams.put("calleeName", getFullMethodPath(methodCall)); + + // 检查 callerName 和 calleeName 是否都存在 + Result result = session.run("MATCH (caller:Method {name: $callerName}), (callee:Method {name: $calleeName}) " + + "RETURN caller, callee", callParams); + + //只有两个业务method,才有必要创建这个边 + if (result.hasNext()) { + // 创建 CALLS 关系 + session.run("MATCH (caller:Method {name: $callerName}) " + + "MATCH (callee:Method {name: $calleeName}) " + + "MERGE (caller)-[:CALLS]->(callee)", + callParams); + } + }); + } + + + // 处理注释 + private void processComments(MethodDeclaration n) { + for (Comment comment : n.getAllContainedComments()) { + createCommentNode(comment, n); + } + + Optional optional = n.getJavadocComment(); + if (optional.isPresent()) { + createCommentNode(optional.get(), n); + } + + Optional commentOptional = n.getComment(); + if (commentOptional.isPresent()) { + createCommentNode(commentOptional.get(), n); + } + } + + private void createCommentNode(Comment comment, MethodDeclaration n) { + Map commentParams = new HashMap<>(); + commentParams.put("text", comment.getContent()); + commentParams.put("text_vector", new float[]{}); // 替换为实际的文本向量 + + session.run("MERGE (comment:Comment {text: $text, text_vector: $text_vector})", commentParams); + + // 创建 DOCUMENTS 关系 (Comment -[:DOCUMENTS]-> Method) + Map documentsParams = new HashMap<>(); + documentsParams.put("commentText", comment.getContent()); + documentsParams.put("methodName", getFullMethodName(n)); + documentsParams.put("methodSignature", n.getSignature().asString()); + session.run("MATCH (comment:Comment {text: $commentText}) " + + "MATCH (m:Method {name: $methodName, signature: $methodSignature}) " + + "MERGE (comment)-[:DOCUMENTS]->(m)", + documentsParams); + } + + } + +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java index 440a1be06..d2b6f63c2 100644 --- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java @@ -3,26 +3,12 @@ import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; import org.junit.Test; -import org.neo4j.driver.Record; -import org.neo4j.driver.Result; -import org.neo4j.driver.Session; -import org.neo4j.driver.internal.InternalRecord; -import org.neo4j.driver.internal.InternalResult; -import org.neo4j.driver.internal.InternalSession; import run.mone.neo4j.BotCall; import run.mone.neo4j.MoneCodeParser; -import run.mone.neo4j.test.MoneCodeParserTest; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; +import run.mone.neo4j.MoneMethodCodeParser; + import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -47,6 +33,12 @@ public void testWriteCatServiceToNeo4j() { // new MoneCodeParser().setPassword(System.getenv("password")).writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java"); } + + @Test + public void testParserMethod() { + new MoneMethodCodeParser().setPassword(System.getenv("password")).writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m"); + } + @Test public void test1() { new MoneCodeParser().queryEntityClasses(); diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/cat/CatService.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/cat/CatService.java index 494c6776c..492be9300 100644 --- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/cat/CatService.java +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/cat/CatService.java @@ -16,7 +16,6 @@ public class CatService { private HashMap data = new HashMap<>(); - //获取小猫的数量 //获取小猫的数量 public int getCatCount() { log.info("abc"); From bd03aa7128dd71d6309f7918ee53027974179bf6 Mon Sep 17 00:00:00 2001 From: wtt <30461027+wtt40122@users.noreply.github.com> Date: Thu, 29 Aug 2024 20:03:18 +0800 Subject: [PATCH 10/11] refactor: update file coll (#889) * refactor: update es and upgrade file * fix: Fixed the issue of reading a large number of files OOM and upgraded the ES version * refactor: update compile jdk version * refactor: update file coll --- .../main/java/com/xiaomi/mone/file/LogFile2.java | 1 + .../java/com/xiaomi/mone/file/ReadListener.java | 3 +++ .../mone/file/listener/OzHeraReadListener.java | 8 ++++++++ .../com/xiaomi/mone/file/ozhera/HeraFile.java | 3 +++ .../xiaomi/mone/file/ozhera/HeraFileMonitor.java | 12 +++++++++--- .../java/com/xiaomi/mone/file/LogFileTest.java | 16 ++++++++++++++-- 6 files changed, 38 insertions(+), 5 deletions(-) diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile2.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile2.java index fbbd399c4..0381a698a 100644 --- a/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile2.java +++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile2.java @@ -190,6 +190,7 @@ public void readLine() throws IOException { readResult.setFilePathName(file); readResult.setLineNumber(++lineNumber); ReadEvent event = new ReadEvent(readResult); + listener.setReadTime(); listener.onEvent(event); } diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/ReadListener.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/ReadListener.java index 0329b66bf..75e92b157 100644 --- a/jcommon/file/src/main/java/com/xiaomi/mone/file/ReadListener.java +++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/ReadListener.java @@ -37,4 +37,7 @@ default void setPointer(Object obj) { default void saveProgress() { } + default void setReadTime() { + } + } diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/listener/OzHeraReadListener.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/listener/OzHeraReadListener.java index e2b3c2d42..43fa64876 100644 --- a/jcommon/file/src/main/java/com/xiaomi/mone/file/listener/OzHeraReadListener.java +++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/listener/OzHeraReadListener.java @@ -72,4 +72,12 @@ public void setPointer(Object obj) { } } } + + @Override + public void setReadTime() { + HeraFile f = monitor.getFileMap().get(logFile.getFileKey()); + if (null != f) { + f.getReadTime().set(System.currentTimeMillis()); + } + } } diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFile.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFile.java index 8acb1a989..fbbde66fe 100644 --- a/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFile.java +++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFile.java @@ -30,4 +30,7 @@ public class HeraFile { @Builder.Default private AtomicLong utime = new AtomicLong(System.currentTimeMillis()); + + @Builder.Default + private AtomicLong readTime = new AtomicLong(System.currentTimeMillis()); } diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFileMonitor.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFileMonitor.java index c8a2d1e84..0f7c7d385 100644 --- a/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFileMonitor.java +++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFileMonitor.java @@ -34,6 +34,7 @@ public class HeraFileMonitor { @Getter private ConcurrentHashMap map = new ConcurrentHashMap<>(); + @Getter private ConcurrentHashMap fileMap = new ConcurrentHashMap<>(); @Setter @@ -52,7 +53,7 @@ public HeraFileMonitor(long removeTime) { List> remList = Lists.newArrayList(); long now = System.currentTimeMillis(); fileMap.values().forEach(it -> { - if (now - it.getUtime().get() >= removeTime) { + if (now - it.getUtime().get() >= removeTime && now - it.getReadTime().get() >= removeTime) { remList.add(Pair.of(it.getFileName(), it.getFileKey())); } }); @@ -129,7 +130,7 @@ public void reg(String path, Predicate predicate) throws IOException, In map.putIfAbsent(k, hf); fileMap.put(filePath, hf); - listener.onEvent(FileEvent.builder().type(EventType.create).fileName(file.getPath()).build()); + listener.onEvent(FileEvent.builder().type(EventType.create).fileKey(k).fileName(file.getPath()).build()); } } } @@ -167,7 +168,12 @@ private HeraFile initFile(File it) { log.info("initFile fileName:{},fileKey:{}", name, fileKey); map.put(hf.getFileKey(), hf); fileMap.put(hf.getFileName(), hf); - this.listener.onEvent(FileEvent.builder().pointer(pointer).type(EventType.init).fileName(hf.getFileName()).build()); + this.listener.onEvent(FileEvent.builder() + .pointer(pointer) + .type(EventType.init) + .fileName(hf.getFileName()) + .fileKey(hf.getFileKey()) + .build()); return hf; } catch (Exception e) { log.error("init file error,fileName:{}", name, e); diff --git a/jcommon/file/src/test/java/com/xiaomi/mone/file/LogFileTest.java b/jcommon/file/src/test/java/com/xiaomi/mone/file/LogFileTest.java index d7b987194..e57859f58 100644 --- a/jcommon/file/src/test/java/com/xiaomi/mone/file/LogFileTest.java +++ b/jcommon/file/src/test/java/com/xiaomi/mone/file/LogFileTest.java @@ -24,6 +24,8 @@ import org.junit.Test; import java.io.IOException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; @@ -74,9 +76,14 @@ public void testLogFileMonitor() { monitor.setListener(new DefaultMonitorListener(monitor, readEvent -> { System.out.println(readEvent.getReadResult().getLines()); })); - String fileName = "/home/work/log/file.log.*"; + String fileName = "/home/work/log/test/file*.txt"; Pattern pattern = Pattern.compile(fileName); - monitor.reg("/home/work/log", it -> { + + ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); + + scheduler.scheduleAtFixedRate(this::test, 1, 2, TimeUnit.SECONDS); + + monitor.reg("/home/work/log/test", it -> { boolean matches = pattern.matcher(it).matches(); log.info("file:{},matches:{}", it, true); return true; @@ -85,6 +92,11 @@ public void testLogFileMonitor() { System.in.read(); } + private void test() { + log.info("test save progress"); + FileInfoCache.ins().shutdown(); + } + @Test public void testLogWS() throws IOException { LogFile log = new LogFile("D:\\test.log", new ReadListener() { From 82c2441546aba2cff304ccbc029bd28af3fb5e74 Mon Sep 17 00:00:00 2001 From: wtt <30461027+wtt40122@users.noreply.github.com> Date: Tue, 3 Sep 2024 14:22:13 +0800 Subject: [PATCH 11/11] feat: mybatis plus plugin update (#890) * refactor: update es and upgrade file * fix: Fixed the issue of reading a large number of files OOM and upgraded the ES version * refactor: update compile jdk version * refactor: update file coll * refactor: mybatis plus plugin supported page --- .../docean/plugin/configuration/ConfigurationPlugin.java | 2 +- .../youpin/docean/plugin/mybatisplus/MybatisPlugin.java | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/jcommon/docean-plugin/docean-plugin-configuration/src/main/java/com/xiaomi/youpin/docean/plugin/configuration/ConfigurationPlugin.java b/jcommon/docean-plugin/docean-plugin-configuration/src/main/java/com/xiaomi/youpin/docean/plugin/configuration/ConfigurationPlugin.java index e78431c34..37d10ecad 100644 --- a/jcommon/docean-plugin/docean-plugin-configuration/src/main/java/com/xiaomi/youpin/docean/plugin/configuration/ConfigurationPlugin.java +++ b/jcommon/docean-plugin/docean-plugin-configuration/src/main/java/com/xiaomi/youpin/docean/plugin/configuration/ConfigurationPlugin.java @@ -38,7 +38,7 @@ * @author shanwb * @date 2022-09-13 */ -@DOceanPlugin(order = Integer.MAX_VALUE) +@DOceanPlugin(order = Integer.MAX_VALUE - 2) @Slf4j public class ConfigurationPlugin implements IPlugin { diff --git a/jcommon/docean-plugin/docean-plugin-mybatis-plus/src/main/java/com/xiaomi/youpin/docean/plugin/mybatisplus/MybatisPlugin.java b/jcommon/docean-plugin/docean-plugin-mybatis-plus/src/main/java/com/xiaomi/youpin/docean/plugin/mybatisplus/MybatisPlugin.java index 76035713b..37278b1d2 100644 --- a/jcommon/docean-plugin/docean-plugin-mybatis-plus/src/main/java/com/xiaomi/youpin/docean/plugin/mybatisplus/MybatisPlugin.java +++ b/jcommon/docean-plugin/docean-plugin-mybatis-plus/src/main/java/com/xiaomi/youpin/docean/plugin/mybatisplus/MybatisPlugin.java @@ -16,6 +16,7 @@ package com.xiaomi.youpin.docean.plugin.mybatisplus; +import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor; import com.xiaomi.youpin.docean.Aop; import com.xiaomi.youpin.docean.Ioc; import com.xiaomi.youpin.docean.anno.DOceanPlugin; @@ -33,6 +34,7 @@ import org.apache.ibatis.plugin.Interceptor; import org.apache.ibatis.session.SqlSession; import org.apache.ibatis.session.SqlSessionFactory; +import org.springframework.util.CollectionUtils; import javax.sql.DataSource; import java.lang.reflect.InvocationHandler; @@ -47,7 +49,7 @@ /** * @author goodjava@qq.com */ -@DOceanPlugin +@DOceanPlugin(order = Integer.MAX_VALUE - 1) @Slf4j public class MybatisPlugin implements IPlugin { @@ -93,6 +95,11 @@ private void addDAO(Ioc ioc, String beanName, String mapperLocation, boolean one bean.setPlugins(new Interceptor[]{interceptor}); } + Set plusInterceptors = ioc.getBeans(MybatisPlusInterceptor.class); + if (!CollectionUtils.isEmpty(plusInterceptors)) { + bean.setPlugins(plusInterceptors.toArray(new Interceptor[]{})); + } + SqlSessionFactory factory = bean.buildSqlSessionFactory(); ioc.putBean("mybatis_" + beanName + config.getName(), factory);