From 2d55920ca9d9ced5febbc36ca334e612de851129 Mon Sep 17 00:00:00 2001 From: EricDing <128116675+sadadw1@users.noreply.github.com> Date: Thu, 15 Aug 2024 14:09:16 +0800 Subject: [PATCH 01/14] feat: add mebatis generator (#873) * feat: add prometheus-trace-etl release * feat: add gitlab log release jdk21 * feat: update athena * feat: update check code file path * feat: add code generator and mybatis generator * feat: add mybtais generator --- .../youpin/codegen/M78DoceanMongoGen.java | 95 +++++++++++++++++++ .../youpin/codegen/common/FileUtils.java | 4 + .../run/mone/ai/codegen/FeatureGenerator.java | 24 +++++ .../run/mone/ai/codegen/MybatisGenerator.java | 31 ++++-- .../mone/ai/codegen/bo/FeatureGenerateBo.java | 6 ++ .../ai/codegen/bo/MybatisGeneratorResult.java | 19 ++++ .../holder/MybatisGeneratorHolder.java | 10 ++ .../ai/codegen/plugin/ClassNamePlugin.java | 44 +++++++++ .../codegen/test/FeatureGeneratorTest.java | 6 +- 9 files changed, 231 insertions(+), 8 deletions(-) create mode 100644 jcommon/codegen/src/main/java/com/xiaomi/youpin/codegen/M78DoceanMongoGen.java create mode 100644 jcommon/codegen/src/main/java/run/mone/ai/codegen/bo/MybatisGeneratorResult.java create mode 100644 jcommon/codegen/src/main/java/run/mone/ai/codegen/holder/MybatisGeneratorHolder.java create mode 100644 jcommon/codegen/src/main/java/run/mone/ai/codegen/plugin/ClassNamePlugin.java diff --git a/jcommon/codegen/src/main/java/com/xiaomi/youpin/codegen/M78DoceanMongoGen.java b/jcommon/codegen/src/main/java/com/xiaomi/youpin/codegen/M78DoceanMongoGen.java new file mode 100644 index 000000000..11b9efc27 --- /dev/null +++ b/jcommon/codegen/src/main/java/com/xiaomi/youpin/codegen/M78DoceanMongoGen.java @@ -0,0 +1,95 @@ +package com.xiaomi.youpin.codegen; + +import com.xiaomi.youpin.codegen.bo.Dependency; +import com.xiaomi.youpin.codegen.common.FileUtils; +import com.xiaomi.youpin.codegen.generator.FileGenerator; +import com.xiaomi.youpin.infra.rpc.Result; +import com.xiaomi.youpin.infra.rpc.errors.GeneralCodes; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +@Slf4j +public class M78DoceanMongoGen { + // private static final String DEFAULT_MODEL = "mi-function"; + private final String tmpPath = "m78-docean-mongo"; + // private static final String handlerDir = "handler"; +// private static final String srcPath = "/src/main/java/"; +// private static final String testPath = "/src/test/java/"; + + public Result generateAndZip(String projectPath, String projectName, String groupId, String packageName, String author, String versionId, HashMap> dep) { + String packagePath = packageName.replaceAll("\\.", "/"); + String camelName = this.adapterProjectNameToCamelName(projectName); + Map m = new HashMap<>(); + m.put("group", groupId); + m.put("artifactId", projectName); + m.put("version", versionId); + m.put("author", author); + m.put("camelName", camelName); + m.put("package", packageName); + try { +// FileUtils.createDirectories(FileUtils.join(projectPath, projectName, projectName + "-common", "src", "test", "java")); + FileUtils.createDirectories(FileUtils.join(projectPath, projectName, projectName + "-server", "src", "main", "java", packagePath, "bootstrap")); + FileUtils.createDirectories(FileUtils.join(projectPath, projectName, projectName + "-server", "src", "test", "java")); + FileUtils.createDirectories(FileUtils.join(projectPath, projectName, projectName + "-service", "src", "test", "java")); +// FileUtils.createDirectories(FileUtils.join(projectPath, projectName, projectName + "-service", "src", "main", "resources")); + FileUtils.createDirectories(FileUtils.join(projectPath, projectName, projectName + "-api", "src", "main", "java", packagePath, "api", "bo")); + FileUtils.createDirectories(FileUtils.join(projectPath, projectName, projectName + "-api", "src", "main", "java", packagePath, "api", "vo")); + FileUtils.createDirectories(FileUtils.join(projectPath, projectName, projectName + "-server", "src", "main", "java", packagePath, "controller")); + FileUtils.createDirectories(FileUtils.join(projectPath, projectName, projectName + "-common", "src", "main", "java", packagePath, "common")); + FileUtils.createDirectories(FileUtils.join(projectPath, projectName, projectName + "-server", "src", "main", "resources", "config")); + FileUtils.createDirectories(FileUtils.join(projectPath, projectName, projectName + "-service", "src", "main", "java", packagePath, "service")); + new FileGenerator(projectPath, projectName, FileUtils.join(projectName + "-service", "src", "main", "java", packagePath, "service", "MongoService.java"), FileUtils.join(tmpPath, "mongoservice.tml")).generator(m); + new FileGenerator(projectPath, projectName, FileUtils.join(projectName + "-api", "src", "main", "java", packagePath, "api", "bo", "MongoBo.java"), FileUtils.join(tmpPath, "mongobo.tml")).generator(m); + new FileGenerator(projectPath, projectName, FileUtils.join(projectName + "-api", "src", "main", "java", packagePath, "api", "bo", "Page.java"), FileUtils.join(tmpPath, "page.tml")).generator(m); + new FileGenerator(projectPath, projectName, FileUtils.join(projectName + "-service", "pom.xml"), FileUtils.join(tmpPath, "service-pom.tml")).generator(m); + new FileGenerator(projectPath, projectName, FileUtils.join(projectName + "-service", "src", "main", "java", packagePath, "service", "ServiceImpl1.java"), FileUtils.join(tmpPath, "serviceimpl1.tml")).generator(m); + new FileGenerator(projectPath, projectName, FileUtils.join(projectName + "-server", "src", "main", "resources", "config", "staging.properties"), FileUtils.join(tmpPath, "staging.tml")).generator(m); + new FileGenerator(projectPath, projectName, FileUtils.join(projectName + "-server", "src", "main", "resources", "config.properties"), FileUtils.join(tmpPath, "config.tml")).generator(m); + new FileGenerator(projectPath, projectName, FileUtils.join(projectName + "-server", "src", "main", "resources", "logback.xml"), FileUtils.join(tmpPath, "logback.tml")).generator(m); + new FileGenerator(projectPath, projectName, FileUtils.join("pom.xml"), FileUtils.join(tmpPath, "pom.tml")).generator(m); + new FileGenerator(projectPath, projectName, FileUtils.join(projectName + "-common", "src", "main", "java", packagePath, "common", "Result.java"), FileUtils.join(tmpPath, "result.tml")).generator(m); + new FileGenerator(projectPath, projectName, FileUtils.join(projectName + "-server", "src", "main", "java", packagePath, "bootstrap", "Bootstrap.java"), FileUtils.join(tmpPath, "bootstrap.tml")).generator(m); + new FileGenerator(projectPath, projectName, FileUtils.join(projectName + "-server", "src", "main", "java", packagePath, "controller", "MongodbController.java"), FileUtils.join(tmpPath, "mongodbcontroller.tml")).generator(m); + new FileGenerator(projectPath, projectName, FileUtils.join(projectName + "-server", "pom.xml"), FileUtils.join(tmpPath, "server-pom.tml")).generator(m); + new FileGenerator(projectPath, projectName, FileUtils.join(projectName + "-server", "src", "main", "resources", "config", "dev.properties"), FileUtils.join(tmpPath, "dev.tml")).generator(m); + new FileGenerator(projectPath, projectName, FileUtils.join(projectName + "-server", "src", "main", "resources", "config", "online.properties"), FileUtils.join(tmpPath, "online.tml")).generator(m); + new FileGenerator(projectPath, projectName, FileUtils.join(projectName + "-api", "pom.xml"), FileUtils.join(tmpPath, "api-pom.tml")).generator(m); + new FileGenerator(projectPath, projectName, FileUtils.join(projectName + "-api", "src", "main", "java", packagePath, "api", "IService.java"), FileUtils.join(tmpPath, "iservice.tml")).generator(m); + new FileGenerator(projectPath, projectName, FileUtils.join(projectName + "-common", "pom.xml"), FileUtils.join(tmpPath, "common-pom.tml")).generator(m); + + new FileGenerator(projectPath, projectName, FileUtils.join(".gitignore"), "springboot_gitignore.tml").generator(m); + FileUtils.compress(projectPath + File.separator + projectName, projectPath + File.separator + projectName + ".zip"); + } catch (Exception e) { + log.error("SpringBootProGen failed ", e); + return Result.fail(GeneralCodes.InternalError, "InternalError"); + } + + return Result.success(projectPath + File.separator + projectName + ".zip"); + } + + private String adapterProjectNameToCamelName(String name) { + if (StringUtils.isEmpty(name)) { + return ""; + } + try { + String[] strings = name.split("-"); + if (strings.length > 1) { + String res = ""; + for (int i = 0; i < strings.length; i++) { + if (!StringUtils.isEmpty(strings[i])) { + res = res + StringUtils.capitalize(strings[i]); + } + } + return res; + } + } catch (Exception e) { + return StringUtils.capitalize(name); + } + return StringUtils.capitalize(name); + } +} diff --git a/jcommon/codegen/src/main/java/com/xiaomi/youpin/codegen/common/FileUtils.java b/jcommon/codegen/src/main/java/com/xiaomi/youpin/codegen/common/FileUtils.java index 14f96af33..97f2cd8e1 100644 --- a/jcommon/codegen/src/main/java/com/xiaomi/youpin/codegen/common/FileUtils.java +++ b/jcommon/codegen/src/main/java/com/xiaomi/youpin/codegen/common/FileUtils.java @@ -39,6 +39,10 @@ */ public class FileUtils { + public static String join(String... ps) { + return String.join(File.separator, ps); + } + public static void createDirectories(String path) { try { Files.createDirectories(Paths.get(path)); diff --git a/jcommon/codegen/src/main/java/run/mone/ai/codegen/FeatureGenerator.java b/jcommon/codegen/src/main/java/run/mone/ai/codegen/FeatureGenerator.java index e9ae8222d..2b1468021 100644 --- a/jcommon/codegen/src/main/java/run/mone/ai/codegen/FeatureGenerator.java +++ b/jcommon/codegen/src/main/java/run/mone/ai/codegen/FeatureGenerator.java @@ -1,5 +1,6 @@ package run.mone.ai.codegen; +import com.google.gson.Gson; import com.mybatisflex.codegen.Generator; import com.mybatisflex.codegen.config.GlobalConfig; import com.zaxxer.hikari.HikariDataSource; @@ -10,6 +11,7 @@ import run.mone.ai.codegen.bo.FeatureGenerateBo; import run.mone.ai.codegen.util.TemplateUtils; +import java.util.Base64; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -21,6 +23,17 @@ @Slf4j public class FeatureGenerator { + private static final Gson gson = new Gson(); + + public static void main(String[] args) { + FeatureGenerateBo featureGenerateBo = null; + //方便ai调用的时候,设置表名 + if (args.length > 0) { + featureGenerateBo = parseArgsAndExtractData(args); + generateWithTemplate(featureGenerateBo); + } + } + public static void generateWithTemplate(FeatureGenerateBo featureGenerateBo) { // 类型检查 @@ -147,4 +160,15 @@ public static String replaceDotWithSlash(String input) { } return input.replace(".", "/"); } + + private static FeatureGenerateBo parseArgsAndExtractData(String[] args) { + String jsonStr = args[0]; + + jsonStr = new String(Base64.getDecoder().decode(jsonStr)); + log.info("jsonStr:{}", jsonStr); + + FeatureGenerateBo bo = gson.fromJson(jsonStr, FeatureGenerateBo.class); + log.info("map:{}", bo); + return bo; + } } diff --git a/jcommon/codegen/src/main/java/run/mone/ai/codegen/MybatisGenerator.java b/jcommon/codegen/src/main/java/run/mone/ai/codegen/MybatisGenerator.java index fdc0b99a5..a3d7cc954 100644 --- a/jcommon/codegen/src/main/java/run/mone/ai/codegen/MybatisGenerator.java +++ b/jcommon/codegen/src/main/java/run/mone/ai/codegen/MybatisGenerator.java @@ -15,6 +15,8 @@ import org.mybatis.generator.config.TableConfiguration; import org.mybatis.generator.internal.DefaultShellCallback; import run.mone.ai.codegen.bo.FeatureGenerateBo; +import run.mone.ai.codegen.bo.MybatisGeneratorResult; +import run.mone.ai.codegen.holder.MybatisGeneratorHolder; import java.util.ArrayList; import java.util.Base64; @@ -31,19 +33,19 @@ public static void main(String[] args) { //方便ai调用的时候,设置表名 if (args.length > 0) { featureGenerateBo = parseArgsAndExtractData(args); + generateMyBatisFiles(featureGenerateBo); } - generateMyBatisFiles(featureGenerateBo); } // 使用mybatis-generator生成指定表对应的XML、Mapper和Entity类 - public static void generateMyBatisFiles(FeatureGenerateBo featureGenerateBo) { + public static MybatisGeneratorResult generateMyBatisFiles(FeatureGenerateBo featureGenerateBo) { + MybatisGeneratorResult result = new MybatisGeneratorResult(); if (Objects.isNull(featureGenerateBo.getTableName())) { log.warn("Table name is empty, cannot generate MyBatis files!"); - return; + return result; } // 配置MyBatis Generator List warnings = new ArrayList<>(); - boolean overwrite = true; Configuration config = new Configuration(); Context context = new Context(ModelType.FLAT); @@ -60,7 +62,7 @@ public static void generateMyBatisFiles(FeatureGenerateBo featureGenerateBo) { // 配置Java模型生成器 JavaModelGeneratorConfiguration javaModelGeneratorConfiguration = new JavaModelGeneratorConfiguration(); - javaModelGeneratorConfiguration.setTargetPackage(featureGenerateBo.getMybatisDaoPath()); + javaModelGeneratorConfiguration.setTargetPackage(featureGenerateBo.getMybatisEntityPath()); javaModelGeneratorConfiguration.setTargetProject(featureGenerateBo.getMybatisDaoModule() + "/src/main/java"); context.setJavaModelGeneratorConfiguration(javaModelGeneratorConfiguration); @@ -100,13 +102,29 @@ public static void generateMyBatisFiles(FeatureGenerateBo featureGenerateBo) { removeGetSetMethodsPlugin.setConfigurationType("run.mone.ai.codegen.plugin.RemoveGetSetMethodsPlugin"); context.addPluginConfiguration(removeGetSetMethodsPlugin); + if(!featureGenerateBo.isOverwrite()) { + // 使得xml文件不会被覆盖 + PluginConfiguration unmergeXML = new PluginConfiguration(); + unmergeXML.setConfigurationType("org.mybatis.generator.plugins.UnmergeableXmlMappersPlugin"); + context.addPluginConfiguration(unmergeXML); + } + + // 存储这次生成的类名，以便返回 + PluginConfiguration classNamePlugin = new PluginConfiguration(); + classNamePlugin.setConfigurationType("run.mone.ai.codegen.plugin.ClassNamePlugin"); + context.addPluginConfiguration(classNamePlugin); + config.addContext(context); - DefaultShellCallback callback = new DefaultShellCallback(overwrite); + DefaultShellCallback callback = new DefaultShellCallback(featureGenerateBo.isOverwrite()); MyBatisGenerator myBatisGenerator; try { myBatisGenerator = new MyBatisGenerator(config, callback, warnings); myBatisGenerator.generate(null); + + // 获取返回结果 + result.setEntity(MybatisGeneratorHolder.entity); + result.setMapper(MybatisGeneratorHolder.mapper); } catch (Exception e) { log.error("Error generating MyBatis files", e); } @@ -114,6 +132,7 @@ public static void generateMyBatisFiles(FeatureGenerateBo featureGenerateBo) { if (!warnings.isEmpty()) { warnings.forEach(log::warn); } + return result; } private static FeatureGenerateBo parseArgsAndExtractData(String[] args) { diff --git a/jcommon/codegen/src/main/java/run/mone/ai/codegen/bo/FeatureGenerateBo.java b/jcommon/codegen/src/main/java/run/mone/ai/codegen/bo/FeatureGenerateBo.java index e8c0fd450..248c6c3fb 100644 --- a/jcommon/codegen/src/main/java/run/mone/ai/codegen/bo/FeatureGenerateBo.java +++ b/jcommon/codegen/src/main/java/run/mone/ai/codegen/bo/FeatureGenerateBo.java @@ -87,4 +87,10 @@ public class FeatureGenerateBo { @Builder.Default private String mybatisEntityPath = ""; + /** + * mybatis 是否覆盖现有类，默认不覆盖，如果存在现有类则跳过 + */ + @Builder.Default + private boolean overwrite = false; + } diff --git a/jcommon/codegen/src/main/java/run/mone/ai/codegen/bo/MybatisGeneratorResult.java b/jcommon/codegen/src/main/java/run/mone/ai/codegen/bo/MybatisGeneratorResult.java new file mode 100644 index 000000000..23fc05999 --- /dev/null +++ b/jcommon/codegen/src/main/java/run/mone/ai/codegen/bo/MybatisGeneratorResult.java @@ -0,0 +1,19 @@ +package run.mone.ai.codegen.bo; + +import lombok.Data; +import lombok.ToString; + + +@Data +@ToString +public class MybatisGeneratorResult { + /** + * 生成的Java Entity类列表 + */ + private String entity; + + /** + * 生成的Java mapper类列表 + */ + private String mapper; +} diff --git a/jcommon/codegen/src/main/java/run/mone/ai/codegen/holder/MybatisGeneratorHolder.java b/jcommon/codegen/src/main/java/run/mone/ai/codegen/holder/MybatisGeneratorHolder.java new file mode 100644 index 000000000..87a4cf333 --- /dev/null +++ b/jcommon/codegen/src/main/java/run/mone/ai/codegen/holder/MybatisGeneratorHolder.java @@ -0,0 +1,10 @@ +package run.mone.ai.codegen.holder; + +import lombok.Data; + +public class MybatisGeneratorHolder { + + public static String entity; + public static String mapper; + +} diff --git a/jcommon/codegen/src/main/java/run/mone/ai/codegen/plugin/ClassNamePlugin.java b/jcommon/codegen/src/main/java/run/mone/ai/codegen/plugin/ClassNamePlugin.java new file mode 100644 index 000000000..2fe66b199 --- /dev/null +++ b/jcommon/codegen/src/main/java/run/mone/ai/codegen/plugin/ClassNamePlugin.java @@ -0,0 +1,44 @@ +package run.mone.ai.codegen.plugin; + +import org.mybatis.generator.api.IntrospectedTable; +import org.mybatis.generator.api.PluginAdapter; +import org.mybatis.generator.api.dom.java.TopLevelClass; +import org.mybatis.generator.api.dom.java.Interface; +import org.mybatis.generator.config.Context; +import run.mone.ai.codegen.holder.MybatisGeneratorHolder; +import org.mybatis.generator.api.JavaFormatter; + +import java.util.List; + +public class ClassNamePlugin extends PluginAdapter { + + private JavaFormatter javaFormatter; + + @Override + public void setContext(Context context) { + super.setContext(context); + this.javaFormatter = context.getJavaFormatter(); + } + + @Override + public boolean validate(List warnings) { + // 这里可以添加一些验证逻辑 + return true; + } + + @Override + public boolean modelBaseRecordClassGenerated(TopLevelClass topLevelClass, IntrospectedTable introspectedTable) { + String formattedContent = javaFormatter.getFormattedContent(topLevelClass); + MybatisGeneratorHolder.entity = formattedContent; + return true; + } + + + @Override + public boolean clientGenerated(Interface interfaze, IntrospectedTable introspectedTable) { + String formattedContent = javaFormatter.getFormattedContent(interfaze); + MybatisGeneratorHolder.mapper = formattedContent; + return true; + } +} + diff --git a/jcommon/codegen/src/test/java/com/xiaomi/youpin/codegen/test/FeatureGeneratorTest.java b/jcommon/codegen/src/test/java/com/xiaomi/youpin/codegen/test/FeatureGeneratorTest.java index 6acd2ef75..ff39ead0e 100644 --- a/jcommon/codegen/src/test/java/com/xiaomi/youpin/codegen/test/FeatureGeneratorTest.java +++ b/jcommon/codegen/src/test/java/com/xiaomi/youpin/codegen/test/FeatureGeneratorTest.java @@ -5,6 +5,7 @@ import run.mone.ai.codegen.MybatisGenerator; import run.mone.ai.codegen.bo.FeatureGeneratType; import run.mone.ai.codegen.bo.FeatureGenerateBo; +import run.mone.ai.codegen.bo.MybatisGeneratorResult; public class FeatureGeneratorTest { @@ -38,12 +39,13 @@ public void testGenerateWithMybatis() { featureGenerateBo.setPassword(dbPwd); featureGenerateBo.setTableName("user"); - featureGenerateBo.setMybatisDaoModule("/your/path/mone/jcommon/codegen"); + featureGenerateBo.setMybatisDaoModule("/your path/mone/jcommon/codegen"); featureGenerateBo.setMybatisXMLPath("com.xiaomi.dao.mapper"); featureGenerateBo.setMybatisDaoPath("com.xiaomi.dao.mapper"); featureGenerateBo.setMybatisEntityPath("com.xiaomi.dao.entity"); - MybatisGenerator.generateMyBatisFiles(featureGenerateBo); + MybatisGeneratorResult mybatisGeneratorResult = MybatisGenerator.generateMyBatisFiles(featureGenerateBo); + System.out.println(mybatisGeneratorResult); } } From 2bbdd85a0a32419179bf398f1958f851044aaf24 Mon Sep 17 00:00:00 2001 From: wtt <30461027+wtt40122@users.noreply.github.com> Date: Thu, 15 Aug 2024 16:22:11 +0800 Subject: [PATCH 02/14] fix: Fixed the issue of reading a large number of files OOM and upgraded the ES version (#871) * refactor: update es and upgrade file * fix: Fixed the issue of reading a large number of files OOM and upgraded the ES version * refactor: update compile jdk version --- .../run/mone/ai/google/test/ClientTest.java | 2 +- jcommon/es/pom.xml | 162 ++++++++---------- .../java/com/xiaomi/mone/es/EsClient.java | 13 +- .../java/com/xiaomi/mone/es/EsProcessor.java | 2 +- .../com/xiaomi/mone/es/test/EsClientTest.java | 2 +- .../mone/es/test/EsProcessorClientTest.java | 146 ++++++++-------- .../java/com/xiaomi/mone/file/ILogFile.java | 2 + .../java/com/xiaomi/mone/file/LogFile.java | 2 +- .../java/com/xiaomi/mone/file/LogFile2.java | 2 - .../java/com/xiaomi/mone/file/LogFileWS.java | 2 - .../main/java/com/xiaomi/mone/file/MLog.java | 4 +- .../mone/file/common/FileInfoCache.java | 9 +- .../xiaomi/mone/file/event/EventListener.java | 3 + .../file/listener/DefaultMonitorListener.java | 22 ++- .../mone/file/ozhera/HeraFileMonitor.java | 6 + .../com/xiaomi/mone/file/LogFileTest.java | 2 +- 16 files changed, 197 insertions(+), 184 deletions(-) diff --git a/jcommon/ai/google/src/test/java/run/mone/ai/google/test/ClientTest.java b/jcommon/ai/google/src/test/java/run/mone/ai/google/test/ClientTest.java index 575d843bf..8d8f74d68 100644 --- a/jcommon/ai/google/src/test/java/run/mone/ai/google/test/ClientTest.java +++ b/jcommon/ai/google/src/test/java/run/mone/ai/google/test/ClientTest.java @@ -22,7 +22,7 @@ public void test1() { RequestPayload payload = RequestPayload.builder().maxTokens(4000).anthropicVersion("vertex-2023-10-16").messages(Lists.newArrayList(Message.builder().role("user") .content(content) .build())).build(); - ResponsePayload r = c.call(c.token(), payload); + ResponsePayload r = c.call(c.token(""), payload); System.out.println(r.getContent().get(0).getText()); } } diff --git a/jcommon/es/pom.xml b/jcommon/es/pom.xml index 8312ca347..b5a376445 100644 --- a/jcommon/es/pom.xml +++ b/jcommon/es/pom.xml @@ -1,91 +1,75 @@ - - 4.0.0 - - run.mone - jcommon - 1.6.0-jdk21-SNAPSHOT - - es - 1.5-jdk21-SNAPSHOT - - - org.elasticsearch.client - elasticsearch-rest-high-level-client - 7.10.0 - - - httpclient - org.apache.httpcomponents - - - httpcore - org.apache.httpcomponents - - - httpcore-nio - org.apache.httpcomponents - - - - - org.elasticsearch.client - elasticsearch-rest-client-sniffer - 7.10.0 - - - org.apache.httpcomponents - httpclient - 4.5.12 - - - org.apache.httpcomponents - httpcore-nio - 4.4.13 - - - org.apache.httpcomponents - httpcore - 4.4.13 - - - run.mone - nacos - 1.4-v1-jdk20-SNAPSHOT - test - - - - - - maven-compiler-plugin - 3.11.0 - - 21 - 21 - true - UTF-8 - - ${project.basedir}/src/main/java - - - - - maven-source-plugin - 2.1 - - - compile - - jar - - - - - true - - - - + + 4.0.0 + + run.mone + jcommon + 1.6.0-jdk21-SNAPSHOT + + es + 1.7-jdk8-SNAPSHOT + + + UTF-8 + 7.17.21 + + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + ${es.version} + + + org.elasticsearch.client + elasticsearch-rest-client-sniffer + ${es.version} + + + org.elasticsearch + elasticsearch + ${es.version} + + + + run.mone + nacos + 1.4-v1-jdk20-SNAPSHOT + test + + + + + + maven-compiler-plugin + 3.11.0 + + 21 + 21 + true + UTF-8 + + ${project.basedir}/src/main/java + + + + + maven-source-plugin + 2.1 + + + compile + + jar + + + + + true + + + + diff --git a/jcommon/es/src/main/java/com/xiaomi/mone/es/EsClient.java b/jcommon/es/src/main/java/com/xiaomi/mone/es/EsClient.java index e2b145d2f..764e1b8cd 100644 --- a/jcommon/es/src/main/java/com/xiaomi/mone/es/EsClient.java +++ b/jcommon/es/src/main/java/com/xiaomi/mone/es/EsClient.java @@ -25,18 +25,14 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestClientBuilder; -import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.*; import org.elasticsearch.client.core.CountRequest; import org.elasticsearch.client.core.CountResponse; import org.elasticsearch.client.indices.*; import org.elasticsearch.client.sniff.ElasticsearchNodesSniffer; import org.elasticsearch.client.sniff.SniffOnFailureListener; import org.elasticsearch.client.sniff.Sniffer; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -47,6 +43,7 @@ import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.histogram.LongBounds; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.xcontent.XContentType; import java.io.IOException; import java.util.*; @@ -186,7 +183,9 @@ private RestClientBuilder createRestClientBuilder(List hosts, Header[] } private void initializeHighLevelClient(RestClientBuilder clientBuilder) { - this.client = new RestHighLevelClient(clientBuilder); + this.client = new RestHighLevelClientBuilder(clientBuilder.build()) + .setApiCompatibilityMode(true) + .build(); this.restClient = client.getLowLevelClient(); } diff --git a/jcommon/es/src/main/java/com/xiaomi/mone/es/EsProcessor.java b/jcommon/es/src/main/java/com/xiaomi/mone/es/EsProcessor.java index 7c9d7e71d..73e04962c 100644 --- a/jcommon/es/src/main/java/com/xiaomi/mone/es/EsProcessor.java +++ b/jcommon/es/src/main/java/com/xiaomi/mone/es/EsProcessor.java @@ -9,7 +9,7 @@ import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.core.TimeValue; import java.util.Map; import java.util.concurrent.TimeUnit; diff --git a/jcommon/es/src/test/java/com/xiaomi/mone/es/test/EsClientTest.java b/jcommon/es/src/test/java/com/xiaomi/mone/es/test/EsClientTest.java index bc9b5a24f..24e4d3e84 100644 --- a/jcommon/es/src/test/java/com/xiaomi/mone/es/test/EsClientTest.java +++ b/jcommon/es/src/test/java/com/xiaomi/mone/es/test/EsClientTest.java @@ -14,7 +14,7 @@ import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.indices.GetMappingsResponse; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; -import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; diff --git a/jcommon/es/src/test/java/com/xiaomi/mone/es/test/EsProcessorClientTest.java b/jcommon/es/src/test/java/com/xiaomi/mone/es/test/EsProcessorClientTest.java index a6a7fb69b..12a50198f 100644 --- a/jcommon/es/src/test/java/com/xiaomi/mone/es/test/EsProcessorClientTest.java +++ b/jcommon/es/src/test/java/com/xiaomi/mone/es/test/EsProcessorClientTest.java @@ -1,69 +1,79 @@ -package com.xiaomi.mone.es.test; - -import com.xiaomi.data.push.nacos.NacosConfig; -import com.xiaomi.mone.es.EsProcessor; -import com.xiaomi.mone.es.EsClient; -import com.xiaomi.mone.es.ProcessorConf; -import org.elasticsearch.action.bulk.BulkProcessor; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; -import org.junit.Test; - -import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; - -public class EsProcessorClientTest { - - @Test - public void bulkInsert() throws InterruptedException { - - NacosConfig config = new NacosConfig(); - config.setDataId("zzy_new"); -// config.init(); - - String ip = config.getConfig("es_ip"); - String user = config.getConfig("es_user"); - String pwd = config.getConfig("es_password"); - ProcessorConf conf = new ProcessorConf(100, 5, 1, 100, 3, 5, new EsClient(ip, user, pwd), new BulkProcessor.Listener() { - @Override - public void beforeBulk(long executionId, BulkRequest request) { - System.out.println("before insert" + request); - } - - @Override - public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { - System.out.println("success after,request:" + request.getDescription() + " resopnse:" + Arrays.toString(response.getItems())); - } - - @Override - public void afterBulk(long executionId, BulkRequest request, Throwable failure) { - System.out.println("success after,request:" + request + " failure:" + failure); - } - }); - EsProcessor processor = new EsProcessor(conf); - try { - String indexName = "zgq_common_milog_staging_free_private_1-" + new SimpleDateFormat("yyyy.MM.dd").format(new Date()); - Map data = new HashMap<>(); - data.put("timestamp", System.currentTimeMillis()); - data.put("filename", "/home/work/log/log-manager/server.log1"); - int n = 1; - int count = 0; - while (true) { +//package com.xiaomi.mone.es.test; +// +//import com.google.common.reflect.TypeToken; +//import com.google.gson.Gson; +//import com.xiaomi.data.push.nacos.NacosConfig; +//import com.xiaomi.mone.es.EsClient; +//import com.xiaomi.mone.es.EsProcessor; +//import com.xiaomi.mone.es.ProcessorConf; +//import org.elasticsearch.action.bulk.BulkProcessor; +//import org.elasticsearch.action.bulk.BulkRequest; +//import org.elasticsearch.action.bulk.BulkResponse; +//import org.junit.Test; +// +//import java.text.SimpleDateFormat; +//import java.time.Instant; +//import java.util.Arrays; +//import java.util.Date; +//import java.util.HashMap; +//import java.util.Map; +// +//public class EsProcessorClientTest { +// +// @Test +// public void bulkInsert() throws InterruptedException { +// +// String str = "{\"@timestamp\":\"2024-06-20T19:39:15.871+08:00\",\"@version\":\"1\",\"message\":\"hello world data test wtt~\",\"logger_name\":\"com.xiaomi.ai.Application\",\"thread_name\":\"http-nio-10010-exec-3\",\"level\":\"INFO\",\"level_value\":20000,\"LOG_NAME\":\"ai-workflow\",\"SENTRY_ENABLED\":\"false\",\"user_name\":\"wangjunfei3\",\"user_team\":\"ncl7150\",\"request_uri\":\"/hello\",\"trace_id\":\"9cf73bfe51e877a83806ac01b6630815\",\"trace_flags\":\"01\",\"span_id\":\"c13434f908acebb4\"}"; +// Map data = new Gson().fromJson(str, new TypeToken>() { +// }.getType()); +// data.put("timeStamp", System.currentTimeMillis()); +// +// NacosConfig config = new NacosConfig(); +// config.setDataId("zzy_new"); +//// config.init(); +// +// String ip = "zjydw.api.es.srv:80"; +// String user = config.getConfig("es_user"); +// String pwd = config.getConfig("es_password"); +// String token = "4244b7014a5c44fea63bea711c7697fe"; +// String catalog = "es_zjy_log"; +// String database = "default"; +// +// EsClient esClient = new EsClient(ip, token, catalog, database); +// ProcessorConf conf = new ProcessorConf(100, 5, 1, 100, 3, 5, esClient, new BulkProcessor.Listener() { +// @Override +// public void beforeBulk(long executionId, BulkRequest request) { +// System.out.println("before insert" + request); +// } +// +// @Override +// public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { +// System.out.println("success after,request:" + request.getDescription() + " resopnse:" + Arrays.toString(response.getItems())); +// } +// +// @Override +// public void afterBulk(long executionId, BulkRequest request, Throwable failure) { +// System.out.println("success after,request:" + request + " failure:" + failure); +// } +// }); +// EsProcessor processor = new EsProcessor(conf); +// try { +// String indexName = "prod_hera_index_95956-" + new SimpleDateFormat("yyyy.MM.dd").format(new Date()); +// int n = 1; +// int count = 0; +// while (true) { +//// processor.bulkInsert(indexName, data); // processor.bulkInsert(indexName, data); - processor.bulkInsert(indexName, data); - count++; - if (count == n) { - break; - } - } -// Thread.sleep(10000l); - System.in.read(); - }catch (Exception e){ - e.printStackTrace(); - } - - } -} \ No newline at end of file +// count++; +// if (count == n) { +// break; +// } +// } +//// Thread.sleep(10000l); +// System.in.read(); +// } catch (Exception e) { +// e.printStackTrace(); +// } +// +// } +//} \ No newline at end of file diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/ILogFile.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/ILogFile.java index f78f883e6..3889f4f08 100644 --- a/jcommon/file/src/main/java/com/xiaomi/mone/file/ILogFile.java +++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/ILogFile.java @@ -8,6 +8,8 @@ */ public interface ILogFile { + int LINE_MAX_LENGTH = 1100000; + void readLine() throws IOException; void setStop(boolean stop); diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile.java index 71654948e..ddc7878a7 100644 --- a/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile.java +++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile.java @@ -52,7 +52,7 @@ public class LogFile implements ILogFile { private String md5; - private static final int LINE_MAX_LENGTH = 50000; + // private static final int LINE_MAX_LENGTH = 50000; public LogFile() { diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile2.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile2.java index e77743c3a..fbbd399c4 100644 --- a/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile2.java +++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile2.java @@ -58,8 +58,6 @@ public class LogFile2 implements ILogFile { private String md5; - private static final int LINE_MAX_LENGTH = 50000; - public LogFile2() { } diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFileWS.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFileWS.java index 08043057b..8b2ade870 100644 --- a/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFileWS.java +++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFileWS.java @@ -40,8 +40,6 @@ public class LogFileWS extends LogFile { private String md5; - private static final int LINE_MAX_LENGTH = 50000; - private WatchService ws; { diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/MLog.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/MLog.java index 0f96fa63d..6e5bbc6f4 100644 --- a/jcommon/file/src/main/java/com/xiaomi/mone/file/MLog.java +++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/MLog.java @@ -53,9 +53,9 @@ public class MLog { // private static final Pattern ERROR_LINE_PATTERN = Pattern.compile(".*ERROR|.*(WARN|INFO).*(Exception|exception|error|Profiler)"); // private static final int MAX_ERROR_LINE_MATCH_LENGTH = 300; /** - * 最多聚合200行错误栈，避免queue无限增长 + * 最多聚合800行错误栈，避免queue无限增长 */ - private static final int MAX_MERGE_LINE = 400; + private static final int MAX_MERGE_LINE = 800; @Deprecated public List append(String msg) { diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/common/FileInfoCache.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/common/FileInfoCache.java index 968a5a83a..b065fb0db 100644 --- a/jcommon/file/src/main/java/com/xiaomi/mone/file/common/FileInfoCache.java +++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/common/FileInfoCache.java @@ -78,9 +78,12 @@ public void load() { } } - @SneakyThrows public void load(String filePath) { - this.filePath = filePath; - this.load(); + try { + this.filePath = filePath; + this.load(); + } catch (Exception e) { + log.error("load cache error,filePath:{}", filePath, e); + } } } diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/event/EventListener.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/event/EventListener.java index 860e6537c..fb35d1cfc 100644 --- a/jcommon/file/src/main/java/com/xiaomi/mone/file/event/EventListener.java +++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/event/EventListener.java @@ -8,4 +8,7 @@ public interface EventListener { void onEvent(FileEvent event); + default void remove(Object fileKey) { + } + } diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/listener/DefaultMonitorListener.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/listener/DefaultMonitorListener.java index 5347cd10f..f7a855994 100644 --- a/jcommon/file/src/main/java/com/xiaomi/mone/file/listener/DefaultMonitorListener.java +++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/listener/DefaultMonitorListener.java @@ -8,11 +8,11 @@ import com.xiaomi.mone.file.event.EventType; import com.xiaomi.mone.file.event.FileEvent; import com.xiaomi.mone.file.ozhera.HeraFileMonitor; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Consumer; @@ -28,8 +28,7 @@ public class DefaultMonitorListener implements EventListener { private Consumer consumer; - @Getter - private List readListenerList = new CopyOnWriteArrayList<>(); + private Map readListenerMap = new ConcurrentHashMap<>(); private ExecutorService pool = Executors.newVirtualThreadPerTaskExecutor(); @@ -44,7 +43,7 @@ public void onEvent(FileEvent event) { log.info("log file:{}", event.getFileName()); LogFile2 logFile = new LogFile2(event.getFileName()); OzHeraReadListener ozHeraReadListener = new OzHeraReadListener(monitor, logFile, consumer); - readListenerList.add(ozHeraReadListener); + readListenerMap.put(event.getFileKey(), ozHeraReadListener); pool.submit(() -> { logFile.setListener(ozHeraReadListener); SafeRun.run(logFile::readLine); @@ -58,6 +57,7 @@ public void onEvent(FileEvent event) { if (event.getType().equals(EventType.delete)) { log.info("delete:{}", event.getFileName()); + readListenerMap.remove(event.getFileKey()); } if (event.getType().equals(EventType.empty)) { @@ -71,11 +71,21 @@ public void onEvent(FileEvent event) { // LogFile2 logFile = new LogFile2(event.getFileName()); LogFile2 logFile = new LogFile2(event.getFileName(), 0, 0); OzHeraReadListener ozHeraReadListener = new OzHeraReadListener(monitor, logFile, consumer); - readListenerList.add(ozHeraReadListener); + readListenerMap.put(event.getFileKey(), ozHeraReadListener); + pool.submit(() -> { logFile.setListener(ozHeraReadListener); SafeRun.run(logFile::readLine); }); } } + + @Override + public void remove(Object fileKey) { + readListenerMap.remove(fileKey); + } + + public List getReadListenerList() { + return this.readListenerMap.values().stream().toList(); + } } diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFileMonitor.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFileMonitor.java index f7add647a..c8a2d1e84 100644 --- a/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFileMonitor.java +++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFileMonitor.java @@ -39,6 +39,9 @@ public class HeraFileMonitor { @Setter private EventListener listener; + @Setter + private volatile boolean stop; + public HeraFileMonitor() { this(TimeUnit.SECONDS.toMillis(30)); } @@ -53,10 +56,12 @@ public HeraFileMonitor(long removeTime) { remList.add(Pair.of(it.getFileName(), it.getFileKey())); } }); + remList.forEach(it -> { log.info("remove file:{}", it.getKey()); fileMap.remove(it.getKey()); map.remove(it.getValue()); + listener.remove(it.getValue()); }); } catch (Throwable ex) { log.error(ex.getMessage(), ex); @@ -85,6 +90,7 @@ public void reg(String path, Predicate predicate) throws IOException, In directory.register(watchService, StandardWatchEventKinds.ENTRY_MODIFY, StandardWatchEventKinds.ENTRY_DELETE, StandardWatchEventKinds.ENTRY_CREATE); while (true) { try { + WatchKey key = watchService.take(); try { for (WatchEvent event : key.pollEvents()) { diff --git a/jcommon/file/src/test/java/com/xiaomi/mone/file/LogFileTest.java b/jcommon/file/src/test/java/com/xiaomi/mone/file/LogFileTest.java index 8644d3e01..d7b987194 100644 --- a/jcommon/file/src/test/java/com/xiaomi/mone/file/LogFileTest.java +++ b/jcommon/file/src/test/java/com/xiaomi/mone/file/LogFileTest.java @@ -78,7 +78,7 @@ public void testLogFileMonitor() { Pattern pattern = Pattern.compile(fileName); monitor.reg("/home/work/log", it -> { boolean matches = pattern.matcher(it).matches(); - log.info("file:{},matches:{}", it, matches); + log.info("file:{},matches:{}", it, true); return true; }); log.info("reg finish"); From 0527f0a3edff6925c731d59870478ea6a7bf5818 Mon Sep 17 00:00:00 2001 From: HawickMason <1914991129@qq.com> Date: Thu, 15 Aug 2024 16:58:13 +0800 Subject: [PATCH 03/14] feat(codegen): support feature and test and table generation based on template or generator - add param parse (#866) * feat(codegen): support feature and test and table generation based on template or generator * feat(codegen): support feature and test and table generation based on template or generator - add param parse * feat(codegen): support feature and test and table generation based on template or generator - add param parse - 1 * feat(codegen): support feature and test and table generation based on template or generator - add param parse - 2 * feat(codegen): support feature and test and table generation based on template or generator - add param parse - 3] --------- Co-authored-by: wangyingjie3 Co-authored-by: wodiwudi <40229449+wodiwudi@users.noreply.github.com> --- .../run/mone/ai/codegen/FeatureGenerator.java | 134 +++++++++++++++--- .../mone/ai/codegen/bo/FeatureGenerateBo.java | 8 +- .../ai/codegen/bo/FeatureGenerateType.java | 42 ++++++ 3 files changed, 165 insertions(+), 19 deletions(-) create mode 100644 jcommon/codegen/src/main/java/run/mone/ai/codegen/bo/FeatureGenerateType.java diff --git a/jcommon/codegen/src/main/java/run/mone/ai/codegen/FeatureGenerator.java b/jcommon/codegen/src/main/java/run/mone/ai/codegen/FeatureGenerator.java index 2b1468021..5ddc8b3df 100644 --- a/jcommon/codegen/src/main/java/run/mone/ai/codegen/FeatureGenerator.java +++ b/jcommon/codegen/src/main/java/run/mone/ai/codegen/FeatureGenerator.java @@ -1,12 +1,18 @@ package run.mone.ai.codegen; import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; import com.mybatisflex.codegen.Generator; import com.mybatisflex.codegen.config.GlobalConfig; import com.zaxxer.hikari.HikariDataSource; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.jdbc.core.JdbcTemplate; +import run.mone.ai.codegen.bo.FeatureGenerateType; +import run.mone.ai.codegen.bo.FeatureGenerateBo; +import run.mone.ai.codegen.util.TemplateUtils; + +import java.lang.reflect.Type; import run.mone.ai.codegen.bo.FeatureGeneratType; import run.mone.ai.codegen.bo.FeatureGenerateBo; import run.mone.ai.codegen.util.TemplateUtils; @@ -23,27 +29,113 @@ @Slf4j public class FeatureGenerator { - private static final Gson gson = new Gson(); - public static void main(String[] args) { - FeatureGenerateBo featureGenerateBo = null; - //方便ai调用的时候,设置表名 + Map map; if (args.length > 0) { - featureGenerateBo = parseArgsAndExtractData(args); - generateWithTemplate(featureGenerateBo); + map = parseArgsAndExtractData(args); + String type = getGenType(map); + if (type == null) { + return; + } + FeatureGenerateType generateType = FeatureGenerateType.getGenerateTypeByCode(Integer.parseInt(type)); + FeatureGenerateBo featureGenerateBo = mapToGenBo(map); + switch (generateType) { + case CODE_WITH_GENERATOR: + case TABLE: + generateWithGenerator(featureGenerateBo); + break; + case CODE_WITH_TEMPLATE: + generateByTemplate(featureGenerateBo); + break; + default: + log.info("did not match any gen type, will do nothing..."); + } } } - public static void generateWithTemplate(FeatureGenerateBo featureGenerateBo) { + private static String getGenType(Map map) { + String type = map.get("type"); + if (StringUtils.isBlank(type)) { + log.error("generate type is not specified!"); + return null; + } + if (!StringUtils.isNumeric(type)) { + log.error("generate type is not valid, must be number! See @FeatureGenerateType.code"); + return null; + } + return type; + } + + private static FeatureGenerateBo mapToGenBo(Map map) { + FeatureGenerateBo.FeatureGenerateBoBuilder builder = FeatureGenerateBo.builder(); + if (map.containsKey("tableName")) { + builder.tableName(map.get("tableName")); + } + if (map.containsKey("testName")) { + builder.sql(map.get("testName")); + } + if (map.containsKey("sql")) { + builder.sql(map.get("sql")); + } + if (map.containsKey("jdbcUrl")) { + builder.jdbcUrl(map.get("jdbcUrl")); + } + if (map.containsKey("userName")) { + builder.userName(map.get("userName")); + } + if (map.containsKey("password")) { + builder.password(map.get("password")); + } + if (map.containsKey("basePackage")) { + builder.basePackage(map.get("basePackage")); + } + if (map.containsKey("className")) { + builder.className(map.get("className")); + } + if (map.containsKey("auth")) { + builder.auth(map.get("auth")); + } + if (map.containsKey("basePath")) { + builder.basePath(map.get("basePath")); + } + if (map.containsKey("serverModulePath")) { + builder.serverModulePath(map.get("serverModulePath")); + } + if (map.containsKey("serviceModulePath")) { + builder.serviceModulePath(map.get("serviceModulePath")); + } + if (map.containsKey("apiModulePath")) { + builder.apiModulePath(map.get("apiModulePath")); + } + if (map.containsKey("createPojo")) { + builder.createPojo(Boolean.parseBoolean(map.get("createPojo"))); + } + if (map.containsKey("createVo")) { + builder.createVo(Boolean.parseBoolean(map.get("createVo"))); + } + if (map.containsKey("createTransfer")) { + builder.createTransfer(Boolean.parseBoolean(map.get("createTransfer"))); + } + if (map.containsKey("createTest")) { + builder.createTest(Boolean.parseBoolean(map.get("createTest"))); + } + if (map.containsKey("createController")) { + builder.createController(Boolean.parseBoolean(map.get("createController"))); + } + return builder.build(); + } + + public static void generateWithGenerator(FeatureGenerateBo featureGenerateBo) { // 类型检查 if (Objects.isNull(featureGenerateBo.getType())) { log.warn("Empty generation type, will do noting!"); return; } - FeatureGeneratType featureGenType = featureGenerateBo.getType(); - if (FeatureGeneratType.CODE_WITH_TEMPLATE != featureGenType - && FeatureGeneratType.TABLE != featureGenType) { + + FeatureGenerateType featureGenType = featureGenerateBo.getType(); + if (FeatureGenerateType.CODE_WITH_TEMPLATE != featureGenType + && FeatureGenerateType.TABLE != featureGenType) { log.warn("generate type:{} is not match with current call", featureGenType); return; } @@ -56,7 +148,8 @@ public static void generateWithTemplate(FeatureGenerateBo featureGenerateBo) { dataSource.setPassword(featureGenerateBo.getPassword()); //创建mapper相关代码 - if (FeatureGeneratType.CODE_WITH_TEMPLATE == featureGenType) { + + if (FeatureGenerateType.CODE_WITH_TEMPLATE == featureGenType) { GlobalConfig globalConfig = createGlobalConfigUseStyle(featureGenerateBo); Generator generator = new Generator(dataSource, globalConfig); generator.generate(); @@ -64,7 +157,8 @@ public static void generateWithTemplate(FeatureGenerateBo featureGenerateBo) { } //创建table - if (FeatureGeneratType.TABLE == featureGenType) { + + if (FeatureGenerateType.TABLE == featureGenType) { JdbcTemplate jt = new JdbcTemplate(dataSource); jt.update(featureGenerateBo.getSql()); } @@ -138,7 +232,7 @@ public static void generateByTemplate(FeatureGenerateBo featureGenerateBo) { } if (featureGenerateBo.isCreateTest()) { - String cn = className + "ServiceTest.java"; + String cn = featureGenerateBo.getTestName(); String test = TemplateUtils.renderTemplateFromFile("tlp/test.java", data); TemplateUtils.writeStringToFile(test, basePath + "/" + featureGenerateBo.getServerModulePath() + "/src/test/java/run/mone/test/service/" + cn); } @@ -161,14 +255,20 @@ public static String replaceDotWithSlash(String input) { return input.replace(".", "/"); } - private static FeatureGenerateBo parseArgsAndExtractData(String[] args) { + private static Map parseArgsAndExtractData(String[] args) { + if (StringUtils.isBlank(args[0])) { + log.warn("no valid input params, will do nothing!"); + } String jsonStr = args[0]; + Type typeOfT = new TypeToken>() { + }.getType(); + jsonStr = new String(Base64.getDecoder().decode(jsonStr)); log.info("jsonStr:{}", jsonStr); - FeatureGenerateBo bo = gson.fromJson(jsonStr, FeatureGenerateBo.class); - log.info("map:{}", bo); - return bo; + Map map = new Gson().fromJson(jsonStr, typeOfT); + log.info("map:{}", map); + return map; } } diff --git a/jcommon/codegen/src/main/java/run/mone/ai/codegen/bo/FeatureGenerateBo.java b/jcommon/codegen/src/main/java/run/mone/ai/codegen/bo/FeatureGenerateBo.java index 248c6c3fb..b1f413fae 100644 --- a/jcommon/codegen/src/main/java/run/mone/ai/codegen/bo/FeatureGenerateBo.java +++ b/jcommon/codegen/src/main/java/run/mone/ai/codegen/bo/FeatureGenerateBo.java @@ -4,8 +4,10 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; + import lombok.ToString; + /** * @author goodjava@qq.com, HawickMason@xiaomi.com * @date 7/12/24 14:10 @@ -14,11 +16,10 @@ @AllArgsConstructor @Data @Builder -@ToString public class FeatureGenerateBo { @Builder.Default - private FeatureGeneratType type = FeatureGeneratType.CODE_WITH_GENERATOR; + private FeatureGenerateType type = FeatureGenerateType.CODE_WITH_GENERATOR; @Builder.Default private String tableName = ""; @@ -42,6 +43,8 @@ public class FeatureGenerateBo { @Builder.Default private String className = "Dummy"; + private String testName = "T"; + @Builder.Default private String auth = ""; @@ -75,6 +78,7 @@ public class FeatureGenerateBo { @Builder.Default private boolean createController = false; + /** * 目前使用module所在的绝对路径 */ diff --git a/jcommon/codegen/src/main/java/run/mone/ai/codegen/bo/FeatureGenerateType.java b/jcommon/codegen/src/main/java/run/mone/ai/codegen/bo/FeatureGenerateType.java new file mode 100644 index 000000000..136328130 --- /dev/null +++ b/jcommon/codegen/src/main/java/run/mone/ai/codegen/bo/FeatureGenerateType.java @@ -0,0 +1,42 @@ +package run.mone.ai.codegen.bo; + +import java.util.Arrays; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * @author goodjava@qq.com, HawickMason@xiaomi.com + * @date 7/12/24 14:15 + */ +public enum FeatureGenerateType { + + CODE_WITH_GENERATOR(1, "使用mybatis-flex-generator生成"), + + CODE_WITH_TEMPLATE(2, "使用预制模板生成"), + + TABLE(3, "创建表"); + + private final int code; + + private final String desc; + + private static final Map valMap = Arrays.stream(values()).collect(Collectors.toMap(FeatureGenerateType::getCode, Function.identity())); + + FeatureGenerateType(int code, String desc) { + this.code = code; + this.desc = desc; + } + + public int getCode() { + return code; + } + + public String getDesc() { + return desc; + } + + public static FeatureGenerateType getGenerateTypeByCode(int code) { + return valMap.getOrDefault(code, CODE_WITH_TEMPLATE); + } +} From 484e2f60efa58ecb3c9ec0de4b0635a400f21298 Mon Sep 17 00:00:00 2001 From: EricDing <128116675+sadadw1@users.noreply.github.com> Date: Fri, 16 Aug 2024 11:03:35 +0800 Subject: [PATCH 04/14] fix: update test error (#875) * feat: add prometheus-trace-etl release * feat: add gitlab log release jdk21 * feat: update athena * feat: update check code file path * feat: add code generator and mybatis generator * feat: add mybtais generator * fix: update test error --- .../xiaomi/youpin/codegen/test/FeatureGeneratorTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jcommon/codegen/src/test/java/com/xiaomi/youpin/codegen/test/FeatureGeneratorTest.java b/jcommon/codegen/src/test/java/com/xiaomi/youpin/codegen/test/FeatureGeneratorTest.java index ff39ead0e..13614e766 100644 --- a/jcommon/codegen/src/test/java/com/xiaomi/youpin/codegen/test/FeatureGeneratorTest.java +++ b/jcommon/codegen/src/test/java/com/xiaomi/youpin/codegen/test/FeatureGeneratorTest.java @@ -5,6 +5,7 @@ import run.mone.ai.codegen.MybatisGenerator; import run.mone.ai.codegen.bo.FeatureGeneratType; import run.mone.ai.codegen.bo.FeatureGenerateBo; +import run.mone.ai.codegen.bo.FeatureGenerateType; import run.mone.ai.codegen.bo.MybatisGeneratorResult; public class FeatureGeneratorTest { @@ -16,7 +17,7 @@ public class FeatureGeneratorTest { @Test public void testCreateTable() { FeatureGenerateBo featureGenerateBo = new FeatureGenerateBo(); - featureGenerateBo.setType(FeatureGeneratType.TABLE); + featureGenerateBo.setType(FeatureGenerateType.TABLE); featureGenerateBo.setJdbcUrl(dbUrl); featureGenerateBo.setUserName(dbUser); featureGenerateBo.setPassword(dbPwd); @@ -27,13 +28,12 @@ public void testCreateTable() { " PRIMARY KEY (`id`)\n" + ");"; featureGenerateBo.setSql(sql); - FeatureGenerator.generateWithTemplate(featureGenerateBo); + FeatureGenerator.generateWithGenerator(featureGenerateBo); } @Test public void testGenerateWithMybatis() { FeatureGenerateBo featureGenerateBo = new FeatureGenerateBo(); - featureGenerateBo.setType(FeatureGeneratType.CODE_WITH_MYBATIS_GENERATOR); featureGenerateBo.setJdbcUrl(dbUrl); featureGenerateBo.setUserName(dbUser); featureGenerateBo.setPassword(dbPwd); From c25f39129618a1b5f644cb65b7e073f19f16e483 Mon Sep 17 00:00:00 2001 From: zhangzhiyong Date: Fri, 16 Aug 2024 11:44:19 +0800 Subject: [PATCH 05/14] chore: update project dependencies and test files - Updated A.java and MoneCodeParserTest.java in neo4j test directory - Updated MoneCodeParser.java in neo4j main directory - Updated pom.xml in neo4j and jcommon directories --- jcommon/ai/neo4j/pom.xml | 51 +++ .../java/run/mone/neo4j/MoneCodeParser.java | 391 ++++++++++++++++++ .../src/test/java/run/mone/neo4j/test/A.java | 35 ++ .../mone/neo4j/test/MoneCodeParserTest.java | 20 + jcommon/pom.xml | 1 + 5 files changed, 498 insertions(+) create mode 100644 jcommon/ai/neo4j/pom.xml create mode 100644 jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java create mode 100644 jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/A.java create mode 100644 jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java diff --git a/jcommon/ai/neo4j/pom.xml b/jcommon/ai/neo4j/pom.xml new file mode 100644 index 000000000..035870b15 --- /dev/null +++ b/jcommon/ai/neo4j/pom.xml @@ -0,0 +1,51 @@ + + + 4.0.0 + + run.mone + jcommon + 1.6.0-jdk21-SNAPSHOT + ../../pom.xml + + + neo4j + + + 21 + 21 + UTF-8 + + + + + + + com.github.javaparser + javaparser-core + 3.26.1 + + + + org.neo4j.driver + neo4j-java-driver + 5.23.0 + + + + com.squareup.okhttp3 + okhttp + 4.9.1 + + + + com.google.code.gson + gson + 2.8.8 + + + + + + \ No newline at end of file diff --git a/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java new file mode 100644 index 000000000..328dc825b --- /dev/null +++ b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java @@ -0,0 +1,391 @@ +package run.mone.neo4j; + +import com.github.javaparser.JavaParser; +import com.github.javaparser.ast.CompilationUnit; +import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration; +import com.github.javaparser.ast.body.MethodDeclaration; +import com.github.javaparser.ast.comments.Comment; +import com.github.javaparser.ast.comments.JavadocComment; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import lombok.Data; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import okhttp3.*; +import org.neo4j.driver.Record; +import org.neo4j.driver.*; + +import java.io.File; +import java.io.IOException; +import java.util.*; + +@Slf4j +@Data +public class MoneCodeParser { + + + private String NEO4J_URI = "bolt://localhost:7687"; + + private String NEO4J_USER = "neo4j"; + + private String NEO4J_PASSWORD = ""; + + private String embeddingUrl = ""; + + + //查询所有Comment的信息(使用neo4j),返回是个List(class) + public List> getAllComments() { + try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, NEO4J_PASSWORD)); + Session session = driver.session()) { + List> comments = new ArrayList<>(); + Result result = session.run("MATCH (comment:Comment) RETURN comment, id(comment) as commentId"); + while (result.hasNext()) { + Record record = result.next(); + Map commentMap = new HashMap<>(record.get("comment").asMap()); + commentMap.put("commentId", record.get("commentId").asLong()); + comments.add(commentMap); + } + return comments; + } + } + + + /** + * 根据文本向量查询评论 + * + * @param text 输入的文本，用于生成查询向量 + * @return 查询结果的列表，包含评论节点和相似度分数 + */ + @SneakyThrows + public List> queryCommentsByTextVector(String text) { + // 替换为你的查询向量 + double[] queryVector = getTextVectorFromHttp(text); + + try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, NEO4J_PASSWORD)); + Session session = driver.session()) { + + // 执行查询 + List results = session.run("MATCH (c:Comment) " + + "WITH c, gds.similarity.cosine(c.text_vector, $query_vector) AS score " + + "WHERE score > 0.8 " + + "RETURN c,score", + Values.parameters("query_vector", queryVector)).list(); + + // 打印结果 + for (Record record : results) { + System.out.println(record.get("c").asNode().get("text").asString()); + System.out.println(record.get("score").asDouble()); + } + } + return null; + } + + //给你一个text,帮我查询Comment 中 text_vector 向量相似度最高的(class) + @SneakyThrows + public Map findMostSimilarComment(String text) { + double[] textVector = getTextVectorFromHttp(text); + List> comments = getAllComments(); + Map mostSimilarComment = null; + double highestSimilarity = -1; + + for (Map comment : comments) { + double[] commentVector = convertListToFloatArray((List) comment.get("text_vector")); + double similarity = calculateCosineSimilarity(textVector, commentVector); + if (similarity > highestSimilarity) { + highestSimilarity = similarity; + mostSimilarComment = comment; + } + } + + return mostSimilarComment; + } + + //List 转变成 float[](method) + public double[] convertListToFloatArray(List floatList) { + double[] floatArray = new double[floatList.size()]; + for (int i = 0; i < floatList.size(); i++) { + floatArray[i] = floatList.get(i); + } + return floatArray; + } + + /** + * 计算两个向量的余弦相似度 + * + * @param vectorA 第一个向量 + * @param vectorB 第二个向量 + * @return 两个向量的余弦相似度 + */ + private double calculateCosineSimilarity(double[] vectorA, double[] vectorB) { + double dotProduct = 0.0; + double normA = 0.0; + double normB = 0.0; + for (int i = 0; i < vectorA.length; i++) { + dotProduct += vectorA[i] * vectorB[i]; + normA += Math.pow(vectorA[i], 2); + normB += Math.pow(vectorB[i], 2); + } + return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)); + } + + + //调用一个Http 接口获取到Comment 的 text_vector float[](根据text),然后填充到 Comment中的text_vector中(class) + @SneakyThrows + public void fillCommentTextVectors() { + List> comments = getAllComments(); + for (Map comment : comments) { + String text = (String) comment.get("text"); + double[] textVector = getTextVectorFromHttp(text); + System.out.println(textVector.length); + comment.put("text_vector", textVector); + } + + updateCommentsInNeo4j(comments); + } + + //把Comment的修改,刷新回neo4j(class) + public void updateCommentsInNeo4j(List> comments) { + try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, NEO4J_PASSWORD)); + Session session = driver.session()) { + for (Map comment : comments) { + Long commentId = (Long) comment.get("commentId"); + double[] textVector = (double[]) comment.get("text_vector"); + Map params = new HashMap<>(); + params.put("id", commentId); + params.put("text_vector", textVector); + session.run("MATCH (c:Comment) WHERE id(c)=$id SET c.text_vector = $text_vector", params); + } + } + } + + + private static OkHttpClient client = new OkHttpClient(); + private static final MediaType JSON = MediaType.get("application/json; charset=utf-8"); + private static Gson gson = new Gson(); + + + private double[] getTextVectorFromHttp(String text) throws IOException { + JsonObject jsonRequest = new JsonObject(); + jsonRequest.addProperty("text", text); + + RequestBody body = RequestBody.create(jsonRequest.toString(), JSON); + Request request = new Request.Builder() + .url(embeddingUrl) + .post(body) + .build(); + + try (Response response = client.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new IOException("Unexpected code " + response); + } + + String responseBody = response.body().string(); + JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class); + + JsonArray dataArray = JsonParser.parseString(jsonObject.getAsJsonPrimitive("data").getAsString()).getAsJsonArray(); + + double[] result = new double[dataArray.size()]; + for (int i = 0; i < dataArray.size(); i++) { + result[i] = dataArray.get(i).getAsFloat(); + } + + return result; + } + } + + + public void writeJavaFilesToNeo4j(String directoryPath) { + if (new File(directoryPath).isFile()) { + writeToNeo4j(directoryPath); + return; + } + + getJavaFilesInDirectory(directoryPath).forEach(it -> { + log.info("parse it:{}", it); + writeToNeo4j(it); + }); + + } + + //给一个文件夹,获取里边是.java文件的列表,注意你需要递归获取(class) + public static List getJavaFilesInDirectory(String directoryPath) { + List javaFiles = new ArrayList<>(); + File directory = new File(directoryPath); + + if (directory.exists() && directory.isDirectory()) { + File[] files = directory.listFiles(); + if (files != null) { + for (File file : files) { + if (file.isDirectory()) { + javaFiles.addAll(getJavaFilesInDirectory(file.getAbsolutePath())); + } else if (file.getName().endsWith(".java")) { + javaFiles.add(file.getAbsolutePath()); + } + } + } + } + + return javaFiles; + } + + //删除所有节点(class) + public void deleteAllNodes() { + try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, NEO4J_PASSWORD)); + Session session = driver.session()) { + session.run("MATCH (n) DETACH DELETE n"); + } + } + + /** + * 将指定的Java文件解析后写入到Neo4j数据库中 + * + * @param filePath Java文件的路径 + */ + @SneakyThrows + private void writeToNeo4j(String filePath) { + //写入到neo4j中 + // 替换成你的 Java 文件路径 + String projectName = "MyProject"; + + try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, NEO4J_PASSWORD)); + Session session = driver.session()) { + + // 解析 Java 文件 + CompilationUnit cu = new JavaParser().parse(new File(filePath)).getResult().get(); + + // 创建 Project 节点 + createProjectNode(session, projectName); + + // 创建 File 节点 + createFileNode(session, projectName, filePath); + + // 遍历类、接口、方法等 + cu.accept(new Visitor(session, projectName, filePath), null); + } + } + + private static void createProjectNode(Session session, String projectName) { + Map projectParams = new HashMap<>(); + projectParams.put("name", projectName); + session.run("MERGE (p:Project {name: $name})", projectParams); + } + + private static void createFileNode(Session session, String projectName, String filePath) { + Map fileParams = new HashMap<>(); + fileParams.put("name", filePath); + fileParams.put("path", filePath); + fileParams.put("language", "Java"); + session.run("MERGE (f:File {name: $name, path: $path, language: $language})", fileParams); + + // 创建 CONTAINS 关系 (Project -[:CONTAINS]-> File) + Map containsParams = new HashMap<>(); + containsParams.put("projectName", projectName); + containsParams.put("fileName", filePath); + session.run("MATCH (p:Project {name: $projectName}) " + + "MATCH (f:File {name: $fileName}) " + + "MERGE (p)-[:CONTAINS]->(f)", + containsParams); + } + + private static class Visitor extends com.github.javaparser.ast.visitor.VoidVisitorAdapter { + + private final Session session; + private final String projectName; + private final String filePath; + + public Visitor(Session session, String projectName, String filePath) { + this.session = session; + this.projectName = projectName; + this.filePath = filePath; + } + + @Override + public void visit(ClassOrInterfaceDeclaration n, Void arg) { + // 创建 Class/Interface 节点 + Map classParams = new HashMap<>(); + classParams.put("name", n.getNameAsString()); + classParams.put("fullName", n.getFullyQualifiedName().orElse("")); + + session.run("MERGE (c:Class {name: $name, full_name: $fullName})", classParams); + + // 创建 CONTAINS 关系 (File -[:CONTAINS]-> Class) + Map containsParams = new HashMap<>(); + containsParams.put("fileName", filePath); + containsParams.put("className", n.getNameAsString()); + session.run("MATCH (f:File {name: $fileName}) " + + "MATCH (c:Class {name: $className}) " + + "MERGE (f)-[:CONTAINS]->(c)", + containsParams); + + super.visit(n, arg); + + } + + @Override + public void visit(MethodDeclaration n, Void arg) { + super.visit(n, arg); + + // 创建 Method 节点 + Map methodParams = new HashMap<>(); + methodParams.put("name", n.getNameAsString()); + methodParams.put("signature", n.getSignature().asString()); + methodParams.put("code_vector", new float[]{}); // 替换为实际的代码向量 + + session.run("MERGE (m:Method {name: $name, signature: $signature, code_vector: $code_vector})", methodParams); + + // 创建 DECLARES 关系 (Class -[:DECLARES]-> Method) + Map declaresParams = new HashMap<>(); + + if (n.findAncestor(ClassOrInterfaceDeclaration.class).isEmpty()) { + return; + } + + declaresParams.put("className", n.findAncestor(ClassOrInterfaceDeclaration.class).get().getNameAsString()); + declaresParams.put("methodName", n.getNameAsString()); + + session.run("MATCH (c:Class {name: $className}) " + + "MATCH (m:Method {name: $methodName}) " + + "MERGE (c)-[:DECLARES]->(m)", + declaresParams); + + // 处理注释 + for (Comment comment : n.getAllContainedComments()) { + createCommentNode(comment, n); + } + + Optional optional = n.getJavadocComment(); + if (optional.isPresent()) { + createCommentNode(optional.get(), n); + } + + Optional commentOptional = n.getComment(); + if (commentOptional.isPresent()) { + createCommentNode(commentOptional.get(), n); + } + + } + + private void createCommentNode(Comment comment, MethodDeclaration n) { + Map commentParams = new HashMap<>(); + commentParams.put("text", comment.getContent()); + commentParams.put("text_vector", new float[]{}); // 替换为实际的文本向量 + + session.run("MERGE (comment:Comment {text: $text, text_vector: $text_vector})", commentParams); + + // 创建 DOCUMENTS 关系 (Comment -[:DOCUMENTS]-> Method) + Map documentsParams = new HashMap<>(); + documentsParams.put("commentText", comment.getContent()); + documentsParams.put("methodName", n.getNameAsString()); + documentsParams.put("methodSignature", n.getSignature().asString()); + session.run("MATCH (comment:Comment {text: $commentText}) " + + "MATCH (m:Method {name: $methodName, signature: $methodSignature}) " + + "MERGE (comment)-[:DOCUMENTS]->(m)", + documentsParams); + } + + } + + +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/A.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/A.java new file mode 100644 index 000000000..8281e28ce --- /dev/null +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/A.java @@ -0,0 +1,35 @@ +package run.mone.neo4j.test; + +/** + * @author goodjava@qq.com + * @date 2024/8/16 10:16 + * AAA + */ +public class A { + + + /** + * 计算两数之和 + * + * @param a 第一个整数 + * @param b 第二个整数 + * @return 两数之和 + */ + //计算两数和 + public static int sum(int a, int b) { + return (a + b); + } + + /** + * 计算两个数的差值 + * + * @param num1 第一个数 + * @param num2 第二个数 + * @return 两数之差 + */ + //计算两个数的差值 + public static int subtract(int num1, int num2) { + return num1 - num2; + } + +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java new file mode 100644 index 000000000..0c83cb788 --- /dev/null +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java @@ -0,0 +1,20 @@ +package run.mone.neo4j.test; + +import org.junit.Test; +import run.mone.neo4j.MoneCodeParser; + +/** + * @author goodjava@qq.com + * @date 2024/8/16 09:42 + */ +public class MoneCodeParserTest { + + + @Test + public void testWriteCatServiceToNeo4j() { + new MoneCodeParser().writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/ai/m78/m78-service/src/main/java/run/mone/m78/service"); +// MoneCodeParser.writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/ai/m78/m78-service/src/main/java/run/mone/m78/service/database"); +// MoneCodeParser.writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/ai/m78/m78-service/src/main/java/run/mone/m78/service/database/SqlParseUtil.java"); +// new MoneCodeParser().writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/A.java"); + } +} diff --git a/jcommon/pom.xml b/jcommon/pom.xml index 66ebff815..317b7d665 100644 --- a/jcommon/pom.xml +++ b/jcommon/pom.xml @@ -88,6 +88,7 @@ infra-common docean-spring-starter ai + ai/neo4j From d5f6724ec9a27a63be885960e4e96eab6c39b36f Mon Sep 17 00:00:00 2001 From: wodiwudi <40229449+wodiwudi@users.noreply.github.com> Date: Mon, 19 Aug 2024 15:01:20 +0800 Subject: [PATCH 06/14] Refactor codegen unit test (#878) * update * refactor codegen * change pom * change infra pom --- jcommon/codegen/pom.xml | 152 ++++++++++-------- .../run/mone/ai/codegen/CodeGenerator.java | 78 ++++++++- .../mone/ai/codegen/util/TemplateUtils.java | 35 ++++ .../src/main/resources/tlp/testSpring.java | 29 ++++ .../xiaomi/youpin/codegen/test/GenTest.java | 17 +- 5 files changed, 231 insertions(+), 80 deletions(-) create mode 100644 jcommon/codegen/src/main/resources/tlp/testSpring.java diff --git a/jcommon/codegen/pom.xml b/jcommon/codegen/pom.xml index cfc3d4421..01f03b86c 100644 --- a/jcommon/codegen/pom.xml +++ b/jcommon/codegen/pom.xml @@ -2,77 +2,87 @@ - 4.0.0 - - run.mone - jcommon + 4.0.0 + + run.mone + jcommon + 1.6.0-jdk21-SNAPSHOT + + codegen 1.6.0-jdk21-SNAPSHOT - - codegen - 1.6.0-jdk21-SNAPSHOT - - - commons-io - commons-io - 2.6 - compile - - - run.mone - codegen-temlates - 1.0-SNAPSHOT - compile - - - com.ibeetl - beetl - 2.7.14 - - - com.mybatis-flex - mybatis-flex-codegen - 1.7.8 - - - org.springframework - spring-jdbc - 5.3.29 - true - - - com.zaxxer - HikariCP - 4.0.3 - true - - - org.apache.commons - commons-lang3 - 3.8.1 - compile - - - run.mone - infra-result - 1.6.0-jdk21-SNAPSHOT - - - thrift - org.apache.thrift - - - - - org.mybatis.generator - mybatis-generator-core - 1.4.0 - - - mysql - mysql-connector-java - 8.0.26 - + + + commons-io + commons-io + 2.6 + compile + + + run.mone + codegen-temlates + 1.0-SNAPSHOT + compile + + + com.ibeetl + beetl + 3.15.4.RELEASE + true + + + com.mybatis-flex + mybatis-flex-codegen + 1.7.8 + + + org.springframework + spring-jdbc + 5.3.29 + true + + + com.zaxxer + HikariCP + 4.0.3 + true + + + org.apache.commons + commons-lang3 + 3.8.1 + compile + + + run.mone + infra-result + 1.6.0-jdk21-SNAPSHOT + + + thrift + org.apache.thrift + + + + + org.mybatis.generator + mybatis-generator-core + 1.4.0 + + + mysql + mysql-connector-java + 8.0.26 + - - + + org.mockito + mockito-core + 3.9.0 + + + org.mockito + mockito-inline + 3.9.0 + + diff --git a/jcommon/codegen/src/main/java/run/mone/ai/codegen/CodeGenerator.java b/jcommon/codegen/src/main/java/run/mone/ai/codegen/CodeGenerator.java index 57483329f..dc2ddbb8b 100644 --- a/jcommon/codegen/src/main/java/run/mone/ai/codegen/CodeGenerator.java +++ b/jcommon/codegen/src/main/java/run/mone/ai/codegen/CodeGenerator.java @@ -24,7 +24,17 @@ public class CodeGenerator { private static String serviceName = "s"; - private static String basePath = "/Users/wodiwudi/java/nr-car-account"; + private static String testPackageName = "TP"; + + private static String testPath = ""; + + private static String mainClass = ""; + + private static String author = ""; + + private static String testType = ""; + + private static String basePath = "you base path"; private static boolean createPojo = false; @@ -38,6 +48,8 @@ public class CodeGenerator { private static boolean createController = false; + private static final String SPRING_BOOT_TEST_TYPE = "springboot"; + /** * 主方法 * 判断输入参数长度是否大于 0，调用方法处理参数并提取数据 @@ -53,6 +65,11 @@ public static void main(String[] args) { className = map.get("pojoName"); testName = map.get("testName"); serviceName = map.get("serviceName"); + testPackageName = map.get("testPackageName"); + testPath = map.get("testPath"); + mainClass = map.get("mainClass"); + author = map.get("author"); + testType = map.get("testType"); } if (StringUtils.isEmpty(className)) { return; @@ -61,6 +78,11 @@ public static void main(String[] args) { data.put("className", className); data.put("author", "goodjava@qq.com"); data.put("serviceName", serviceName); + data.put("testPackageName", testPackageName); + data.put("testPath", testPath); + data.put("mainClass", mainClass); + data.put("testAuthor", author); + data.put("testType", testType); Optional first = Arrays.stream(testName.split("\\.")).findFirst(); data.put("testName", first.get()); // 调用方法并获取结果 @@ -86,10 +108,14 @@ public static void main(String[] args) { if (createTest) { String cn = testName; - System.out.println("create test:" + cn); - String test = TemplateUtils.renderTemplateFromFile("tlp/test.java", data); - TemplateUtils.writeStringToFile(test, basePath + "/your project/src/test/java/com/xiaomi/nr/car/account/constant/" + cn); - + System.out.println("create test cn :" + cn + "path:" + testPath); + String test; + if (SPRING_BOOT_TEST_TYPE.equals(testType)) { + test = TemplateUtils.renderTemplateFromFile("tlp/testSpring.java", data); + } else { + test = TemplateUtils.renderTemplateFromFile("tlp/test.java", data); + } + TemplateUtils.writeStringToFile(test, testPath + cn); } if (createController) { @@ -98,6 +124,48 @@ public static void main(String[] args) { } } + public static void createAllUnitTest(Map map) { + if (map.isEmpty()) { + return; + } + //方便ai调用的时候,设置表名 + className = map.get("pojoName"); + testName = map.get("testName"); + serviceName = map.get("serviceName"); + testPackageName = map.get("testPackageName"); + testPath = map.get("testPath"); + mainClass = map.get("mainClass"); + author = map.get("author"); + testType = map.get("testType"); + + if (StringUtils.isEmpty(className)) { + return; + } + Map data = new HashMap<>(); + data.put("className", className); + data.put("author", "goodjava@qq.com"); + data.put("serviceName", serviceName); + data.put("testPackageName", testPackageName); + data.put("testPath", testPath); + data.put("mainClass", mainClass); + data.put("testAuthor", author); + data.put("testType", testType); + Optional first = Arrays.stream(testName.split("\\.")).findFirst(); + data.put("testName", first.get()); + + if (createTest) { + String cn = testName; + System.out.println("create test cn :" + cn + "path:" + testPath); + String test; + if (SPRING_BOOT_TEST_TYPE.equals(testType)) { + test = TemplateUtils.renderTemplateFromFileV2("tlp/testSpring.java", data); + } else { + test = TemplateUtils.renderTemplateFromFileV2("tlp/test.java", data); + } + TemplateUtils.writeStringToFile(test, testPath + cn); + } + } + private static Map parseArgsAndExtractData(String[] args) { String jsonStr = args[0]; diff --git a/jcommon/codegen/src/main/java/run/mone/ai/codegen/util/TemplateUtils.java b/jcommon/codegen/src/main/java/run/mone/ai/codegen/util/TemplateUtils.java index 4d2805cbe..1c3aaac36 100644 --- a/jcommon/codegen/src/main/java/run/mone/ai/codegen/util/TemplateUtils.java +++ b/jcommon/codegen/src/main/java/run/mone/ai/codegen/util/TemplateUtils.java @@ -14,6 +14,8 @@ import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; @@ -62,6 +64,39 @@ public static String renderTemplateFromFile(String templateFileName, Map m) { + try { + InputStream is = TemplateUtils.class.getClassLoader().getResourceAsStream(templateFileName); + if (is == null) { + throw new IOException("Resource not found: " + templateFileName); + } + + // 使用Channels和ByteBuffer读取所有字节 + ByteBuffer byteBuffer = ByteBuffer.allocate(1024); + int bytesRead; + while ((bytesRead = Channels.newChannel(is).read(byteBuffer)) != -1) { + if (byteBuffer.remaining() == 0) { + ByteBuffer newBuffer = ByteBuffer.allocate(byteBuffer.capacity() * 2); + byteBuffer.flip(); + newBuffer.put(byteBuffer); + byteBuffer = newBuffer; + } + } + byteBuffer.flip(); + byte[] bytes = new byte[byteBuffer.limit()]; + byteBuffer.get(bytes); + + // 将字节数组转换为字符串 + String template = new String(bytes, StandardCharsets.UTF_8); + + return renderTemplate(template, m); + } catch (IOException ex) { + log.error("Error reading template file", ex); + } + return ""; + } + //把String写到指定文件中(class) public static void writeStringToFile(String content, String filePath) { try (BufferedWriter writer = new BufferedWriter(new FileWriter(filePath))) { diff --git a/jcommon/codegen/src/main/resources/tlp/testSpring.java b/jcommon/codegen/src/main/resources/tlp/testSpring.java new file mode 100644 index 000000000..3e639ed7d --- /dev/null +++ b/jcommon/codegen/src/main/resources/tlp/testSpring.java @@ -0,0 +1,29 @@ +package ${testPackageName}; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import javax.annotation.Resource; +import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; +import static org.mockito.ArgumentMatchers.*; + +/** + * @author ${testAuthor} + */ +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = ${mainClass}.class) +public class ${testName} { + + +@Resource +private ${serviceName} ${strutil.toLowerCase(serviceName)}Service; + + +} \ No newline at end of file diff --git a/jcommon/codegen/src/test/java/com/xiaomi/youpin/codegen/test/GenTest.java b/jcommon/codegen/src/test/java/com/xiaomi/youpin/codegen/test/GenTest.java index 281b2c8ee..8f34f10f4 100644 --- a/jcommon/codegen/src/test/java/com/xiaomi/youpin/codegen/test/GenTest.java +++ b/jcommon/codegen/src/test/java/com/xiaomi/youpin/codegen/test/GenTest.java @@ -16,19 +16,27 @@ package com.xiaomi.youpin.codegen.test; +import com.google.common.collect.Lists; import com.xiaomi.youpin.codegen.*; import com.xiaomi.youpin.codegen.bo.ApiHeaderBo; import com.xiaomi.youpin.codegen.bo.Dependency; import com.xiaomi.youpin.infra.rpc.Result; +import org.apache.commons.lang3.tuple.Pair; import org.junit.Test; +import run.mone.ai.codegen.util.TemplateUtils; +import javax.annotation.Resource; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; public class GenTest { + @Resource + private TemplateUtils templateUtils; + @Test public void testFilter() throws Exception { FilterGen filterGen = new FilterGen(); @@ -38,8 +46,8 @@ public void testFilter() throws Exception { @Test public void testHttpRequestGen() throws Exception { HttpRequestGen httpRequestGen = new HttpRequestGen(); - httpRequestGen.generateJavaReq("getUserInfo",2,13,"[{\"paramNotNull\":\"0\",\"paramType\":\"13\",\"paramName\":\"obj\",\"paramKey\":\"subObj\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[{\"paramNotNull\":\"0\",\"paramType\":\"0\",\"paramName\":\"sub_dzx\",\"paramKey\":\"subName\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]},{\"paramNotNull\":\"0\",\"paramType\":\"3\",\"paramName\":\"666\",\"paramKey\":\"subID\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]}]}]"); - httpRequestGen.generateJavaReq("getUserInfo",2,13,"[{\"paramNotNull\":\"0\",\"paramType\":\"13\",\"paramName\":\"obj\",\"paramKey\":\"subObj\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[{\"paramNotNull\":\"0\",\"paramType\":\"0\",\"paramName\":\"sub_dzx\",\"paramKey\":\"subName\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]},{\"paramNotNull\":\"0\",\"paramType\":\"3\",\"paramName\":\"666\",\"paramKey\":\"subID\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]}]},{\"paramNotNull\":\"0\",\"paramType\":\"0\",\"paramName\":\"dzx\",\"paramKey\":\"userName\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]}]"); + httpRequestGen.generateJavaReq("getUserInfo", 2, 13, "[{\"paramNotNull\":\"0\",\"paramType\":\"13\",\"paramName\":\"obj\",\"paramKey\":\"subObj\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[{\"paramNotNull\":\"0\",\"paramType\":\"0\",\"paramName\":\"sub_dzx\",\"paramKey\":\"subName\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]},{\"paramNotNull\":\"0\",\"paramType\":\"3\",\"paramName\":\"666\",\"paramKey\":\"subID\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]}]}]"); + httpRequestGen.generateJavaReq("getUserInfo", 2, 13, "[{\"paramNotNull\":\"0\",\"paramType\":\"13\",\"paramName\":\"obj\",\"paramKey\":\"subObj\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[{\"paramNotNull\":\"0\",\"paramType\":\"0\",\"paramName\":\"sub_dzx\",\"paramKey\":\"subName\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]},{\"paramNotNull\":\"0\",\"paramType\":\"3\",\"paramName\":\"666\",\"paramKey\":\"subID\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]}]},{\"paramNotNull\":\"0\",\"paramType\":\"0\",\"paramName\":\"dzx\",\"paramKey\":\"userName\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]}]"); List headers = new ArrayList<>(); ApiHeaderBo header1 = new ApiHeaderBo(); @@ -50,7 +58,7 @@ public void testHttpRequestGen() throws Exception { header2.setHeaderValue("true"); headers.add(header1); headers.add(header2); - Result result = httpRequestGen.generateCurlReq(0,"/Api/getUserInfo",1,"[{\"paramNotNull\":\"0\",\"paramType\":\"13\",\"paramName\":\"obj\",\"paramKey\":\"subObj\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[{\"paramNotNull\":\"0\",\"paramType\":\"0\",\"paramName\":\"sub_dzx\",\"paramKey\":\"subName\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]},{\"paramNotNull\":\"0\",\"paramType\":\"3\",\"paramName\":\"666\",\"paramKey\":\"subID\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]}]}]",headers); + Result result = httpRequestGen.generateCurlReq(0, "/Api/getUserInfo", 1, "[{\"paramNotNull\":\"0\",\"paramType\":\"13\",\"paramName\":\"obj\",\"paramKey\":\"subObj\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[{\"paramNotNull\":\"0\",\"paramType\":\"0\",\"paramName\":\"sub_dzx\",\"paramKey\":\"subName\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]},{\"paramNotNull\":\"0\",\"paramType\":\"3\",\"paramName\":\"666\",\"paramKey\":\"subID\",\"paramValue\":\"\",\"paramLimit\":\"\",\"paramNote\":\"\",\"paramValueList\":[],\"default\":0,\"childList\":[]}]}]", headers); System.out.print(result.getData()); } @@ -137,9 +145,10 @@ public void testDDD() throws Exception { new DDDProGen().generateAndZip("/tmp/work", "abcd", "com.xiaomi.youpin", "com.xiaomi.youpin.abcdefg", "dfz", "1.0.0", dep); //new CNSalesCrmGen().generateAndZip("/home/work", "testt", "com.xiaomi.test", "com.xiaomi.test.testt", "dfz", "1.0.0", dep); } + @Test public void testFaas() throws Exception { FaasGen gen = new FaasGen(); - gen.generateAndZip("/tmp/work", "project-faas", "com.xiaomi.youpin", "com.xiaomi.youpin.projectmimimi", "dfz", "1.0.0","modulex","functiony"); + gen.generateAndZip("/tmp/work", "project-faas", "com.xiaomi.youpin", "com.xiaomi.youpin.projectmimimi", "dfz", "1.0.0", "modulex", "functiony"); } } From f3cdec08040029e389b1da44413ed99934e71c87 Mon Sep 17 00:00:00 2001 From: EricDing <128116675+sadadw1@users.noreply.github.com> Date: Mon, 19 Aug 2024 17:08:39 +0800 Subject: [PATCH 07/14] fix: datasource close 20240819 (#879) * feat: add prometheus-trace-etl release * feat: add gitlab log release jdk21 * feat: update athena * feat: update check code file path * feat: add code generator and mybatis generator * feat: add mybtais generator * fix: update test error * fix: add HikariDataSource close --- .../run/mone/ai/codegen/FeatureGenerator.java | 45 +++++++++---------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/jcommon/codegen/src/main/java/run/mone/ai/codegen/FeatureGenerator.java b/jcommon/codegen/src/main/java/run/mone/ai/codegen/FeatureGenerator.java index 5ddc8b3df..b5f946410 100644 --- a/jcommon/codegen/src/main/java/run/mone/ai/codegen/FeatureGenerator.java +++ b/jcommon/codegen/src/main/java/run/mone/ai/codegen/FeatureGenerator.java @@ -8,15 +8,11 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.jdbc.core.JdbcTemplate; -import run.mone.ai.codegen.bo.FeatureGenerateType; import run.mone.ai.codegen.bo.FeatureGenerateBo; +import run.mone.ai.codegen.bo.FeatureGenerateType; import run.mone.ai.codegen.util.TemplateUtils; import java.lang.reflect.Type; -import run.mone.ai.codegen.bo.FeatureGeneratType; -import run.mone.ai.codegen.bo.FeatureGenerateBo; -import run.mone.ai.codegen.util.TemplateUtils; - import java.util.Base64; import java.util.HashMap; import java.util.Map; @@ -141,26 +137,29 @@ public static void generateWithGenerator(FeatureGenerateBo featureGenerateBo) { } //配置数据源 - HikariDataSource dataSource = new HikariDataSource(); - dataSource.setDriverClassName("com.mysql.cj.jdbc.Driver"); - dataSource.setJdbcUrl(featureGenerateBo.getJdbcUrl()); - dataSource.setUsername(featureGenerateBo.getUserName()); - dataSource.setPassword(featureGenerateBo.getPassword()); - - //创建mapper相关代码 - - if (FeatureGenerateType.CODE_WITH_TEMPLATE == featureGenType) { - GlobalConfig globalConfig = createGlobalConfigUseStyle(featureGenerateBo); - Generator generator = new Generator(dataSource, globalConfig); - generator.generate(); - return; - } + try (HikariDataSource dataSource = new HikariDataSource()) { + dataSource.setDriverClassName("com.mysql.cj.jdbc.Driver"); + dataSource.setJdbcUrl(featureGenerateBo.getJdbcUrl()); + dataSource.setUsername(featureGenerateBo.getUserName()); + dataSource.setPassword(featureGenerateBo.getPassword()); + + //创建mapper相关代码 + + if (FeatureGenerateType.CODE_WITH_TEMPLATE == featureGenType) { + GlobalConfig globalConfig = createGlobalConfigUseStyle(featureGenerateBo); + Generator generator = new Generator(dataSource, globalConfig); + generator.generate(); + return; + } - //创建table + //创建table - if (FeatureGenerateType.TABLE == featureGenType) { - JdbcTemplate jt = new JdbcTemplate(dataSource); - jt.update(featureGenerateBo.getSql()); + if (FeatureGenerateType.TABLE == featureGenType) { + JdbcTemplate jt = new JdbcTemplate(dataSource); + jt.update(featureGenerateBo.getSql()); + } + } catch (Exception e) { + log.error("generate with template error", e); } } From 8a3985f9bf398a732c724ed7795b9d6de5048af3 Mon Sep 17 00:00:00 2001 From: caobaoyu Date: Tue, 20 Aug 2024 11:20:21 +0800 Subject: [PATCH 08/14] feat: update Gitlab.java with change clone from private to public --- .../gitlab/src/main/java/com/xiaomi/youpin/gitlab/Gitlab.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jcommon/gitlab/src/main/java/com/xiaomi/youpin/gitlab/Gitlab.java b/jcommon/gitlab/src/main/java/com/xiaomi/youpin/gitlab/Gitlab.java index 718043bc2..418d060bd 100644 --- a/jcommon/gitlab/src/main/java/com/xiaomi/youpin/gitlab/Gitlab.java +++ b/jcommon/gitlab/src/main/java/com/xiaomi/youpin/gitlab/Gitlab.java @@ -516,7 +516,7 @@ private static boolean modifyFile(String changedBody, String filePathToReplace, return true; } - private static boolean clone(String gitUrl, String branch, String username, String token, String gitPath) { + public static boolean clone(String gitUrl, String branch, String username, String token, String gitPath) { clearIfPresent(gitPath); CloneCommand cloneCommand = Git.cloneRepository().setURI(gitUrl) .setCredentialsProvider(new UsernamePasswordCredentialsProvider(username, token)) From fa5254ed6543a3f7488699b894e5b204994471c8 Mon Sep 17 00:00:00 2001 From: zhangzhiyong Date: Tue, 20 Aug 2024 14:29:18 +0800 Subject: [PATCH 09/14] chore: update test files and MoneCodeParser.java with no specific changes --- .../java/run/mone/neo4j/MoneCodeParser.java | 112 +++++++++++++++++- .../mone/neo4j/test/MoneCodeParserTest.java | 8 +- .../run/mone/neo4j/test/anno/Resource.java | 8 ++ .../mone/neo4j/test/anno/RestController.java | 8 ++ .../java/run/mone/neo4j/test/anno/Table.java | 8 ++ .../test/java/run/mone/neo4j/test/m/Cat.java | 21 ++++ .../run/mone/neo4j/test/m/CatService.java | 25 ++++ .../test/java/run/mone/neo4j/test/m/Dog.java | 11 ++ .../java/run/mone/neo4j/test/m/Person.java | 11 ++ .../test/{A.java => m/PersonService.java} | 12 +- 10 files changed, 216 insertions(+), 8 deletions(-) create mode 100644 jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Resource.java create mode 100644 jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/RestController.java create mode 100644 jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Table.java create mode 100644 jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Cat.java create mode 100644 jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java create mode 100644 jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Dog.java create mode 100644 jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Person.java rename jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/{A.java => m/PersonService.java} (72%) diff --git a/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java index 328dc825b..db7acf099 100644 --- a/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java +++ b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java @@ -3,9 +3,12 @@ import com.github.javaparser.JavaParser; import com.github.javaparser.ast.CompilationUnit; import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration; +import com.github.javaparser.ast.body.FieldDeclaration; import com.github.javaparser.ast.body.MethodDeclaration; import com.github.javaparser.ast.comments.Comment; import com.github.javaparser.ast.comments.JavadocComment; +import com.github.javaparser.ast.expr.AnnotationExpr; +import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonObject; @@ -16,6 +19,7 @@ import okhttp3.*; import org.neo4j.driver.Record; import org.neo4j.driver.*; +import org.neo4j.driver.types.Node; import java.io.File; import java.io.IOException; @@ -35,6 +39,29 @@ public class MoneCodeParser { private String embeddingUrl = ""; + public void queryEntityClasses() { + try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, NEO4J_PASSWORD)); + Session session = driver.session()) { + // 查询 type 为 'entity' 的所有 Class 节点 + String query = "MATCH (c:Class {type: 'entity'}) RETURN c"; + Result result = session.run(query); + while (result.hasNext()) { + Record record = result.next(); + Node classNode = record.get("c").asNode(); + String name = classNode.get("name").asString(); + String fullName = classNode.get("full_name").asString(); + String type = classNode.get("type").asString(); + + // 输出或处理查询结果 + System.out.println("Class Name: " + name); + System.out.println("Full Name: " + fullName); + System.out.println("Type: " + type); + System.out.println(classNode.get("code").asString()); + } + } + } + + //查询所有Comment的信息(使用neo4j),返回是个List(class) public List> getAllComments() { try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, NEO4J_PASSWORD)); @@ -52,13 +79,25 @@ public List> getAllComments() { } + //给你ClassOrInterfaceDeclaration,帮我过滤掉所有method中的body,返回这个class的String内容(class) + public static String filterMethodBodies(ClassOrInterfaceDeclaration classOrInterfaceDeclaration) { + classOrInterfaceDeclaration.getMethods().forEach(method -> method.setBody(null)); + return classOrInterfaceDeclaration.toString(); + } + + //给你ClassOrInterfaceDeclaration,帮我删除掉所有method,返回这个class的String内容(class) + public static String removeAllMethods(ClassOrInterfaceDeclaration classOrInterfaceDeclaration) { + classOrInterfaceDeclaration.getMethods().forEach(MethodDeclaration::remove); + return classOrInterfaceDeclaration.toString(); + } + /** * 根据文本向量查询评论 * * @param text 输入的文本，用于生成查询向量 * @return 查询结果的列表，包含评论节点和相似度分数 */ - @SneakyThrows + @SneakyThrows public List> queryCommentsByTextVector(String text) { // 替换为你的查询向量 double[] queryVector = getTextVectorFromHttp(text); @@ -118,7 +157,7 @@ public double[] convertListToFloatArray(List floatList) { * @param vectorB 第二个向量 * @return 两个向量的余弦相似度 */ - private double calculateCosineSimilarity(double[] vectorA, double[] vectorB) { + private double calculateCosineSimilarity(double[] vectorA, double[] vectorB) { double dotProduct = 0.0; double normA = 0.0; double normB = 0.0; @@ -166,7 +205,7 @@ public void updateCommentsInNeo4j(List> comments) { private static Gson gson = new Gson(); - private double[] getTextVectorFromHttp(String text) throws IOException { + private double[] getTextVectorFromHttp(String text) throws IOException { JsonObject jsonRequest = new JsonObject(); jsonRequest.addProperty("text", text); @@ -243,7 +282,7 @@ public void deleteAllNodes() { * * @param filePath Java文件的路径 */ - @SneakyThrows + @SneakyThrows private void writeToNeo4j(String filePath) { //写入到neo4j中 // 替换成你的 Java 文件路径 @@ -272,6 +311,7 @@ private static void createProjectNode(Session session, String projectName) { session.run("MERGE (p:Project {name: $name})", projectParams); } + private static void createFileNode(Session session, String projectName, String filePath) { Map fileParams = new HashMap<>(); fileParams.put("name", filePath); @@ -308,7 +348,27 @@ public void visit(ClassOrInterfaceDeclaration n, Void arg) { classParams.put("name", n.getNameAsString()); classParams.put("fullName", n.getFullyQualifiedName().orElse("")); - session.run("MERGE (c:Class {name: $name, full_name: $fullName})", classParams); + //class 的类型 + String type = getControllerType(n); + classParams.put("type", type); + + String code = ""; + + if (type.equals("entity")) { + code = removeAllMethods(n); + } + + classParams.put("code", code); + + System.out.println(classParams); + + + session.run( + "MERGE (c:Class {name: $name}) " + + "ON CREATE SET c.full_name = $fullName, c.type = $type, c.code = $code " + + "ON MATCH SET c.full_name = $fullName, c.type = $type, c.code = $code", + classParams + ); // 创建 CONTAINS 关系 (File -[:CONTAINS]-> Class) Map containsParams = new HashMap<>(); @@ -319,6 +379,33 @@ public void visit(ClassOrInterfaceDeclaration n, Void arg) { "MERGE (f)-[:CONTAINS]->(c)", containsParams); + + // 处理字段声明，查找 @Resource 注解 + n.findAll(FieldDeclaration.class).forEach(field -> { + field.getAnnotations().forEach(annotation -> { + if (annotation.getNameAsString().equals("Resource")) { + String fieldName = field.getVariables().get(0).getNameAsString(); + String fieldType = field.getElementType().asString(); + + // 创建 DEPENDS_ON 关系 (Class -[:DEPENDS_ON]-> Service) + Map dependsOnParams = new HashMap<>(); + dependsOnParams.put("className", n.getNameAsString()); + dependsOnParams.put("serviceName", fieldType); + dependsOnParams.put("fieldName", fieldName); + + + session.run("MERGE (c:Class {name: $name})", ImmutableMap.of("name",fieldType)); + + session.run("MATCH (c:Class {name: $className}) " + + "MATCH (s:Class {name: $serviceName}) " + + "MERGE (c)-[:DEPENDS_ON {field: $fieldName}]->(s)", + dependsOnParams); + } + }); + }); + + + super.visit(n, arg); } @@ -387,5 +474,20 @@ private void createCommentNode(Comment comment, MethodDeclaration n) { } + private static String getControllerType(ClassOrInterfaceDeclaration n) { + String type = ""; + Optional optional = n.getAnnotationByName("RestController"); + if (optional.isPresent()) { + type = "controller"; + } + + optional = n.getAnnotationByName("Table"); + if (optional.isPresent()) { + type = "entity"; + } + + return type; + } + } diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java index 0c83cb788..057a02e35 100644 --- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java @@ -12,9 +12,15 @@ public class MoneCodeParserTest { @Test public void testWriteCatServiceToNeo4j() { - new MoneCodeParser().writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/ai/m78/m78-service/src/main/java/run/mone/m78/service"); +// new MoneCodeParser().writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/ai/m78/m78-service/src/main/java/run/mone/m78/service"); // MoneCodeParser.writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/ai/m78/m78-service/src/main/java/run/mone/m78/service/database"); // MoneCodeParser.writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/ai/m78/m78-service/src/main/java/run/mone/m78/service/database/SqlParseUtil.java"); // new MoneCodeParser().writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/A.java"); + new MoneCodeParser().writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m"); + } + + @Test + public void test1() { + new MoneCodeParser().queryEntityClasses(); } } diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Resource.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Resource.java new file mode 100644 index 000000000..2366ed8d8 --- /dev/null +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Resource.java @@ -0,0 +1,8 @@ +package run.mone.neo4j.test.anno; + +/** + * @author goodjava@qq.com + * @date 2024/8/19 18:20 + */ +public @interface Resource { +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/RestController.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/RestController.java new file mode 100644 index 000000000..770f21261 --- /dev/null +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/RestController.java @@ -0,0 +1,8 @@ +package run.mone.neo4j.test.anno; + +/** + * @author goodjava@qq.com + * @date 2024/8/19 16:51 + */ +public @interface RestController { +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Table.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Table.java new file mode 100644 index 000000000..e009de88d --- /dev/null +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Table.java @@ -0,0 +1,8 @@ +package run.mone.neo4j.test.anno; + +/** + * @author goodjava@qq.com + * @date 2024/8/19 16:50 + */ +public @interface Table { +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Cat.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Cat.java new file mode 100644 index 000000000..5e3c87714 --- /dev/null +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Cat.java @@ -0,0 +1,21 @@ +package run.mone.neo4j.test.m; + +import run.mone.neo4j.test.anno.Table; + +/** + * @author goodjava@qq.com + * @date 2024/8/19 17:03 + */ +@Table +public class Cat { + + private int id; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java new file mode 100644 index 000000000..92ed980cb --- /dev/null +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java @@ -0,0 +1,25 @@ +package run.mone.neo4j.test.m; + +import run.mone.neo4j.test.anno.RestController; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author goodjava@qq.com + * @date 2024/8/19 18:21 + */ +@RestController +public class CatService { + + private Map data = new HashMap<>(); + + + //获取小猫的数量 + //获取小猫的数量 + public int getCatCount() { + return data.size(); + } + + +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Dog.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Dog.java new file mode 100644 index 000000000..2811cc06d --- /dev/null +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Dog.java @@ -0,0 +1,11 @@ +package run.mone.neo4j.test.m; + +import run.mone.neo4j.test.anno.Table; + +/** + * @author goodjava@qq.com + * @date 2024/8/19 17:03 + */ +@Table +public class Dog { +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Person.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Person.java new file mode 100644 index 000000000..1c67046be --- /dev/null +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Person.java @@ -0,0 +1,11 @@ +package run.mone.neo4j.test.m; + +import run.mone.neo4j.test.anno.Table; + +/** + * @author goodjava@qq.com + * @date 2024/8/19 16:51 + */ +@Table +public class Person { +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/A.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java similarity index 72% rename from jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/A.java rename to jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java index 8281e28ce..65794057e 100644 --- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/A.java +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java @@ -1,11 +1,19 @@ -package run.mone.neo4j.test; +package run.mone.neo4j.test.m; + +import run.mone.neo4j.test.anno.Resource; +import run.mone.neo4j.test.anno.RestController; /** * @author goodjava@qq.com * @date 2024/8/16 10:16 * AAA */ -public class A { +@RestController +public class PersonService { + + + @Resource + private CatService catService; /** From 8229cdc0655b24290520849dcc25bd5ed7c1ff9c Mon Sep 17 00:00:00 2001 From: zhangzhiyong Date: Wed, 21 Aug 2024 18:20:02 +0800 Subject: [PATCH 10/14] chore: update test resources and service classes --- .../src/main/java/run/mone/neo4j/BotCall.java | 64 ++++++++++++++ .../java/run/mone/neo4j/MoneCodeParser.java | 84 +++++++++++++++---- .../mone/neo4j/test/MoneCodeParserTest.java | 78 ++++++++++++++++- .../run/mone/neo4j/test/anno/Service.java | 8 ++ .../run/mone/neo4j/test/m/CatService.java | 3 +- .../test/java/run/mone/neo4j/test/m/Dog.java | 3 + .../run/mone/neo4j/test/m/PersonService.java | 4 +- jcommon/ai/neo4j/src/test/resources/entity.md | 15 ++++ .../ai/neo4j/src/test/resources/service.md | 16 ++++ 9 files changed, 257 insertions(+), 18 deletions(-) create mode 100644 jcommon/ai/neo4j/src/main/java/run/mone/neo4j/BotCall.java create mode 100644 jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Service.java create mode 100644 jcommon/ai/neo4j/src/test/resources/entity.md create mode 100644 jcommon/ai/neo4j/src/test/resources/service.md diff --git a/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/BotCall.java b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/BotCall.java new file mode 100644 index 000000000..0930633ac --- /dev/null +++ b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/BotCall.java @@ -0,0 +1,64 @@ +package run.mone.neo4j; + +import com.google.gson.JsonObject; +import lombok.Setter; +import okhttp3.*; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +/** + * @author goodjava@qq.com + * @date 2024/8/21 13:50 + */ +public class BotCall { + + //调用bot的地址 + @Setter + private static String url = ""; + + + public static String call(String desc, String input) { + OkHttpClient client = new OkHttpClient.Builder() + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(30, TimeUnit.SECONDS) + .writeTimeout(30, TimeUnit.SECONDS) + .build(); + + // 使用 Gson 的 JsonObject 构建请求体 + JsonObject mainObject = new JsonObject(); + mainObject.addProperty("userName", ""); + mainObject.addProperty("botId", ""); + mainObject.addProperty("input", ""); + + JsonObject paramsObject = new JsonObject(); + paramsObject.addProperty("desc", desc); + paramsObject.addProperty("input", input); + mainObject.add("params", paramsObject); + + // 将 JsonObject 转换为字符串 + String jsonBody = mainObject.toString(); + + RequestBody body = RequestBody.create(MediaType.parse("application/json"), jsonBody); + + Request request = new Request.Builder() + .url(url) + .post(body) + .addHeader("Accept", "application/json, text/plain, */*") + .addHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8") + .addHeader("Cache-Control", "no-cache") + .addHeader("Connection", "keep-alive") + .addHeader("Content-Type", "application/json") + .addHeader("Authorization", "") + .build(); + + try { + Response response = client.newCall(request).execute(); + return response.body().string(); + } catch (IOException e) { + e.printStackTrace(); + } + return ""; + } + +} diff --git a/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java index db7acf099..aa83443de 100644 --- a/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java +++ b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java @@ -8,6 +8,7 @@ import com.github.javaparser.ast.comments.Comment; import com.github.javaparser.ast.comments.JavadocComment; import com.github.javaparser.ast.expr.AnnotationExpr; +import com.github.javaparser.ast.nodeTypes.NodeWithName; import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -21,9 +22,10 @@ import org.neo4j.driver.*; import org.neo4j.driver.types.Node; -import java.io.File; -import java.io.IOException; +import java.io.*; +import java.nio.charset.StandardCharsets; import java.util.*; +import java.util.stream.Collectors; @Slf4j @Data @@ -34,13 +36,18 @@ public class MoneCodeParser { private String NEO4J_USER = "neo4j"; - private String NEO4J_PASSWORD = ""; + private String password = ""; private String embeddingUrl = ""; + public MoneCodeParser setPassword(String password) { + this.password = password; + return this; + } + public void queryEntityClasses() { - try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, NEO4J_PASSWORD)); + try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, password)); Session session = driver.session()) { // 查询 type 为 'entity' 的所有 Class 节点 String query = "MATCH (c:Class {type: 'entity'}) RETURN c"; @@ -61,10 +68,15 @@ public void queryEntityClasses() { } } + //获取session(class) + public Session getSession() { + return GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, password)).session(); + } + //查询所有Comment的信息(使用neo4j),返回是个List(class) public List> getAllComments() { - try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, NEO4J_PASSWORD)); + try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, password)); Session session = driver.session()) { List> comments = new ArrayList<>(); Result result = session.run("MATCH (comment:Comment) RETURN comment, id(comment) as commentId"); @@ -102,7 +114,7 @@ public List> queryCommentsByTextVector(String text) { // 替换为你的查询向量 double[] queryVector = getTextVectorFromHttp(text); - try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, NEO4J_PASSWORD)); + try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, password)); Session session = driver.session()) { // 执行查询 @@ -186,7 +198,7 @@ public void fillCommentTextVectors() { //把Comment的修改,刷新回neo4j(class) public void updateCommentsInNeo4j(List> comments) { - try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, NEO4J_PASSWORD)); + try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, password)); Session session = driver.session()) { for (Map comment : comments) { Long commentId = (Long) comment.get("commentId"); @@ -271,7 +283,7 @@ public static List getJavaFilesInDirectory(String directoryPath) { //删除所有节点(class) public void deleteAllNodes() { - try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, NEO4J_PASSWORD)); + try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, password)); Session session = driver.session()) { session.run("MATCH (n) DETACH DELETE n"); } @@ -288,7 +300,7 @@ private void writeToNeo4j(String filePath) { // 替换成你的 Java 文件路径 String projectName = "MyProject"; - try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, NEO4J_PASSWORD)); + try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, password)); Session session = driver.session()) { // 解析 Java 文件 @@ -311,6 +323,31 @@ private static void createProjectNode(Session session, String projectName) { session.run("MERGE (p:Project {name: $name})", projectParams); } + /** + * 查找具有指定注解的类 + * + * @param session 数据库会话 + * @param annotationToFind 要查找的注解 + * @return 具有指定注解的类的列表，每个类以Map形式表示 + */ + public List> findClassesWithAnnotation(Session session, String annotationToFind) { + Map params = new HashMap<>(); + params.put("annotation", annotationToFind); + Result result = session.run( + "MATCH (c:Class) " + + "WHERE ANY(anno IN c.anno WHERE anno = $annotation) " + + "RETURN c", + params + ); + List> list = new ArrayList<>(); + while (result.hasNext()) { + Record record = result.next(); + System.out.println(record.get("c").asMap()); + list.add(record.get("c").asMap()); + } + return list; + } + private static void createFileNode(Session session, String projectName, String filePath) { Map fileParams = new HashMap<>(); @@ -341,6 +378,7 @@ public Visitor(Session session, String projectName, String filePath) { this.filePath = filePath; } + @Override public void visit(ClassOrInterfaceDeclaration n, Void arg) { // 创建 Class/Interface 节点 @@ -360,13 +398,20 @@ public void visit(ClassOrInterfaceDeclaration n, Void arg) { classParams.put("code", code); - System.out.println(classParams); + //获取ClassOrInterfaceDeclaration中的注解 + List annoList = n.getAnnotations().stream().peek(annotation -> { + System.out.println("Annotation: " + annotation.getNameAsString()); + }).map(NodeWithName::getNameAsString).toList(); + //注解 + classParams.put("annotations", annoList); + + System.out.println(classParams); session.run( "MERGE (c:Class {name: $name}) " + - "ON CREATE SET c.full_name = $fullName, c.type = $type, c.code = $code " + - "ON MATCH SET c.full_name = $fullName, c.type = $type, c.code = $code", + "ON CREATE SET c.full_name = $fullName, c.type = $type, c.code = $code, c.anno = $annotations " + + "ON MATCH SET c.full_name = $fullName, c.type = $type, c.code = $code, c.anno = $annotations", classParams ); @@ -394,7 +439,7 @@ public void visit(ClassOrInterfaceDeclaration n, Void arg) { dependsOnParams.put("fieldName", fieldName); - session.run("MERGE (c:Class {name: $name})", ImmutableMap.of("name",fieldType)); + session.run("MERGE (c:Class {name: $name})", ImmutableMap.of("name", fieldType)); session.run("MATCH (c:Class {name: $className}) " + "MATCH (s:Class {name: $serviceName}) " + @@ -405,7 +450,6 @@ public void visit(ClassOrInterfaceDeclaration n, Void arg) { }); - super.visit(n, arg); } @@ -489,5 +533,17 @@ private static String getControllerType(ClassOrInterfaceDeclaration n) { return type; } + //读取resource下某个文件的文本内容(class) + public String readResourceFileContent(String fileName) { + try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream(fileName); + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + return reader.lines().collect(Collectors.joining(System.lineSeparator())); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + } diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java index 057a02e35..64577800a 100644 --- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java @@ -1,7 +1,32 @@ package run.mone.neo4j.test; +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; import org.junit.Test; +import org.neo4j.driver.Record; +import org.neo4j.driver.Result; +import org.neo4j.driver.Session; +import org.neo4j.driver.internal.InternalRecord; +import org.neo4j.driver.internal.InternalResult; +import org.neo4j.driver.internal.InternalSession; +import run.mone.neo4j.BotCall; import run.mone.neo4j.MoneCodeParser; +import run.mone.neo4j.test.MoneCodeParserTest; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + /** * @author goodjava@qq.com @@ -10,17 +35,68 @@ public class MoneCodeParserTest { + @Test public void testWriteCatServiceToNeo4j() { // new MoneCodeParser().writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/ai/m78/m78-service/src/main/java/run/mone/m78/service"); // MoneCodeParser.writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/ai/m78/m78-service/src/main/java/run/mone/m78/service/database"); // MoneCodeParser.writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/ai/m78/m78-service/src/main/java/run/mone/m78/service/database/SqlParseUtil.java"); // new MoneCodeParser().writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/A.java"); - new MoneCodeParser().writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m"); + new MoneCodeParser().setPassword(System.getenv("password")).writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m"); } @Test public void test1() { new MoneCodeParser().queryEntityClasses(); } + + + + + @Test + public void testFindClassesWithAnnotation() { + MoneCodeParser moneCodeParser = new MoneCodeParser().setPassword(System.getenv("password")); + List> actual = moneCodeParser.findClassesWithAnnotation(moneCodeParser.getSession(), "Table"); + System.out.println(actual); + +// String xuqiu = "获取狗的主人"; + String xuqiu = "管理宠物鸟"; + String res = BotCall.call(moneCodeParser.readResourceFileContent("entity.md") + xuqiu, new Gson().toJson(actual)); + System.out.println(res); + } + + + + @Test + public void testFindClassesWithAnnotation2() { + MoneCodeParser moneCodeParser = new MoneCodeParser().setPassword(System.getenv("password")); + List> actual = moneCodeParser.findClassesWithAnnotation(moneCodeParser.getSession(), "RestController"); + System.out.println(actual); + + String xuqiu = "获取狗的主人"; +// String xuqiu = "管理宠物鸟"; + String res = BotCall.call(moneCodeParser.readResourceFileContent("service.md") + xuqiu, new Gson().toJson(actual)); + System.out.println(res); + } + + + @Test + public void testJson() { + Map m = ImmutableMap.of("input", "a+b=?"); + System.out.println(new Gson().toJson(m)); + } + + + @Test + public void testReadResourceFileContent() { + String fileName = "entity.md"; + // Assuming the test resource file is already placed in the resources directory + String actualContent = new MoneCodeParser().readResourceFileContent(fileName); + + assertNotNull(actualContent); + } + + + + } diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Service.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Service.java new file mode 100644 index 000000000..b76563edf --- /dev/null +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Service.java @@ -0,0 +1,8 @@ +package run.mone.neo4j.test.anno; + +/** + * @author goodjava@qq.com + * @date 2024/8/21 16:06 + */ +public @interface Service { +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java index 92ed980cb..623cf6578 100644 --- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java @@ -1,6 +1,7 @@ package run.mone.neo4j.test.m; import run.mone.neo4j.test.anno.RestController; +import run.mone.neo4j.test.anno.Service; import java.util.HashMap; import java.util.Map; @@ -9,7 +10,7 @@ * @author goodjava@qq.com * @date 2024/8/19 18:21 */ -@RestController +@Service public class CatService { private Map data = new HashMap<>(); diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Dog.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Dog.java index 2811cc06d..4b5dc12c3 100644 --- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Dog.java +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/Dog.java @@ -8,4 +8,7 @@ */ @Table public class Dog { + + private int id; + } diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java index 65794057e..a94ae0dfa 100644 --- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java @@ -2,13 +2,13 @@ import run.mone.neo4j.test.anno.Resource; import run.mone.neo4j.test.anno.RestController; +import run.mone.neo4j.test.anno.Service; /** * @author goodjava@qq.com * @date 2024/8/16 10:16 - * AAA */ -@RestController +@Service public class PersonService { diff --git a/jcommon/ai/neo4j/src/test/resources/entity.md b/jcommon/ai/neo4j/src/test/resources/entity.md new file mode 100644 index 000000000..0b8529b60 --- /dev/null +++ b/jcommon/ai/neo4j/src/test/resources/entity.md @@ -0,0 +1,15 @@ +我有这些表映射类,你帮我分析.我给你一个新功能,你看看能不能满足. +不满足的原因: +1.缺少相应的字段 +2.就没有这个类 + +如果不满足,你需要按如下规定返回: +1.缺少字段 satisfies:false reason:missing_fields code:返回缺少字段的代码 (example: private String name;) +2.缺少某个类 satisfies:false reason:missing_class code:返回缺少的类的代码 + +一些规则: +1.缺少字段的时候,尽量不要添加类字段,尽量引用那个类的id即可. +2.如果缺少类,你要帮我尽量补全这个类中缺少的字段. +3.生成的字段或者类尽量添加合适的中文注释 + +需求: \ No newline at end of file diff --git a/jcommon/ai/neo4j/src/test/resources/service.md b/jcommon/ai/neo4j/src/test/resources/service.md new file mode 100644 index 000000000..0670fbd6e --- /dev/null +++ b/jcommon/ai/neo4j/src/test/resources/service.md @@ -0,0 +1,16 @@ +我有这些表映射类和service类的信息,然后我给你一个新功能,你分析下看看能不能满足. + +不满足的原因: +1.缺少相应的方法 +2.缺少某个service类 + +如果不满足,你需要按如下规定返回: +1.缺少字段 satisfies:false reason:missing_methods code:返回缺少方法的代码 (example: private String name;) +2.缺少某个service类 satisfies:false reason:missing_class code:返回缺少的类的代码 + +一些规则: +1.缺少method的时候,生成的方法只需要给方法定义即可. +2.如果缺少类,你要帮我生成这个类,类中的方法也只需要生成接口定义. +3.生成的方法或者类尽量添加合适的中文注释 + +需求: \ No newline at end of file From b5f078b66fdc4432b673070cf89d30aab2bdb1e1 Mon Sep 17 00:00:00 2001 From: dingpei Date: Thu, 22 Aug 2024 22:31:20 +0800 Subject: [PATCH 11/14] chore: update RequestPayload.java with no content changes --- .../src/main/java/run/mone/ai/google/bo/RequestPayload.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jcommon/ai/google/src/main/java/run/mone/ai/google/bo/RequestPayload.java b/jcommon/ai/google/src/main/java/run/mone/ai/google/bo/RequestPayload.java index b649a1331..ac55968a9 100644 --- a/jcommon/ai/google/src/main/java/run/mone/ai/google/bo/RequestPayload.java +++ b/jcommon/ai/google/src/main/java/run/mone/ai/google/bo/RequestPayload.java @@ -14,6 +14,8 @@ @Builder public class RequestPayload { + @SerializedName("system") + private String system; @SerializedName("anthropic_version") private String anthropicVersion; From 5303918a15a5300f3d6cde1d4d6e8553fc2e14fc Mon Sep 17 00:00:00 2001 From: zhangzhiyong Date: Tue, 27 Aug 2024 14:44:44 +0800 Subject: [PATCH 12/14] Code Map Update Test --- .../java/run/mone/neo4j/MoneCodeParser.java | 171 ++++++++++++++++-- .../mone/neo4j/test/MoneCodeParserTest.java | 2 + .../java/run/mone/neo4j/test/anno/Test.java | 8 + .../run/mone/neo4j/test/m/PersonService.java | 12 +- .../mone/neo4j/test/m/PersonServiceTest.java | 22 +++ .../neo4j/test/m/{ => cat}/CatService.java | 10 +- 6 files changed, 209 insertions(+), 16 deletions(-) create mode 100644 jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Test.java create mode 100644 jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonServiceTest.java rename jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/{ => cat}/CatService.java (60%) diff --git a/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java index aa83443de..df1647094 100644 --- a/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java +++ b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java @@ -2,13 +2,16 @@ import com.github.javaparser.JavaParser; import com.github.javaparser.ast.CompilationUnit; +import com.github.javaparser.ast.ImportDeclaration; import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration; import com.github.javaparser.ast.body.FieldDeclaration; import com.github.javaparser.ast.body.MethodDeclaration; import com.github.javaparser.ast.comments.Comment; import com.github.javaparser.ast.comments.JavadocComment; -import com.github.javaparser.ast.expr.AnnotationExpr; +import com.github.javaparser.ast.expr.*; import com.github.javaparser.ast.nodeTypes.NodeWithName; +import com.github.javaparser.ast.type.ClassOrInterfaceType; +import com.github.javaparser.resolution.declarations.ResolvedMethodDeclaration; import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -326,11 +329,11 @@ private static void createProjectNode(Session session, String projectName) { /** * 查找具有指定注解的类 * - * @param session 数据库会话 + * @param session 数据库会话 * @param annotationToFind 要查找的注解 * @return 具有指定注解的类的列表，每个类以Map形式表示 */ - public List> findClassesWithAnnotation(Session session, String annotationToFind) { + public List> findClassesWithAnnotation(Session session, String annotationToFind) { Map params = new HashMap<>(); params.put("annotation", annotationToFind); Result result = session.run( @@ -339,7 +342,7 @@ public List> findClassesWithAnnotation(Session session, Stri "RETURN c", params ); - List> list = new ArrayList<>(); + List> list = new ArrayList<>(); while (result.hasNext()) { Record record = result.next(); System.out.println(record.get("c").asMap()); @@ -406,8 +409,6 @@ public void visit(ClassOrInterfaceDeclaration n, Void arg) { //注解 classParams.put("annotations", annoList); - System.out.println(classParams); - session.run( "MERGE (c:Class {name: $name}) " + "ON CREATE SET c.full_name = $fullName, c.type = $type, c.code = $code, c.anno = $annotations " + @@ -454,13 +455,135 @@ public void visit(ClassOrInterfaceDeclaration n, Void arg) { } + + private String getFullMethodName(MethodDeclaration method) { + String packageName = method.findCompilationUnit() + .flatMap(cu -> cu.getPackageDeclaration()) + .map(pd -> pd.getNameAsString()) + .orElse(""); + String className = method.findAncestor(com.github.javaparser.ast.body.ClassOrInterfaceDeclaration.class) + .map(c -> c.getNameAsString()) + .orElse(""); + String methodName = method.getNameAsString(); + return packageName + "." + className + "." + methodName; + } + + /** + * 获取方法调用的完整路径，包括包名、类名和方法名 + * + * @param methodCall 方法调用表达式 + * @return 方法调用的完整路径 + */ + public String getFullMethodPath(MethodCallExpr methodCall) { + StringBuilder fullPath = new StringBuilder(); + + // 获取包名 + Optional cu = methodCall.findCompilationUnit(); + if (cu.isPresent()) { + cu.get().getPackageDeclaration().ifPresent(pkg -> + fullPath.append(pkg.getNameAsString()).append(".") + ); + } + + // 获取类名 + String className = methodCall.findAncestor(ClassOrInterfaceDeclaration.class) + .map(ClassOrInterfaceDeclaration::getNameAsString) + .orElse(""); + + // 获取方法调用的对象 + String objectName = methodCall.getScope() + .map(scope -> scope.toString()) + .orElse(""); + + + //静态调用 + if (methodCall.getScope().isPresent() && methodCall.getScope().get() instanceof FieldAccessExpr) { + return objectName + "." + methodCall.getNameAsString(); + } + + //lombok 的log + if (isLogCall(methodCall)) { + return objectName + "." + methodCall.getNameAsString(); + } + + // 如果对象名不为空，尝试找到它的类型 + if (!objectName.isEmpty()) { + Optional field = methodCall.findAncestor(ClassOrInterfaceDeclaration.class) + .flatMap(classDecl -> classDecl.getFieldByName(objectName)); + + if (field.isPresent()) { + ClassOrInterfaceType type = field.get().getVariable(0).getType().asClassOrInterfaceType(); + String v = resolveTypePath(type); + return v + "." + methodCall.getNameAsString(); + } + } + + + // 构建完整路径 + fullPath.append(className).append("."); + fullPath.append(methodCall.getNameAsString()); + + return fullPath.toString(); + } + + public static String resolveTypePath(ClassOrInterfaceType type) { + String typeName = type.getNameAsString(); + + Optional cu = type.findAncestor(CompilationUnit.class); + if (cu.isPresent()) { + // 尝试从导入声明中查找匹配 + Optional importedPath = findMatchingImport(cu.get(), typeName); + if (importedPath.isPresent()) { + return importedPath.get(); + } + + // 如果没有找到匹配的导入，检查是否在同一包中 + Optional currentPackage = getCurrentPackage(cu.get()); + if (currentPackage.isPresent()) { + return currentPackage.get() + "." + typeName; + } + } + + // 如果无法解析，返回原始类型名称 + return typeName; + } + + private static Optional findMatchingImport(CompilationUnit cu, String typeName) { + return cu.getImports().stream() + .filter(importDecl -> !importDecl.isAsterisk() && importDecl.getNameAsString().endsWith("." + typeName)) + .map(ImportDeclaration::getNameAsString) + .findFirst(); + } + + private static Optional getCurrentPackage(CompilationUnit cu) { + return cu.getPackageDeclaration().map(pd -> pd.getNameAsString()); + } + + /** + * 判断方法调用是否为日志调用 + * + * @param n 方法调用表达式 + * @return 如果方法调用是日志调用则返回true，否则返回false + */ + private boolean isLogCall(MethodCallExpr n) { + if (!n.getScope().isPresent()) { + return false; + } + String scope = n.getScope().get().toString(); + String method = n.getNameAsString(); + return scope.equals("log") && + (method.equals("trace") || method.equals("debug") || method.equals("info") || + method.equals("warn") || method.equals("error")); + } + + @Override public void visit(MethodDeclaration n, Void arg) { super.visit(n, arg); // 创建 Method 节点 Map methodParams = new HashMap<>(); - methodParams.put("name", n.getNameAsString()); + methodParams.put("name", getFullMethodName(n)); methodParams.put("signature", n.getSignature().asString()); methodParams.put("code_vector", new float[]{}); // 替换为实际的代码向量 @@ -474,7 +597,7 @@ public void visit(MethodDeclaration n, Void arg) { } declaresParams.put("className", n.findAncestor(ClassOrInterfaceDeclaration.class).get().getNameAsString()); - declaresParams.put("methodName", n.getNameAsString()); + declaresParams.put("methodName", getFullMethodName(n)); session.run("MATCH (c:Class {name: $className}) " + "MATCH (m:Method {name: $methodName}) " + @@ -482,6 +605,34 @@ public void visit(MethodDeclaration n, Void arg) { declaresParams); // 处理注释 + processComments(n); + + // 处理方法调用 + processMethodCalls(n); + + } + + // 处理方法调用 + private void processMethodCalls(MethodDeclaration n) { + n.findAll(MethodCallExpr.class).forEach(methodCall -> { + Map callParams = new HashMap<>(); + callParams.put("callerName", getFullMethodName(n)); + callParams.put("calleeName", getFullMethodPath(methodCall)); + + // 创建目标方法节点（如果不存在） + session.run("MERGE (callee:Method {name: $calleeName})", callParams); + + // 创建 CALLS 关系 + session.run("MATCH (caller:Method {name: $callerName}) " + + "MATCH (callee:Method {name: $calleeName}) " + + "MERGE (caller)-[:CALLS]->(callee)", + callParams); + }); + } + + + // 处理注释 + private void processComments(MethodDeclaration n) { for (Comment comment : n.getAllContainedComments()) { createCommentNode(comment, n); } @@ -495,7 +646,6 @@ public void visit(MethodDeclaration n, Void arg) { if (commentOptional.isPresent()) { createCommentNode(commentOptional.get(), n); } - } private void createCommentNode(Comment comment, MethodDeclaration n) { @@ -508,7 +658,7 @@ private void createCommentNode(Comment comment, MethodDeclaration n) { // 创建 DOCUMENTS 关系 (Comment -[:DOCUMENTS]-> Method) Map documentsParams = new HashMap<>(); documentsParams.put("commentText", comment.getContent()); - documentsParams.put("methodName", n.getNameAsString()); + documentsParams.put("methodName", getFullMethodName(n)); documentsParams.put("methodSignature", n.getSignature().asString()); session.run("MATCH (comment:Comment {text: $commentText}) " + "MATCH (m:Method {name: $methodName, signature: $methodSignature}) " + @@ -545,5 +695,4 @@ public String readResourceFileContent(String fileName) { } - } diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java index 64577800a..440a1be06 100644 --- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java @@ -43,6 +43,8 @@ public void testWriteCatServiceToNeo4j() { // MoneCodeParser.writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/ai/m78/m78-service/src/main/java/run/mone/m78/service/database/SqlParseUtil.java"); // new MoneCodeParser().writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/A.java"); new MoneCodeParser().setPassword(System.getenv("password")).writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m"); +// new MoneCodeParser().setPassword(System.getenv("password")).writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java"); +// new MoneCodeParser().setPassword(System.getenv("password")).writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java"); } @Test diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Test.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Test.java new file mode 100644 index 000000000..a28757ccb --- /dev/null +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/anno/Test.java @@ -0,0 +1,8 @@ +package run.mone.neo4j.test.anno; + +/** + * @author goodjava@qq.com + * @date 2024/8/21 16:06 + */ +public @interface Test { +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java index a94ae0dfa..3201894ad 100644 --- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java @@ -1,14 +1,16 @@ package run.mone.neo4j.test.m; +import lombok.extern.slf4j.Slf4j; import run.mone.neo4j.test.anno.Resource; -import run.mone.neo4j.test.anno.RestController; import run.mone.neo4j.test.anno.Service; +import run.mone.neo4j.test.m.cat.CatService; /** * @author goodjava@qq.com * @date 2024/8/16 10:16 */ @Service +@Slf4j public class PersonService { @@ -16,6 +18,14 @@ public class PersonService { private CatService catService; + //获取猫的数量 + public int getCatCount() { + log.info("info"); + System.out.println("abc"); + return catService.getCatCount(); + } + + /** * 计算两数之和 * diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonServiceTest.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonServiceTest.java new file mode 100644 index 000000000..a1ebd24a3 --- /dev/null +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonServiceTest.java @@ -0,0 +1,22 @@ +package run.mone.neo4j.test.m; + +import run.mone.neo4j.test.anno.Resource; +import run.mone.neo4j.test.anno.Service; +import run.mone.neo4j.test.anno.Test; + +/** + * @author goodjava@qq.com + * @date 2024/8/26 18:23 + */ +@Service +public class PersonServiceTest { + + @Resource + private PersonService personService; + + @Test + public void test1() { + int res = personService.getCatCount(); + System.out.println(res); + } +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/cat/CatService.java similarity index 60% rename from jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java rename to jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/cat/CatService.java index 623cf6578..494c6776c 100644 --- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/CatService.java +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/cat/CatService.java @@ -1,24 +1,26 @@ -package run.mone.neo4j.test.m; +package run.mone.neo4j.test.m.cat; -import run.mone.neo4j.test.anno.RestController; +import lombok.extern.slf4j.Slf4j; import run.mone.neo4j.test.anno.Service; import java.util.HashMap; -import java.util.Map; /** * @author goodjava@qq.com * @date 2024/8/19 18:21 */ @Service +@Slf4j public class CatService { - private Map data = new HashMap<>(); + private HashMap data = new HashMap<>(); //获取小猫的数量 //获取小猫的数量 public int getCatCount() { + log.info("abc"); + System.out.println("123"); return data.size(); } From 05ec8e90e9b2574a0702664a7173b4e1954ed329 Mon Sep 17 00:00:00 2001 From: dingpei Date: Wed, 28 Aug 2024 14:07:41 +0800 Subject: [PATCH 13/14] chore: update multiple files in gpt module without content changes --- jcommon/ai/gpt/pom.xml | 54 ++++++++++++ .../main/java/run/mone/ai/gpt/GptClient.java | 54 ++++++++++++ .../run/mone/ai/gpt/bo/ResponsePayload.java | 87 +++++++++++++++++++ .../gpt/bo/multiModal/GptVisionContent.java | 22 +++++ .../ai/gpt/bo/multiModal/GptVisionMsg.java | 23 +++++ .../gpt/bo/multiModal/GptVisionRequest.java | 46 ++++++++++ jcommon/ai/pom.xml | 1 + 7 files changed, 287 insertions(+) create mode 100644 jcommon/ai/gpt/pom.xml create mode 100644 jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/GptClient.java create mode 100644 jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/ResponsePayload.java create mode 100644 jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionContent.java create mode 100644 jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionMsg.java create mode 100644 jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionRequest.java diff --git a/jcommon/ai/gpt/pom.xml b/jcommon/ai/gpt/pom.xml new file mode 100644 index 000000000..1e751705d --- /dev/null +++ b/jcommon/ai/gpt/pom.xml @@ -0,0 +1,54 @@ + + + 4.0.0 + + run.mone + ai + 1.4-jdk20-SNAPSHOT + + + gpt + 1.5-jdk8-SNAPSHOT + + + 8 + 8 + UTF-8 + + + + + + com.squareup.okhttp3 + okhttp + 4.10.0 + + + + + + + + + + maven-compiler-plugin + 3.11.0 + + 8 + 8 + true + UTF-8 + + ${project.basedir}/src/main/java + + + + + + + + + + \ No newline at end of file diff --git a/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/GptClient.java b/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/GptClient.java new file mode 100644 index 000000000..49eb76c8a --- /dev/null +++ b/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/GptClient.java @@ -0,0 +1,54 @@ +package run.mone.ai.gpt; + +import com.google.gson.Gson; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import okhttp3.*; +import run.mone.ai.gpt.bo.ResponsePayload; +import run.mone.ai.gpt.bo.multiModal.GptVisionRequest; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +@Data +@Slf4j +public class GptClient { + + private static Gson gson = new Gson(); + + public ResponsePayload visionCall(String url, String token, GptVisionRequest gptVisionRequest) { + return baseCall(url, token, gson.toJson(gptVisionRequest)); + } + + private ResponsePayload baseCall(String url, String token, String bodyStr) { + OkHttpClient client = new OkHttpClient.Builder().readTimeout(5, TimeUnit.MINUTES).build(); + MediaType mediaType = MediaType.parse("application/json; charset=utf-8"); + RequestBody body = RequestBody.create(mediaType, bodyStr); + Request request = new Request.Builder() + .url(url) + .post(body) + .addHeader("api-key", token) + .addHeader("Content-Type", "application/json; charset=utf-8") + .build(); + + try (Response response = client.newCall(request).execute()) { + if (response.code() == 429) { + ResponsePayload res = new ResponsePayload(); + ResponsePayload.Error error = new ResponsePayload.Error(); + error.setCode("429"); + error.setMessage("被gpt限流了"); + res.setError(error); + return res; + } + if (!response.isSuccessful()) throw new IOException("Unexpected code " + response); + // Handle the response + String res = response.body().string(); + log.info("claude3 res:{}", res); + return new Gson().fromJson(res, ResponsePayload.class); + } catch (Throwable e) { + log.error(e.getMessage(), e); + } + return null; + } + +} diff --git a/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/ResponsePayload.java b/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/ResponsePayload.java new file mode 100644 index 000000000..1aabe2c5d --- /dev/null +++ b/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/ResponsePayload.java @@ -0,0 +1,87 @@ +package run.mone.ai.gpt.bo; + +import lombok.Data; + +import java.util.List; + +@Data +public class ResponsePayload { + private List choices; + private long created; + private String id; + private String model; + private String object; + private List prompt_filter_results; + private String system_fingerprint; + private Usage usage; + private Error error; + + @Data + public static class Error { + private String code; + private String message; + } + + @Data + public static class Choice { + private ContentFilterResults content_filter_results; + private String finish_reason; + private int index; + private Message message; + + + @Data + public static class ContentFilterResults { + private FilterResult hate; + private FilterResult self_harm; + private FilterResult sexual; + private FilterResult violence; + + } + + @Data + public static class FilterResult { + private boolean filtered; + private String severity; + + } + + @Data + public static class Message { + private String content; + private String role; + + } + } + + @Data + public static class PromptFilterResult { + private int prompt_index; + private ContentFilterResult content_filter_result; + + + @Data + public static class ContentFilterResult { + private JailbreakResult jailbreak; + private Choice.FilterResult sexual; + private Choice.FilterResult violence; + private Choice.FilterResult hate; + private Choice.FilterResult self_harm; + + + @Data + public static class JailbreakResult { + private boolean filtered; + private boolean detected; + + } + } + } + + @Data + public static class Usage { + private int completion_tokens; + private int prompt_tokens; + private int total_tokens; + } +} diff --git a/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionContent.java b/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionContent.java new file mode 100644 index 000000000..36343a787 --- /dev/null +++ b/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionContent.java @@ -0,0 +1,22 @@ +package run.mone.ai.gpt.bo.multiModal; + +import com.google.gson.annotations.SerializedName; +import lombok.Builder; +import lombok.Data; + +import java.util.Map; + +@Data +@Builder +public class GptVisionContent { + + @SerializedName("type") + private String type; + + @SerializedName("text") + private String text; + + @SerializedName("image_url") + private Map imageUrl; + +} diff --git a/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionMsg.java b/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionMsg.java new file mode 100644 index 000000000..6787b3146 --- /dev/null +++ b/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionMsg.java @@ -0,0 +1,23 @@ +package run.mone.ai.gpt.bo.multiModal; + +import com.google.gson.annotations.SerializedName; +import lombok.Builder; +import lombok.Data; + +import java.io.Serializable; +import java.util.List; + +/** + * @author goodjava@qq.com + * @date 2023/5/25 14:16 + */ +@Data +@Builder +public class GptVisionMsg implements Serializable { + + @SerializedName("role") + private String role; + + @SerializedName("content") + private List content; +} diff --git a/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionRequest.java b/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionRequest.java new file mode 100644 index 000000000..fab363962 --- /dev/null +++ b/jcommon/ai/gpt/src/main/java/run/mone/ai/gpt/bo/multiModal/GptVisionRequest.java @@ -0,0 +1,46 @@ +package run.mone.ai.gpt.bo.multiModal; + +import com.google.gson.annotations.SerializedName; +import lombok.Builder; +import lombok.Data; + +import java.util.List; + +/** + * @author goodjava@qq.com + * @date 2024/4/9 16:36 + */ +@Data +@Builder +public class GptVisionRequest { + + @SerializedName("model") + private String model; + + @SerializedName("temperature") + private double temperature; + + @SerializedName("n") + @Builder.Default + private int n = 1; + + @SerializedName("stream") + private boolean stream; + + @SerializedName("top_p") + private double topP; + + @SerializedName("max_tokens") + private int maxTokens; + + @SerializedName("presence_penalty") + private double presencePenalty; + + @SerializedName("frequency_penalty") + private double frequencyPenalty; + + @SerializedName("messages") + private List messages; + + +} diff --git a/jcommon/ai/pom.xml b/jcommon/ai/pom.xml index 41a73b29f..748017515 100644 --- a/jcommon/ai/pom.xml +++ b/jcommon/ai/pom.xml @@ -18,6 +18,7 @@ aws minimax bytedance + gpt From 766ce45680947a69b85b6d62eba9cd74545b5a86 Mon Sep 17 00:00:00 2001 From: zhangzhiyong Date: Thu, 29 Aug 2024 13:45:09 +0800 Subject: [PATCH 14/14] chore: update test and main classes without code changes --- .../java/run/mone/neo4j/MoneCodeParser.java | 27 +- .../run/mone/neo4j/MoneMethodCodeParser.java | 299 ++++++++++++++++++ .../mone/neo4j/test/MoneCodeParserTest.java | 24 +- .../run/mone/neo4j/test/m/cat/CatService.java | 1 - 4 files changed, 310 insertions(+), 41 deletions(-) create mode 100644 jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneMethodCodeParser.java diff --git a/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java index df1647094..b9521ab5e 100644 --- a/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java +++ b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneCodeParser.java @@ -8,10 +8,11 @@ import com.github.javaparser.ast.body.MethodDeclaration; import com.github.javaparser.ast.comments.Comment; import com.github.javaparser.ast.comments.JavadocComment; -import com.github.javaparser.ast.expr.*; +import com.github.javaparser.ast.expr.AnnotationExpr; +import com.github.javaparser.ast.expr.FieldAccessExpr; +import com.github.javaparser.ast.expr.MethodCallExpr; import com.github.javaparser.ast.nodeTypes.NodeWithName; import com.github.javaparser.ast.type.ClassOrInterfaceType; -import com.github.javaparser.resolution.declarations.ResolvedMethodDeclaration; import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -607,30 +608,8 @@ public void visit(MethodDeclaration n, Void arg) { // 处理注释 processComments(n); - // 处理方法调用 - processMethodCalls(n); - } - // 处理方法调用 - private void processMethodCalls(MethodDeclaration n) { - n.findAll(MethodCallExpr.class).forEach(methodCall -> { - Map callParams = new HashMap<>(); - callParams.put("callerName", getFullMethodName(n)); - callParams.put("calleeName", getFullMethodPath(methodCall)); - - // 创建目标方法节点（如果不存在） - session.run("MERGE (callee:Method {name: $calleeName})", callParams); - - // 创建 CALLS 关系 - session.run("MATCH (caller:Method {name: $callerName}) " + - "MATCH (callee:Method {name: $calleeName}) " + - "MERGE (caller)-[:CALLS]->(callee)", - callParams); - }); - } - - // 处理注释 private void processComments(MethodDeclaration n) { for (Comment comment : n.getAllContainedComments()) { diff --git a/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneMethodCodeParser.java b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneMethodCodeParser.java new file mode 100644 index 000000000..05833ffde --- /dev/null +++ b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneMethodCodeParser.java @@ -0,0 +1,299 @@ +package run.mone.neo4j; + +import com.github.javaparser.JavaParser; +import com.github.javaparser.ast.CompilationUnit; +import com.github.javaparser.ast.ImportDeclaration; +import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration; +import com.github.javaparser.ast.body.FieldDeclaration; +import com.github.javaparser.ast.body.MethodDeclaration; +import com.github.javaparser.ast.comments.Comment; +import com.github.javaparser.ast.comments.JavadocComment; +import com.github.javaparser.ast.expr.FieldAccessExpr; +import com.github.javaparser.ast.expr.MethodCallExpr; +import com.github.javaparser.ast.type.ClassOrInterfaceType; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.neo4j.driver.*; + +import java.io.File; +import java.util.*; + +/** + * @author goodjava@qq.com + *

+ * 建立方法调用之间的关系(call),且只建立业务调用的关系 + */ +@Slf4j +public class MoneMethodCodeParser { + + private String NEO4J_URI = "bolt://localhost:7687"; + + private String NEO4J_USER = "neo4j"; + + private String password = ""; + + public MoneMethodCodeParser setPassword(String password) { + this.password = password; + return this; + } + + + public void writeJavaFilesToNeo4j(String directoryPath) { + if (new File(directoryPath).isFile()) { + writeToNeo4j(directoryPath); + return; + } + + getJavaFilesInDirectory(directoryPath).forEach(it -> { + log.info("parse it:{}", it); + writeToNeo4j(it); + }); + } + + //给一个文件夹,获取里边是.java文件的列表,注意你需要递归获取(class) + public static List getJavaFilesInDirectory(String directoryPath) { + List javaFiles = new ArrayList<>(); + File directory = new File(directoryPath); + + if (directory.exists() && directory.isDirectory()) { + File[] files = directory.listFiles(); + if (files != null) { + for (File file : files) { + if (file.isDirectory()) { + javaFiles.addAll(getJavaFilesInDirectory(file.getAbsolutePath())); + } else if (file.getName().endsWith(".java")) { + javaFiles.add(file.getAbsolutePath()); + } + } + } + } + return javaFiles; + } + + + @SneakyThrows + private void writeToNeo4j(String filePath) { + //写入到neo4j中 + // 替换成你的 Java 文件路径 + String projectName = "MyProject"; + + try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, password)); + Session session = driver.session()) { + // 解析 Java 文件 + CompilationUnit cu = new JavaParser().parse(new File(filePath)).getResult().get(); + + // 遍历类、接口、方法等 + cu.accept(new MoneMethodCodeParser.Visitor(session, projectName, filePath), null); + } + } + + + private static class Visitor extends com.github.javaparser.ast.visitor.VoidVisitorAdapter { + + private final Session session; + + public Visitor(Session session, String projectName, String filePath) { + this.session = session; + } + + + @Override + public void visit(ClassOrInterfaceDeclaration n, Void arg) { + super.visit(n, arg); + } + + + private String getFullMethodName(MethodDeclaration method) { + String packageName = method.findCompilationUnit() + .flatMap(cu -> cu.getPackageDeclaration()) + .map(pd -> pd.getNameAsString()) + .orElse(""); + String className = method.findAncestor(com.github.javaparser.ast.body.ClassOrInterfaceDeclaration.class) + .map(c -> c.getNameAsString()) + .orElse(""); + String methodName = method.getNameAsString(); + return packageName + "." + className + "." + methodName; + } + + /** + * 获取方法调用的完整路径，包括包名、类名和方法名 + * + * @param methodCall 方法调用表达式 + * @return 方法调用的完整路径 + */ + public String getFullMethodPath(MethodCallExpr methodCall) { + StringBuilder fullPath = new StringBuilder(); + + // 获取包名 + Optional cu = methodCall.findCompilationUnit(); + if (cu.isPresent()) { + cu.get().getPackageDeclaration().ifPresent(pkg -> + fullPath.append(pkg.getNameAsString()).append(".") + ); + } + + // 获取类名 + String className = methodCall.findAncestor(ClassOrInterfaceDeclaration.class) + .map(ClassOrInterfaceDeclaration::getNameAsString) + .orElse(""); + + // 获取方法调用的对象 + String objectName = methodCall.getScope() + .map(scope -> scope.toString()) + .orElse(""); + + + //静态调用 + if (methodCall.getScope().isPresent() && methodCall.getScope().get() instanceof FieldAccessExpr) { + return objectName + "." + methodCall.getNameAsString(); + } + + //lombok 的log + if (isLogCall(methodCall)) { + return objectName + "." + methodCall.getNameAsString(); + } + + // 如果对象名不为空，尝试找到它的类型 + if (!objectName.isEmpty()) { + Optional field = methodCall.findAncestor(ClassOrInterfaceDeclaration.class) + .flatMap(classDecl -> classDecl.getFieldByName(objectName)); + + if (field.isPresent()) { + ClassOrInterfaceType type = field.get().getVariable(0).getType().asClassOrInterfaceType(); + String v = resolveTypePath(type); + return v + "." + methodCall.getNameAsString(); + } + } + + + // 构建完整路径 + fullPath.append(className).append("."); + fullPath.append(methodCall.getNameAsString()); + + return fullPath.toString(); + } + + public static String resolveTypePath(ClassOrInterfaceType type) { + String typeName = type.getNameAsString(); + + Optional cu = type.findAncestor(CompilationUnit.class); + if (cu.isPresent()) { + // 尝试从导入声明中查找匹配 + Optional importedPath = findMatchingImport(cu.get(), typeName); + if (importedPath.isPresent()) { + return importedPath.get(); + } + + // 如果没有找到匹配的导入，检查是否在同一包中 + Optional currentPackage = getCurrentPackage(cu.get()); + if (currentPackage.isPresent()) { + return currentPackage.get() + "." + typeName; + } + } + + // 如果无法解析，返回原始类型名称 + return typeName; + } + + private static Optional findMatchingImport(CompilationUnit cu, String typeName) { + return cu.getImports().stream() + .filter(importDecl -> !importDecl.isAsterisk() && importDecl.getNameAsString().endsWith("." + typeName)) + .map(ImportDeclaration::getNameAsString) + .findFirst(); + } + + private static Optional getCurrentPackage(CompilationUnit cu) { + return cu.getPackageDeclaration().map(pd -> pd.getNameAsString()); + } + + /** + * 判断方法调用是否为日志调用 + * + * @param n 方法调用表达式 + * @return 如果方法调用是日志调用则返回true，否则返回false + */ + private boolean isLogCall(MethodCallExpr n) { + if (!n.getScope().isPresent()) { + return false; + } + String scope = n.getScope().get().toString(); + String method = n.getNameAsString(); + return scope.equals("log") && + (method.equals("trace") || method.equals("debug") || method.equals("info") || + method.equals("warn") || method.equals("error")); + } + + + @Override + public void visit(MethodDeclaration n, Void arg) { + super.visit(n, arg); + if (n.findAncestor(ClassOrInterfaceDeclaration.class).isEmpty()) { + return; + } + // 处理方法调用 + processMethodCalls(n); + + } + + + // 处理方法调用 + private void processMethodCalls(MethodDeclaration n) { + n.findAll(MethodCallExpr.class).forEach(methodCall -> { + Map callParams = new HashMap<>(); + callParams.put("callerName", getFullMethodName(n)); + callParams.put("calleeName", getFullMethodPath(methodCall)); + + // 检查 callerName 和 calleeName 是否都存在 + Result result = session.run("MATCH (caller:Method {name: $callerName}), (callee:Method {name: $calleeName}) " + + "RETURN caller, callee", callParams); + + //只有两个业务method,才有必要创建这个边 + if (result.hasNext()) { + // 创建 CALLS 关系 + session.run("MATCH (caller:Method {name: $callerName}) " + + "MATCH (callee:Method {name: $calleeName}) " + + "MERGE (caller)-[:CALLS]->(callee)", + callParams); + } + }); + } + + + // 处理注释 + private void processComments(MethodDeclaration n) { + for (Comment comment : n.getAllContainedComments()) { + createCommentNode(comment, n); + } + + Optional optional = n.getJavadocComment(); + if (optional.isPresent()) { + createCommentNode(optional.get(), n); + } + + Optional commentOptional = n.getComment(); + if (commentOptional.isPresent()) { + createCommentNode(commentOptional.get(), n); + } + } + + private void createCommentNode(Comment comment, MethodDeclaration n) { + Map commentParams = new HashMap<>(); + commentParams.put("text", comment.getContent()); + commentParams.put("text_vector", new float[]{}); // 替换为实际的文本向量 + + session.run("MERGE (comment:Comment {text: $text, text_vector: $text_vector})", commentParams); + + // 创建 DOCUMENTS 关系 (Comment -[:DOCUMENTS]-> Method) + Map documentsParams = new HashMap<>(); + documentsParams.put("commentText", comment.getContent()); + documentsParams.put("methodName", getFullMethodName(n)); + documentsParams.put("methodSignature", n.getSignature().asString()); + session.run("MATCH (comment:Comment {text: $commentText}) " + + "MATCH (m:Method {name: $methodName, signature: $methodSignature}) " + + "MERGE (comment)-[:DOCUMENTS]->(m)", + documentsParams); + } + + } + +} diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java index 440a1be06..d2b6f63c2 100644 --- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java @@ -3,26 +3,12 @@ import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; import org.junit.Test; -import org.neo4j.driver.Record; -import org.neo4j.driver.Result; -import org.neo4j.driver.Session; -import org.neo4j.driver.internal.InternalRecord; -import org.neo4j.driver.internal.InternalResult; -import org.neo4j.driver.internal.InternalSession; import run.mone.neo4j.BotCall; import run.mone.neo4j.MoneCodeParser; -import run.mone.neo4j.test.MoneCodeParserTest; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; +import run.mone.neo4j.MoneMethodCodeParser; + import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -47,6 +33,12 @@ public void testWriteCatServiceToNeo4j() { // new MoneCodeParser().setPassword(System.getenv("password")).writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java"); } + + @Test + public void testParserMethod() { + new MoneMethodCodeParser().setPassword(System.getenv("password")).writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m"); + } + @Test public void test1() { new MoneCodeParser().queryEntityClasses(); diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/cat/CatService.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/cat/CatService.java index 494c6776c..492be9300 100644 --- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/cat/CatService.java +++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/cat/CatService.java @@ -16,7 +16,6 @@ public class CatService { private HashMap data = new HashMap<>(); - //获取小猫的数量 //获取小猫的数量 public int getCatCount() { log.info("abc");