callParams = new HashMap<>();
- callParams.put("callerName", getFullMethodName(n));
- callParams.put("calleeName", getFullMethodPath(methodCall));
-
- // 创建目标方法节点(如果不存在)
- session.run("MERGE (callee:Method {name: $calleeName})", callParams);
-
- // 创建 CALLS 关系
- session.run("MATCH (caller:Method {name: $callerName}) " +
- "MATCH (callee:Method {name: $calleeName}) " +
- "MERGE (caller)-[:CALLS]->(callee)",
- callParams);
- });
- }
-
-
// 处理注释
private void processComments(MethodDeclaration n) {
for (Comment comment : n.getAllContainedComments()) {
diff --git a/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneMethodCodeParser.java b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneMethodCodeParser.java
new file mode 100644
index 000000000..05833ffde
--- /dev/null
+++ b/jcommon/ai/neo4j/src/main/java/run/mone/neo4j/MoneMethodCodeParser.java
@@ -0,0 +1,299 @@
+package run.mone.neo4j;
+
+import com.github.javaparser.JavaParser;
+import com.github.javaparser.ast.CompilationUnit;
+import com.github.javaparser.ast.ImportDeclaration;
+import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration;
+import com.github.javaparser.ast.body.FieldDeclaration;
+import com.github.javaparser.ast.body.MethodDeclaration;
+import com.github.javaparser.ast.comments.Comment;
+import com.github.javaparser.ast.comments.JavadocComment;
+import com.github.javaparser.ast.expr.FieldAccessExpr;
+import com.github.javaparser.ast.expr.MethodCallExpr;
+import com.github.javaparser.ast.type.ClassOrInterfaceType;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.neo4j.driver.*;
+
+import java.io.File;
+import java.util.*;
+
+/**
+ * @author goodjava@qq.com
+ *
+ * 建立方法调用之间的关系(call),且只建立业务调用的关系
+ */
+@Slf4j
+public class MoneMethodCodeParser {
+
+ private String NEO4J_URI = "bolt://localhost:7687";
+
+ private String NEO4J_USER = "neo4j";
+
+ private String password = "";
+
+ public MoneMethodCodeParser setPassword(String password) {
+ this.password = password;
+ return this;
+ }
+
+
+ public void writeJavaFilesToNeo4j(String directoryPath) {
+ if (new File(directoryPath).isFile()) {
+ writeToNeo4j(directoryPath);
+ return;
+ }
+
+ getJavaFilesInDirectory(directoryPath).forEach(it -> {
+ log.info("parse it:{}", it);
+ writeToNeo4j(it);
+ });
+ }
+
+ //给一个文件夹,获取里边是.java文件的列表,注意你需要递归获取(class)
+ public static List getJavaFilesInDirectory(String directoryPath) {
+ List javaFiles = new ArrayList<>();
+ File directory = new File(directoryPath);
+
+ if (directory.exists() && directory.isDirectory()) {
+ File[] files = directory.listFiles();
+ if (files != null) {
+ for (File file : files) {
+ if (file.isDirectory()) {
+ javaFiles.addAll(getJavaFilesInDirectory(file.getAbsolutePath()));
+ } else if (file.getName().endsWith(".java")) {
+ javaFiles.add(file.getAbsolutePath());
+ }
+ }
+ }
+ }
+ return javaFiles;
+ }
+
+
+ @SneakyThrows
+ private void writeToNeo4j(String filePath) {
+ //写入到neo4j中
+ // 替换成你的 Java 文件路径
+ String projectName = "MyProject";
+
+ try (Driver driver = GraphDatabase.driver(NEO4J_URI, AuthTokens.basic(NEO4J_USER, password));
+ Session session = driver.session()) {
+ // 解析 Java 文件
+ CompilationUnit cu = new JavaParser().parse(new File(filePath)).getResult().get();
+
+ // 遍历类、接口、方法等
+ cu.accept(new MoneMethodCodeParser.Visitor(session, projectName, filePath), null);
+ }
+ }
+
+
+ private static class Visitor extends com.github.javaparser.ast.visitor.VoidVisitorAdapter {
+
+ private final Session session;
+
+ public Visitor(Session session, String projectName, String filePath) {
+ this.session = session;
+ }
+
+
+ @Override
+ public void visit(ClassOrInterfaceDeclaration n, Void arg) {
+ super.visit(n, arg);
+ }
+
+
+ private String getFullMethodName(MethodDeclaration method) {
+ String packageName = method.findCompilationUnit()
+ .flatMap(cu -> cu.getPackageDeclaration())
+ .map(pd -> pd.getNameAsString())
+ .orElse("");
+ String className = method.findAncestor(com.github.javaparser.ast.body.ClassOrInterfaceDeclaration.class)
+ .map(c -> c.getNameAsString())
+ .orElse("");
+ String methodName = method.getNameAsString();
+ return packageName + "." + className + "." + methodName;
+ }
+
+ /**
+ * 获取方法调用的完整路径,包括包名、类名和方法名
+ *
+ * @param methodCall 方法调用表达式
+ * @return 方法调用的完整路径
+ */
+ public String getFullMethodPath(MethodCallExpr methodCall) {
+ StringBuilder fullPath = new StringBuilder();
+
+ // 获取包名
+ Optional cu = methodCall.findCompilationUnit();
+ if (cu.isPresent()) {
+ cu.get().getPackageDeclaration().ifPresent(pkg ->
+ fullPath.append(pkg.getNameAsString()).append(".")
+ );
+ }
+
+ // 获取类名
+ String className = methodCall.findAncestor(ClassOrInterfaceDeclaration.class)
+ .map(ClassOrInterfaceDeclaration::getNameAsString)
+ .orElse("");
+
+ // 获取方法调用的对象
+ String objectName = methodCall.getScope()
+ .map(scope -> scope.toString())
+ .orElse("");
+
+
+ //静态调用
+ if (methodCall.getScope().isPresent() && methodCall.getScope().get() instanceof FieldAccessExpr) {
+ return objectName + "." + methodCall.getNameAsString();
+ }
+
+ //lombok 的log
+ if (isLogCall(methodCall)) {
+ return objectName + "." + methodCall.getNameAsString();
+ }
+
+ // 如果对象名不为空,尝试找到它的类型
+ if (!objectName.isEmpty()) {
+ Optional field = methodCall.findAncestor(ClassOrInterfaceDeclaration.class)
+ .flatMap(classDecl -> classDecl.getFieldByName(objectName));
+
+ if (field.isPresent()) {
+ ClassOrInterfaceType type = field.get().getVariable(0).getType().asClassOrInterfaceType();
+ String v = resolveTypePath(type);
+ return v + "." + methodCall.getNameAsString();
+ }
+ }
+
+
+ // 构建完整路径
+ fullPath.append(className).append(".");
+ fullPath.append(methodCall.getNameAsString());
+
+ return fullPath.toString();
+ }
+
+ public static String resolveTypePath(ClassOrInterfaceType type) {
+ String typeName = type.getNameAsString();
+
+ Optional cu = type.findAncestor(CompilationUnit.class);
+ if (cu.isPresent()) {
+ // 尝试从导入声明中查找匹配
+ Optional importedPath = findMatchingImport(cu.get(), typeName);
+ if (importedPath.isPresent()) {
+ return importedPath.get();
+ }
+
+ // 如果没有找到匹配的导入,检查是否在同一包中
+ Optional currentPackage = getCurrentPackage(cu.get());
+ if (currentPackage.isPresent()) {
+ return currentPackage.get() + "." + typeName;
+ }
+ }
+
+ // 如果无法解析,返回原始类型名称
+ return typeName;
+ }
+
+ private static Optional findMatchingImport(CompilationUnit cu, String typeName) {
+ return cu.getImports().stream()
+ .filter(importDecl -> !importDecl.isAsterisk() && importDecl.getNameAsString().endsWith("." + typeName))
+ .map(ImportDeclaration::getNameAsString)
+ .findFirst();
+ }
+
+ private static Optional getCurrentPackage(CompilationUnit cu) {
+ return cu.getPackageDeclaration().map(pd -> pd.getNameAsString());
+ }
+
+ /**
+ * 判断方法调用是否为日志调用
+ *
+ * @param n 方法调用表达式
+ * @return 如果方法调用是日志调用则返回true,否则返回false
+ */
+ private boolean isLogCall(MethodCallExpr n) {
+ if (!n.getScope().isPresent()) {
+ return false;
+ }
+ String scope = n.getScope().get().toString();
+ String method = n.getNameAsString();
+ return scope.equals("log") &&
+ (method.equals("trace") || method.equals("debug") || method.equals("info") ||
+ method.equals("warn") || method.equals("error"));
+ }
+
+
+ @Override
+ public void visit(MethodDeclaration n, Void arg) {
+ super.visit(n, arg);
+ if (n.findAncestor(ClassOrInterfaceDeclaration.class).isEmpty()) {
+ return;
+ }
+ // 处理方法调用
+ processMethodCalls(n);
+
+ }
+
+
+ // 处理方法调用
+ private void processMethodCalls(MethodDeclaration n) {
+ n.findAll(MethodCallExpr.class).forEach(methodCall -> {
+ Map callParams = new HashMap<>();
+ callParams.put("callerName", getFullMethodName(n));
+ callParams.put("calleeName", getFullMethodPath(methodCall));
+
+ // 检查 callerName 和 calleeName 是否都存在
+ Result result = session.run("MATCH (caller:Method {name: $callerName}), (callee:Method {name: $calleeName}) " +
+ "RETURN caller, callee", callParams);
+
+ //只有两个业务method,才有必要创建这个边
+ if (result.hasNext()) {
+ // 创建 CALLS 关系
+ session.run("MATCH (caller:Method {name: $callerName}) " +
+ "MATCH (callee:Method {name: $calleeName}) " +
+ "MERGE (caller)-[:CALLS]->(callee)",
+ callParams);
+ }
+ });
+ }
+
+
+ // 处理注释
+ private void processComments(MethodDeclaration n) {
+ for (Comment comment : n.getAllContainedComments()) {
+ createCommentNode(comment, n);
+ }
+
+ Optional optional = n.getJavadocComment();
+ if (optional.isPresent()) {
+ createCommentNode(optional.get(), n);
+ }
+
+ Optional commentOptional = n.getComment();
+ if (commentOptional.isPresent()) {
+ createCommentNode(commentOptional.get(), n);
+ }
+ }
+
+ private void createCommentNode(Comment comment, MethodDeclaration n) {
+ Map commentParams = new HashMap<>();
+ commentParams.put("text", comment.getContent());
+ commentParams.put("text_vector", new float[]{}); // 替换为实际的文本向量
+
+ session.run("MERGE (comment:Comment {text: $text, text_vector: $text_vector})", commentParams);
+
+ // 创建 DOCUMENTS 关系 (Comment -[:DOCUMENTS]-> Method)
+ Map documentsParams = new HashMap<>();
+ documentsParams.put("commentText", comment.getContent());
+ documentsParams.put("methodName", getFullMethodName(n));
+ documentsParams.put("methodSignature", n.getSignature().asString());
+ session.run("MATCH (comment:Comment {text: $commentText}) " +
+ "MATCH (m:Method {name: $methodName, signature: $methodSignature}) " +
+ "MERGE (comment)-[:DOCUMENTS]->(m)",
+ documentsParams);
+ }
+
+ }
+
+}
diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java
index 440a1be06..d2b6f63c2 100644
--- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java
+++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/MoneCodeParserTest.java
@@ -3,26 +3,12 @@
import com.google.common.collect.ImmutableMap;
import com.google.gson.Gson;
import org.junit.Test;
-import org.neo4j.driver.Record;
-import org.neo4j.driver.Result;
-import org.neo4j.driver.Session;
-import org.neo4j.driver.internal.InternalRecord;
-import org.neo4j.driver.internal.InternalResult;
-import org.neo4j.driver.internal.InternalSession;
import run.mone.neo4j.BotCall;
import run.mone.neo4j.MoneCodeParser;
-import run.mone.neo4j.test.MoneCodeParserTest;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.HashMap;
+import run.mone.neo4j.MoneMethodCodeParser;
+
import java.util.List;
import java.util.Map;
-import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
@@ -47,6 +33,12 @@ public void testWriteCatServiceToNeo4j() {
// new MoneCodeParser().setPassword(System.getenv("password")).writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/PersonService.java");
}
+
+ @Test
+ public void testParserMethod() {
+ new MoneMethodCodeParser().setPassword(System.getenv("password")).writeJavaFilesToNeo4j("/Users/zhangzhiyong/IdeaProjects/goodjava/mone/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m");
+ }
+
@Test
public void test1() {
new MoneCodeParser().queryEntityClasses();
diff --git a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/cat/CatService.java b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/cat/CatService.java
index 494c6776c..492be9300 100644
--- a/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/cat/CatService.java
+++ b/jcommon/ai/neo4j/src/test/java/run/mone/neo4j/test/m/cat/CatService.java
@@ -16,7 +16,6 @@ public class CatService {
private HashMap data = new HashMap<>();
- //获取小猫的数量
//获取小猫的数量
public int getCatCount() {
log.info("abc");
From bd03aa7128dd71d6309f7918ee53027974179bf6 Mon Sep 17 00:00:00 2001
From: wtt <30461027+wtt40122@users.noreply.github.com>
Date: Thu, 29 Aug 2024 20:03:18 +0800
Subject: [PATCH 10/11] refactor: update file coll (#889)
* refactor: update es and upgrade file
* fix: Fixed the issue of reading a large number of files OOM and upgraded the ES version
* refactor: update compile jdk version
* refactor: update file coll
---
.../main/java/com/xiaomi/mone/file/LogFile2.java | 1 +
.../java/com/xiaomi/mone/file/ReadListener.java | 3 +++
.../mone/file/listener/OzHeraReadListener.java | 8 ++++++++
.../com/xiaomi/mone/file/ozhera/HeraFile.java | 3 +++
.../xiaomi/mone/file/ozhera/HeraFileMonitor.java | 12 +++++++++---
.../java/com/xiaomi/mone/file/LogFileTest.java | 16 ++++++++++++++--
6 files changed, 38 insertions(+), 5 deletions(-)
diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile2.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile2.java
index fbbd399c4..0381a698a 100644
--- a/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile2.java
+++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile2.java
@@ -190,6 +190,7 @@ public void readLine() throws IOException {
readResult.setFilePathName(file);
readResult.setLineNumber(++lineNumber);
ReadEvent event = new ReadEvent(readResult);
+ listener.setReadTime();
listener.onEvent(event);
}
diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/ReadListener.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/ReadListener.java
index 0329b66bf..75e92b157 100644
--- a/jcommon/file/src/main/java/com/xiaomi/mone/file/ReadListener.java
+++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/ReadListener.java
@@ -37,4 +37,7 @@ default void setPointer(Object obj) {
default void saveProgress() {
}
+ default void setReadTime() {
+ }
+
}
diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/listener/OzHeraReadListener.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/listener/OzHeraReadListener.java
index e2b3c2d42..43fa64876 100644
--- a/jcommon/file/src/main/java/com/xiaomi/mone/file/listener/OzHeraReadListener.java
+++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/listener/OzHeraReadListener.java
@@ -72,4 +72,12 @@ public void setPointer(Object obj) {
}
}
}
+
+ @Override
+ public void setReadTime() {
+ HeraFile f = monitor.getFileMap().get(logFile.getFileKey());
+ if (null != f) {
+ f.getReadTime().set(System.currentTimeMillis());
+ }
+ }
}
diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFile.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFile.java
index 8acb1a989..fbbde66fe 100644
--- a/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFile.java
+++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFile.java
@@ -30,4 +30,7 @@ public class HeraFile {
@Builder.Default
private AtomicLong utime = new AtomicLong(System.currentTimeMillis());
+
+ @Builder.Default
+ private AtomicLong readTime = new AtomicLong(System.currentTimeMillis());
}
diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFileMonitor.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFileMonitor.java
index c8a2d1e84..0f7c7d385 100644
--- a/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFileMonitor.java
+++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFileMonitor.java
@@ -34,6 +34,7 @@ public class HeraFileMonitor {
@Getter
private ConcurrentHashMap