From 65ebcb8e127794a5a129988d0be38b19ef575851 Mon Sep 17 00:00:00 2001
From: wtt <1136220284@qq.com>
Date: Mon, 11 Dec 2023 11:00:42 +0800
Subject: [PATCH 1/2] feat: update file listening create file bug
---
jcommon/file/pom.xml | 13 +++++--------
.../main/java/com/xiaomi/mone/file/LogFile2.java | 10 ++++++++++
.../file/listener/DefaultMonitorListener.java | 8 +++++---
.../mone/file/listener/OzHeraReadListener.java | 1 +
.../xiaomi/mone/file/ozhera/HeraFileMonitor.java | 16 ++++++++++------
5 files changed, 31 insertions(+), 17 deletions(-)
diff --git a/jcommon/file/pom.xml b/jcommon/file/pom.xml
index 3a9b97364..8f4a84f9a 100644
--- a/jcommon/file/pom.xml
+++ b/jcommon/file/pom.xml
@@ -9,7 +9,7 @@
1.4-jdk20-SNAPSHOT
file
-
+ 1.4-jdk21-SNAPSHOT
@@ -26,16 +26,13 @@
+ org.apache.maven.plugins
maven-compiler-plugin
3.11.0
-
- --add-modules=jdk.incubator.concurrent
- --enable-preview
-
- 20
-
- 20
+ 21
+
+ 21
diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile2.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile2.java
index 83e12ca20..a242d580b 100644
--- a/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile2.java
+++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile2.java
@@ -79,6 +79,15 @@ public LogFile2(String file) {
this.pointer = readPointer();
}
+ public LogFile2(String file, long pointer, long lineNumber) {
+ this.file = file;
+ File f = new File(this.file);
+ this.fileKey = FileUtils.fileKey(f);
+ this.md5 = md5(file);
+ this.pointer = pointer;
+ this.lineNumber = lineNumber;
+ }
+
public LogFile2(String file, ReadListener listener, long pointer, long lineNumber) {
this.file = file;
@@ -114,6 +123,7 @@ public void readLine() throws IOException {
} catch (Exception e) {
log.error("file.length() IOException, file:{}", this.file, e);
}
+ log.info("rel open file:{},pointer:{}", file, this.pointer);
raf.seek(pointer);
while (true) {
diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/listener/DefaultMonitorListener.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/listener/DefaultMonitorListener.java
index 5592c651f..7681e3fa9 100644
--- a/jcommon/file/src/main/java/com/xiaomi/mone/file/listener/DefaultMonitorListener.java
+++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/listener/DefaultMonitorListener.java
@@ -38,7 +38,7 @@ public void onEvent(FileEvent event) {
LogFile2 logFile = new LogFile2(event.getFileName());
pool.submit(() -> {
logFile.setListener(new OzHeraReadListener(monitor, logFile, consumer));
- SafeRun.run(() -> logFile.readLine());
+ SafeRun.run(logFile::readLine);
});
}
@@ -58,10 +58,12 @@ public void onEvent(FileEvent event) {
if (event.getType().equals(EventType.create)) {
log.info("create:{}", event.getFileName());
- LogFile2 logFile = new LogFile2(event.getFileName());
+
+// LogFile2 logFile = new LogFile2(event.getFileName());
+ LogFile2 logFile = new LogFile2(event.getFileName(), 0, 0);
pool.submit(() -> {
logFile.setListener(new OzHeraReadListener(monitor, logFile, consumer));
- SafeRun.run(() -> logFile.readLine());
+ SafeRun.run(logFile::readLine);
});
}
}
diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/listener/OzHeraReadListener.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/listener/OzHeraReadListener.java
index 1e8eca36a..687db6f05 100644
--- a/jcommon/file/src/main/java/com/xiaomi/mone/file/listener/OzHeraReadListener.java
+++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/listener/OzHeraReadListener.java
@@ -49,6 +49,7 @@ public boolean isBreak(String line) {
if (null == line) {
HeraFile f = monitor.getMap().get(logFile.getFileKey());
if (null == f || f.getState().get() == 1) {
+ log.info("file isBreak,file:{},f:{}", logFile.getFile(), f);
return true;
}
}
diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFileMonitor.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFileMonitor.java
index 124a52293..b65243289 100644
--- a/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFileMonitor.java
+++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/ozhera/HeraFileMonitor.java
@@ -92,7 +92,6 @@ public void reg(String path, Predicate predicate) throws IOException, In
if (!predicate.test(filePath) || modifiedFile.getFileName().toString().startsWith(".")) {
continue;
}
- log.debug("epoll result,path:{}", event.kind() + filePath);
HeraFile hfile = fileMap.get(filePath);
if (event.kind() == StandardWatchEventKinds.ENTRY_MODIFY) {
@@ -111,17 +110,20 @@ public void reg(String path, Predicate predicate) throws IOException, In
}
if (event.kind() == StandardWatchEventKinds.ENTRY_CREATE) {
- File file = new File(path + "" + modifiedFile.getFileName());
+ File file = new File(filePath);
Object k = FileUtils.fileKey(file);
+
+ log.info("ENTRY_CREATE filePath:{},fileKey:{}", filePath, k);
+ HeraFile hf = HeraFile.builder().file(file).fileKey(k).fileName(filePath).build();
+ map.putIfAbsent(k, hf);
+ fileMap.put(filePath, hf);
+
if (map.containsKey(k)) {
log.info("change name " + map.get(k) + "--->" + file);
listener.onEvent(FileEvent.builder().fileKey(k).type(EventType.rename).build());
} else {
listener.onEvent(FileEvent.builder().type(EventType.create).fileName(file.getPath()).build());
}
- HeraFile hf = HeraFile.builder().file(file).fileKey(k).fileName(filePath).build();
- map.putIfAbsent(k, hf);
- fileMap.put(filePath, hf);
}
}
key.reset();
@@ -136,9 +138,9 @@ private HeraFile initFile(File it) {
if (name.startsWith(".")) {
return null;
}
- Object fileKey = FileUtils.fileKey(it);
lock.lock();
try {
+ Object fileKey = FileUtils.fileKey(it);
if (map.containsKey(fileKey)) {
return map.get(fileKey);
}
@@ -152,6 +154,8 @@ private HeraFile initFile(File it) {
fileMap.put(hf.getFileName(), hf);
this.listener.onEvent(FileEvent.builder().pointer(pointer).type(EventType.init).fileName(hf.getFileName()).build());
return hf;
+ } catch (Exception e) {
+ log.error("init file error,fileName:{}", name, e);
} finally {
lock.unlock();
}
From 7c4401e1db636fffb5be9b76d58a315fc7638cfb Mon Sep 17 00:00:00 2001
From: wtt <1136220284@qq.com>
Date: Wed, 13 Dec 2023 16:31:30 +0800
Subject: [PATCH 2/2] refactor: update file monitor
---
.../demo/mydemo/DefaultDemoInterface.java | 17 ++++++++
.../docean/test/demo/mydemo/DemoCall.java | 4 ++
.../test/demo/mydemo/DemoInterface.java | 11 ++++++
.../java/com/xiaomi/mone/file/LogFile2.java | 14 ++++---
.../main/java/com/xiaomi/mone/file/MLog.java | 39 ++++++++++++++-----
.../com/xiaomi/mone/file/ReadListener.java | 3 ++
.../com/xiaomi/mone/file/common/FileInfo.java | 2 +
.../mone/file/common/FileInfoCache.java | 28 ++++++++++---
.../file/listener/DefaultMonitorListener.java | 15 ++++++-
.../file/listener/OzHeraReadListener.java | 5 +++
.../mone/file/ozhera/HeraFileMonitor.java | 12 +++---
jcommon/pom.xml | 2 +-
12 files changed, 122 insertions(+), 30 deletions(-)
create mode 100644 jcommon/docean/src/test/java/com/xiaomi/youpin/docean/test/demo/mydemo/DefaultDemoInterface.java
create mode 100644 jcommon/docean/src/test/java/com/xiaomi/youpin/docean/test/demo/mydemo/DemoInterface.java
diff --git a/jcommon/docean/src/test/java/com/xiaomi/youpin/docean/test/demo/mydemo/DefaultDemoInterface.java b/jcommon/docean/src/test/java/com/xiaomi/youpin/docean/test/demo/mydemo/DefaultDemoInterface.java
new file mode 100644
index 000000000..81cde2590
--- /dev/null
+++ b/jcommon/docean/src/test/java/com/xiaomi/youpin/docean/test/demo/mydemo/DefaultDemoInterface.java
@@ -0,0 +1,17 @@
+package com.xiaomi.youpin.docean.test.demo.mydemo;
+
+import com.xiaomi.youpin.docean.anno.Component;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2023/11/20 10:19
+ */
+@Component
+public class DefaultDemoInterface implements DemoInterface {
+ @Override
+ public String hi() {
+ return "test";
+ }
+}
diff --git a/jcommon/docean/src/test/java/com/xiaomi/youpin/docean/test/demo/mydemo/DemoCall.java b/jcommon/docean/src/test/java/com/xiaomi/youpin/docean/test/demo/mydemo/DemoCall.java
index 30d76f925..fe00af668 100644
--- a/jcommon/docean/src/test/java/com/xiaomi/youpin/docean/test/demo/mydemo/DemoCall.java
+++ b/jcommon/docean/src/test/java/com/xiaomi/youpin/docean/test/demo/mydemo/DemoCall.java
@@ -23,6 +23,9 @@ public class DemoCall {
@Value("$val")
private String val;
+ @Resource
+ private DemoInterface demoInterface;
+
//This is just an interface, but if it only has one implementation class, then Ioc will automatically find this unique implementation class.
@Resource
@@ -31,6 +34,7 @@ public class DemoCall {
public String hi() {
+ System.out.println(demoInterface.hi());
return demo.hi();
}
diff --git a/jcommon/docean/src/test/java/com/xiaomi/youpin/docean/test/demo/mydemo/DemoInterface.java b/jcommon/docean/src/test/java/com/xiaomi/youpin/docean/test/demo/mydemo/DemoInterface.java
new file mode 100644
index 000000000..270b274a4
--- /dev/null
+++ b/jcommon/docean/src/test/java/com/xiaomi/youpin/docean/test/demo/mydemo/DemoInterface.java
@@ -0,0 +1,11 @@
+package com.xiaomi.youpin.docean.test.demo.mydemo;
+
+/**
+ * @author goodjava@qq.com
+ * @date 2023/11/18 14:50
+ */
+public interface DemoInterface {
+
+ String hi();
+
+}
diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile2.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile2.java
index a242d580b..27a921211 100644
--- a/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile2.java
+++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/LogFile2.java
@@ -46,7 +46,7 @@ public class LogFile2 implements ILogFile {
private int beforePointerHashCode;
@Setter
- private long pointer;
+ private volatile long pointer;
//行号
private long lineNumber;
@@ -174,7 +174,6 @@ public void readLine() throws IOException {
continue;
}
-
try {
pointer = raf.getFilePointer();
maxPointer = raf.length();
@@ -194,8 +193,8 @@ public void readLine() throws IOException {
}
raf.close();
if (stop) {
- log.info("stop:{}", this.file);
- FileInfoCache.ins().put(this.fileKey.toString(), FileInfo.builder().pointer(this.pointer).build());
+ log.info("stop:{},pointer:{},fileKey:{}", this.file, this.pointer, this.fileKey);
+ FileInfoCache.ins().put(this.fileKey.toString(), FileInfo.builder().pointer(this.pointer).fileName(this.file).build());
break;
}
}
@@ -245,11 +244,16 @@ private boolean contentHasCutting(String line) throws IOException {
return false;
}
+ public void saveProgress() {
+ if (!stop) {
+ FileInfoCache.ins().put(this.fileKey.toString(), FileInfo.builder().pointer(this.pointer).fileName(this.file).build());
+ }
+ }
public void shutdown() {
try {
this.stop = true;
- FileInfoCache.ins().put(this.fileKey.toString(), FileInfo.builder().pointer(this.pointer).build());
+ FileInfoCache.ins().put(this.fileKey.toString(), FileInfo.builder().pointer(this.pointer).fileName(this.file).build());
} catch (Throwable ex) {
log.error(ex.getMessage());
}
diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/MLog.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/MLog.java
index bf0807914..2f83fca3c 100644
--- a/jcommon/file/src/main/java/com/xiaomi/mone/file/MLog.java
+++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/MLog.java
@@ -24,6 +24,7 @@
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
+import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -43,6 +44,8 @@ public class MLog {
*/
private Pattern customLinePattern;
+ private ReentrantLock lock = new ReentrantLock();
+
/**
* 匹配 20xx or [20xx
*/
@@ -52,8 +55,7 @@ public class MLog {
/**
* 最多聚合200行错误栈,避免queue无限增长
*/
- private static final int MAX_MERGE_LINE = 200;
- private byte[] lock = new byte[0];
+ private static final int MAX_MERGE_LINE = 400;
@Deprecated
public List append(String msg) {
@@ -61,7 +63,8 @@ public List append(String msg) {
List res = new ArrayList<>();
boolean isNew = isNew(msg);
if (isNew) {
- synchronized (lock) {
+ try {
+ lock.lock();
if (msgQueue.size() > 0) {
String e = msgQueue.stream().collect(Collectors.joining("\r\n"));
res.add(e);
@@ -69,9 +72,12 @@ public List append(String msg) {
}
//消息入队列
msgQueue.offer(msg);
+ } finally {
+ lock.unlock();
}
} else {
- synchronized (lock) {
+ try {
+ lock.lock();
//最大错误栈行数判断
if (msgQueue.size() >= MAX_MERGE_LINE) {
String e = msgQueue.stream().collect(Collectors.joining("\r\n"));
@@ -80,6 +86,8 @@ public List append(String msg) {
}
//消息入队列
msgQueue.offer(msg);
+ } finally {
+ lock.unlock();
}
}
@@ -91,16 +99,21 @@ public String append2(String msg) {
String res = null;
boolean isNew = isNew(msg);
if (isNew) {
- synchronized (lock) {
+
+ try {
+ lock.lock();
if (msgQueue.size() > 0) {
res = msgQueue.stream().collect(Collectors.joining("\r\n"));
msgQueue.clear();
}
//消息入队列
msgQueue.offer(msg);
+ } finally {
+ lock.unlock();
}
} else {
- synchronized (lock) {
+ try {
+ lock.lock();
//最大错误栈行数判断
if (msgQueue.size() >= MAX_MERGE_LINE) {
res = msgQueue.stream().collect(Collectors.joining("\r\n"));
@@ -108,6 +121,8 @@ public String append2(String msg) {
}
//消息入队列
msgQueue.offer(msg);
+ } finally {
+ lock.unlock();
}
}
@@ -122,26 +137,30 @@ public String append2(String msg) {
@Deprecated
public List takeRemainMsg() {
List res = new ArrayList<>();
- synchronized (lock) {
+ try {
+ lock.lock();
if (msgQueue.size() > 0) {
String e = msgQueue.stream().collect(Collectors.joining("\r\n"));
res.add(e);
msgQueue.clear();
}
+ } finally {
+ lock.unlock();
}
-
return res;
}
public String takeRemainMsg2() {
String res = null;
- synchronized (lock) {
+ try {
+ lock.lock();
if (msgQueue.size() > 0) {
res = msgQueue.stream().collect(Collectors.joining("\r\n"));
msgQueue.clear();
}
+ } finally {
+ lock.unlock();
}
-
return res;
}
diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/ReadListener.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/ReadListener.java
index 2f5d10ee9..0329b66bf 100644
--- a/jcommon/file/src/main/java/com/xiaomi/mone/file/ReadListener.java
+++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/ReadListener.java
@@ -34,4 +34,7 @@ default void setPointer(Object obj) {
}
+ default void saveProgress() {
+ }
+
}
diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/common/FileInfo.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/common/FileInfo.java
index 98ac9bd68..7a50f817e 100644
--- a/jcommon/file/src/main/java/com/xiaomi/mone/file/common/FileInfo.java
+++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/common/FileInfo.java
@@ -13,4 +13,6 @@ public class FileInfo {
private long pointer;
+ private String fileName;
+
}
diff --git a/jcommon/file/src/main/java/com/xiaomi/mone/file/common/FileInfoCache.java b/jcommon/file/src/main/java/com/xiaomi/mone/file/common/FileInfoCache.java
index c9b395ad9..968a5a83a 100644
--- a/jcommon/file/src/main/java/com/xiaomi/mone/file/common/FileInfoCache.java
+++ b/jcommon/file/src/main/java/com/xiaomi/mone/file/common/FileInfoCache.java
@@ -3,35 +3,40 @@
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
import lombok.Setter;
import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
import java.io.File;
+import java.io.FileWriter;
import java.lang.reflect.Type;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
/**
* @author goodjava@qq.com
* @date 2023/9/26 11:50
*/
+@Slf4j
public class FileInfoCache {
+ private Cache cache = CacheBuilder.newBuilder().maximumSize(50000).build();
- private Cache cache = CacheBuilder.newBuilder().maximumSize(10000).build();
-
- private Gson gson = new Gson();
+ private Gson gson = new GsonBuilder().setLenient().create();
@Setter
private String filePath = "/tmp/.ozhera_pointer";
+ private volatile boolean loaded = false;
+
private static final class LazyHolder {
private static final FileInfoCache ins = new FileInfoCache();
}
-
public static final FileInfoCache ins() {
return LazyHolder.ins;
}
@@ -45,15 +50,26 @@ public FileInfo get(String key) {
return cache.getIfPresent(key);
}
+ public void remove(String key) {
+ cache.invalidate(key);
+ }
+
+ public ConcurrentMap caches() {
+ return cache.asMap();
+ }
+
@SneakyThrows
public void shutdown() {
String str = gson.toJson(cache.asMap());
- Files.write(Paths.get(filePath), str.getBytes());
+ FileWriter writer = new FileWriter(filePath, false);
+ writer.append(str);
+ writer.flush();
}
@SneakyThrows
public void load() {
- if (new File(filePath).exists()) {
+ if (!loaded && new File(filePath).exists()) {
+ loaded = true;
String str = new String(Files.readAllBytes(Paths.get(filePath)));
Type typeOfT = new TypeToken