diff --git a/ozhera-app/app-api/pom.xml b/ozhera-app/app-api/pom.xml index 27f7cdc66..80ec1d8f6 100644 --- a/ozhera-app/app-api/pom.xml +++ b/ozhera-app/app-api/pom.xml @@ -11,12 +11,12 @@ 4.0.0 app-api - ${hera-app.release.version} org.apache.ozhera app-common + 2.0.0-SNAPSHOT diff --git a/ozhera-intelligence/ozhera-intelligence-service/pom.xml b/ozhera-intelligence/ozhera-intelligence-service/pom.xml index 2ed57496f..4d8799f15 100644 --- a/ozhera-intelligence/ozhera-intelligence-service/pom.xml +++ b/ozhera-intelligence/ozhera-intelligence-service/pom.xml @@ -56,8 +56,9 @@ - org.apache.ozhera + run.mone trace-etl-domain + 1.0.1-jdk21 diff --git a/ozhera-intelligence/pom.xml b/ozhera-intelligence/pom.xml index 3294c4a29..a09432ce4 100644 --- a/ozhera-intelligence/pom.xml +++ b/ozhera-intelligence/pom.xml @@ -5,7 +5,7 @@ 4.0.0 org.apache.ozhera - ozhera + ozhera-parent 2.0.0-SNAPSHOT diff --git a/ozhera-log/log-agent-server/pom.xml b/ozhera-log/log-agent-server/pom.xml index 773cace5d..565476889 100644 --- a/ozhera-log/log-agent-server/pom.xml +++ b/ozhera-log/log-agent-server/pom.xml @@ -3,9 +3,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + org.apache.ozhera ozhera-log - run.mone - 1.4.0-jdk21 + 2.0.0-SNAPSHOT 4.0.0 @@ -153,7 +153,7 @@ false - com.xiaomi.mone.log.server.LogAgentServerBootstrap + org.apache.ozhera.log.server.LogAgentServerBootstrap diff --git a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/LogAgentServerBootstrap.java b/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/LogAgentServerBootstrap.java deleted file mode 100644 index 00a82c7d4..000000000 --- a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/LogAgentServerBootstrap.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.server; - -import com.google.common.collect.Lists; -import com.xiaomi.data.push.rpc.RpcCmd; -import com.xiaomi.data.push.rpc.RpcServer; -import com.xiaomi.data.push.rpc.common.Pair; -import com.xiaomi.mone.log.common.Config; -import com.xiaomi.mone.log.common.Constant; -import com.xiaomi.mone.log.server.porcessor.AgentCollectProgressProcessor; -import com.xiaomi.mone.log.server.porcessor.AgentConfigProcessor; -import com.xiaomi.mone.log.server.porcessor.PingProcessor; -import com.xiaomi.youpin.docean.Ioc; -import lombok.extern.slf4j.Slf4j; - -import java.io.IOException; - -import static com.xiaomi.mone.log.server.common.ServerConstant.SERVER_PORT; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/12/5 11:24 - */ -@Slf4j -public class LogAgentServerBootstrap { - - public static void main(String[] args) throws IOException { - String nacosAddr = Config.ins().get("nacosAddr", ""); - String serverName = Config.ins().get("serverName", ""); - log.info("nacos:{} name:{}", nacosAddr, serverName); - RpcServer rpcServer = new RpcServer(nacosAddr, serverName); - rpcServer.setListenPort(SERVER_PORT); - //Register the processor - rpcServer.setProcessorList(Lists.newArrayList( - new Pair<>(RpcCmd.pingReq, new PingProcessor()), - new Pair<>(Constant.RPCCMD_AGENT_CODE, new AgentCollectProgressProcessor()), - new Pair<>(Constant.RPCCMD_AGENT_CONFIG_CODE, new AgentConfigProcessor()) - )); - rpcServer.init(); - rpcServer.start(); - - Ioc.ins().putBean(rpcServer); - Ioc.ins().init("com.xiaomi.mone", "com.xiaomi.youpin"); - log.info("log server start finish"); - } -} diff --git a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/common/ServerConstant.java b/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/common/ServerConstant.java deleted file mode 100644 index 8b6070e77..000000000 --- a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/common/ServerConstant.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.server.common; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/10/9 19:34 - */ -public class ServerConstant { - public static final Integer SERVER_PORT = 9899; -} diff --git a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/common/Version.java b/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/common/Version.java deleted file mode 100644 index 841e59f45..000000000 --- a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/common/Version.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.server.common; - -import java.io.Serializable; - -/** - * @author wangtao - */ -public class Version implements Serializable { - - - @Override - public String toString() { - return "log-agent-server:2022-12-05:0.0.2"; - } -} diff --git a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/porcessor/AgentCollectProgressProcessor.java b/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/porcessor/AgentCollectProgressProcessor.java deleted file mode 100644 index bc74b354f..000000000 --- a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/porcessor/AgentCollectProgressProcessor.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.server.porcessor; - -import com.xiaomi.data.push.rpc.netty.NettyRequestProcessor; -import com.xiaomi.data.push.rpc.protocol.RemotingCommand; -import com.xiaomi.mone.log.api.model.vo.UpdateLogProcessCmd; -import com.xiaomi.mone.log.common.Constant; -import com.xiaomi.mone.log.server.common.Version; -import com.xiaomi.mone.log.server.service.DefaultLogProcessCollector; -import com.xiaomi.youpin.docean.Ioc; -import com.xiaomi.youpin.docean.anno.Component; -import io.netty.channel.ChannelHandlerContext; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Resource; -import java.nio.charset.StandardCharsets; - -import static com.xiaomi.mone.log.common.Constant.GSON; - -/** - * @author wtt - * @version 1.0 - * @description The receiver that communicates with the agent ---- the acquisition progress - * @date 2021/8/19 15:32 - */ -@Slf4j -@Component -public class AgentCollectProgressProcessor implements NettyRequestProcessor { - - @Resource - DefaultLogProcessCollector processService; - - private static Version version = new Version(); - - @Override - public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws Exception { - log.debug("received a message from the agent"); - RemotingCommand response = RemotingCommand.createResponseCommand(Constant.RPCCMD_AGENT_CODE); - String body = new String(request.getBody(), StandardCharsets.UTF_8); - UpdateLogProcessCmd cmd = GSON.fromJson(body, UpdateLogProcessCmd.class); - log.debug("a request from the client sent by the agent:{}", cmd.getIp()); - if (null == processService && Ioc.ins().containsBean(DefaultLogProcessCollector.class.getCanonicalName())) { - processService = Ioc.ins().getBean(DefaultLogProcessCollector.class); - } - if (null != processService) { - processService.collectLogProcess(cmd); - } - response.setBody(version.toString().getBytes()); - response.setBody(Constant.SUCCESS_MESSAGE.getBytes()); - return response; - } - - @Override - public boolean rejectRequest() { - return false; - } -} diff --git a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/porcessor/AgentConfigProcessor.java b/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/porcessor/AgentConfigProcessor.java deleted file mode 100644 index 6b4e41d69..000000000 --- a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/porcessor/AgentConfigProcessor.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.xiaomi.mone.log.server.porcessor; - -import com.xiaomi.data.push.rpc.netty.NettyRequestProcessor; -import com.xiaomi.data.push.rpc.protocol.RemotingCommand; -import com.xiaomi.mone.log.api.model.meta.LogCollectMeta; -import com.xiaomi.mone.log.api.service.AgentConfigService; -import com.xiaomi.mone.log.common.Constant; -import com.xiaomi.mone.log.server.service.DefaultAgentConfigAcquirer; -import com.xiaomi.youpin.docean.Ioc; -import io.netty.channel.ChannelHandlerContext; -import lombok.extern.slf4j.Slf4j; - -import static com.xiaomi.mone.log.common.Constant.GSON; - -/** - * @author wtt - * @version 1.0 - * @description The receiver that communicates with the agent ---- the agent starts to get the configuration - * @date 2021/8/19 15:32 - */ -@Slf4j -public class AgentConfigProcessor implements NettyRequestProcessor { - - @Override - public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws Exception { - RemotingCommand response = RemotingCommand.createResponseCommand(Constant.RPCCMD_AGENT_CONFIG_CODE); - String ip = new String(request.getBody()); - log.info("agent start get metadata config，agent ip:{}", ip); - - AgentConfigService agentConfigService = Ioc.ins().getBean(DefaultAgentConfigAcquirer.class); - - LogCollectMeta logCollectMeta = agentConfigService.getLogCollectMetaFromManager(ip); - String responseInfo = GSON.toJson(logCollectMeta); - log.info("agent start get metadata config info:{}", responseInfo); - response.setBody(responseInfo.getBytes()); - return response; - } - - @Override - public boolean rejectRequest() { - return false; - } -} diff --git a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/porcessor/PingProcessor.java b/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/porcessor/PingProcessor.java deleted file mode 100644 index 0edc8af9d..000000000 --- a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/porcessor/PingProcessor.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.server.porcessor; - -import com.xiaomi.data.push.context.AgentContext; -import com.xiaomi.data.push.rpc.RpcCmd; -import com.xiaomi.data.push.rpc.common.RemotingHelper; -import com.xiaomi.data.push.rpc.netty.AgentChannel; -import com.xiaomi.data.push.rpc.netty.NettyRequestProcessor; -import com.xiaomi.data.push.rpc.protocol.RemotingCommand; -import com.xiaomi.mone.log.api.model.meta.AppLogMeta; -import com.xiaomi.mone.log.api.model.vo.PingReq; -import com.xiaomi.mone.log.server.common.Version; -import com.xiaomi.mone.log.utils.NetUtil; -import io.netty.channel.ChannelHandlerContext; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ThreadLocalRandom; - -import static com.xiaomi.mone.log.common.Constant.GSON; -import static com.xiaomi.mone.log.server.common.ServerConstant.SERVER_PORT; - -/** - * @Author goodjava@qq.com - * @Date 2021/6/24 11:38 - */ -@Slf4j -public class PingProcessor implements NettyRequestProcessor { - - public static Map agentHeartTimeStampMap = new ConcurrentHashMap<>(1024); - - private static Version version = new Version(); - - private DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss SSS"); - - @Override - public RemotingCommand processRequest(ChannelHandlerContext channelHandlerContext, RemotingCommand remotingCommand) { - final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(channelHandlerContext.channel()); - RemotingCommand response = RemotingCommand.createResponseCommand(RpcCmd.pingRes); - String body = new String(remotingCommand.getBody()); - PingReq pr = GSON.fromJson(body, PingReq.class); - - AgentChannel ch = AgentContext.ins().map.get(remoteAddress); - if (null != ch) { - ch.setIp(pr.getIp()); - } - String requestBody = String.format("%s->%s->%s:%s->%s", version.toString(), dateTimeFormatter.format(LocalDateTime.now()), NetUtil.getLocalIp(), SERVER_PORT, remoteAddress); - response.setBody(requestBody.getBytes()); - if (null != pr && StringUtils.isNotBlank(pr.getIp())) { - agentHeartTimeStampMap.put(pr.getIp(), Instant.now().toEpochMilli()); - } - - if (pr.getMessage().equals("load")) { - AppLogMeta meta = new AppLogMeta(); - meta.setAppName("log-manager"); - meta.setAppId(ThreadLocalRandom.current().nextLong()); - response.setBody(GSON.toJson(meta).getBytes()); - } - - return response; - } - - @Override - public boolean rejectRequest() { - return false; - } -} diff --git a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/service/ApplicationShutdownHook.java b/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/service/ApplicationShutdownHook.java deleted file mode 100644 index 6bf2d1b83..000000000 --- a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/service/ApplicationShutdownHook.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.server.service; - -import com.xiaomi.data.push.rpc.RpcServer; -import com.xiaomi.youpin.docean.anno.Component; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Resource; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/10/13 14:11 - */ -@Component -@Slf4j -public class ApplicationShutdownHook { - - @Resource - private RpcServer rpcServer; - - public void init() { - addRuntimeShutdownHook(); - } - - /** - * addRuntimeShutdownHook server deregisterInstance - */ - private void addRuntimeShutdownHook() { - Runtime.getRuntime().addShutdownHook(new Thread(() -> rpcServer.shutdown())); - } - -} diff --git a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/service/DefaultAgentConfigAcquirer.java b/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/service/DefaultAgentConfigAcquirer.java deleted file mode 100644 index cde3e56aa..000000000 --- a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/service/DefaultAgentConfigAcquirer.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.xiaomi.mone.log.server.service; - -import com.xiaomi.mone.log.api.model.meta.LogCollectMeta; -import com.xiaomi.mone.log.api.service.AgentConfigService; -import com.xiaomi.youpin.docean.anno.Component; -import com.xiaomi.youpin.docean.plugin.dubbo.anno.Reference; -import lombok.extern.slf4j.Slf4j; - -/** - * @author wtt - * @version 1.0 - * @description Obtain configuration from dashboard through dubbo interface - * @date 2022/12/6 14:30 - */ -@Component -@Slf4j -public class DefaultAgentConfigAcquirer implements AgentConfigService { - - @Reference(interfaceClass = AgentConfigService.class, group = "$dubbo.group", check = false, timeout = 10000) - private AgentConfigService agentConfigService; - - @Override - public LogCollectMeta getLogCollectMetaFromManager(String ip) { - LogCollectMeta logCollectMeta = new LogCollectMeta(); - try { - logCollectMeta = agentConfigService.getLogCollectMetaFromManager(ip); - } catch (Exception e) { - log.error("getLogCollectMetaFromManager error,ip:{}", ip, e); - } - return logCollectMeta; - } -} - - diff --git a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/service/DefaultLogProcessCollector.java b/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/service/DefaultLogProcessCollector.java deleted file mode 100644 index c6a98e953..000000000 --- a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/service/DefaultLogProcessCollector.java +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.server.service; - -import cn.hutool.core.util.NumberUtil; -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.api.model.vo.AgentLogProcessDTO; -import com.xiaomi.mone.log.api.model.vo.TailLogProcessDTO; -import com.xiaomi.mone.log.api.model.vo.UpdateLogProcessCmd; -import com.xiaomi.mone.log.api.service.LogProcessCollector; -import com.xiaomi.youpin.docean.anno.Component; -import com.xiaomi.youpin.docean.plugin.dubbo.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.time.Instant; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.GSON; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/12/6 14:32 - */ -@Slf4j -@Component -@Service(interfaceClass = LogProcessCollector.class, group = "$dubbo.group", timeout = 10000) -public class DefaultLogProcessCollector implements LogProcessCollector { - - private final Map> tailProgressMap = new ConcurrentHashMap<>(256); - - private static final Integer MAX_INTERRUPT_TIME = 10; - - private static final Integer MAX_STATIC_INTERRUPT_TIME_HOUR = 4; - - private static final String PROCESS_SEPARATOR = "%"; - - @Override - public void collectLogProcess(UpdateLogProcessCmd cmd) { - log.debug("[LogProcess.updateLogProcess] cmd:{} ", cmd); - if (cmd == null || StringUtils.isEmpty(cmd.getIp())) { - return; - } - tailProgressMap.put(cmd.getIp(), cmd.getCollectList()); - } - - @Override - public List getTailLogProcess(Long tailId, String tailName, String targetIp) { - if (null == tailId || StringUtils.isBlank(tailName)) { - return new ArrayList<>(); - } - List dtoList = tailProgressMap.values().stream() - .flatMap(Collection::stream) - .filter(collectDetail -> Objects.equals(tailId.toString(), collectDetail.getTailId())) - .flatMap(collectDetail -> collectDetail.getFileProgressDetails().stream()) - .map(fileProgressDetail -> TailLogProcessDTO.builder() - .tailName(tailName) - .collectTime(fileProgressDetail.getCollectTime()) - .collectPercentage(fileProgressDetail.getCollectPercentage()) - .ip(fileProgressDetail.getConfigIp()) - .path(fileProgressDetail.getPattern()) - .fileRowNumber(fileProgressDetail.getFileRowNumber()).build()) - .filter(processDTO -> StringUtils.isNotBlank(processDTO.getIp())) - .collect(Collectors.toList()); - if (StringUtils.isNotBlank(targetIp)) { - dtoList = dtoList.stream().filter(processDTO -> Objects.equals(targetIp, processDTO.getIp())).collect(Collectors.toList()); - } - List perOneIpProgressList = Lists.newArrayList(); - try { - perOneIpProgressList = getTailLogProcessDTOS(dtoList, perOneIpProgressList); - perOneIpProgressList = filterExpireTimePath(perOneIpProgressList); - } catch (Exception e) { - log.error("getTailLogProcess error,dtoList:{}", GSON.toJson(dtoList), e); - } - return perOneIpProgressList; - } - - @Override - public List getAgentLogProcess(String ip) { - List dtoList = Lists.newArrayList(); - if (StringUtils.isEmpty(ip) || tailProgressMap.isEmpty()) { - return dtoList; - } - List collect = tailProgressMap.values().stream() - .flatMap(Collection::stream) - .collect(Collectors.toList()); - collect.stream().forEach(collectDetail -> { - try { - String appName = collectDetail.getAppName(); - if (CollectionUtils.isNotEmpty(collectDetail.getFileProgressDetails())) { - dtoList.addAll(collectDetail.getFileProgressDetails().stream() - .filter(processDTO -> StringUtils.isNotBlank(processDTO.getConfigIp())) - .filter(processDTO -> Objects.equals(ip, processDTO.getConfigIp())) - .map(fileProgressDetail -> { - AgentLogProcessDTO agentLogProcessDTO = new AgentLogProcessDTO(); - agentLogProcessDTO.setPath(fileProgressDetail.getPattern()); - agentLogProcessDTO.setFileRowNumber(fileProgressDetail.getFileRowNumber()); - agentLogProcessDTO.setPointer(fileProgressDetail.getPointer()); - agentLogProcessDTO.setFileMaxPointer(fileProgressDetail.getFileMaxPointer()); - agentLogProcessDTO.setAppName(appName); - agentLogProcessDTO.setCollectPercentage(fileProgressDetail.getCollectPercentage()); - agentLogProcessDTO.setCollectTime(fileProgressDetail.getCollectTime()); - return agentLogProcessDTO; - }).collect(Collectors.toList())); - } - } catch (Exception e) { - log.error("getAgentLogProcess error,ip:{},CollectDetail:{}", ip, GSON.toJson(collectDetail), e); - } - }); - return dtoList; - } - - @Override - public List getColProcessImperfect(Double progressRation) { - List resultList = Lists.newArrayList(); - if (null == progressRation || tailProgressMap.isEmpty()) { - return resultList; - } - resultList = tailProgressMap.values().stream().flatMap(Collection::stream) - .map(collectDetail -> { - List fileProgressDetails = collectDetail.getFileProgressDetails(); - if (CollectionUtils.isNotEmpty(fileProgressDetails)) { - List progressDetails = fileProgressDetails.stream() - .filter(fileProgressDetail -> lessThenRation(fileProgressDetail.getCollectPercentage(), progressRation)) - .filter(tailLogProcessDTO -> null != tailLogProcessDTO.getCollectTime() && - Instant.now().toEpochMilli() - tailLogProcessDTO.getCollectTime() < TimeUnit.HOURS.toMillis(MAX_STATIC_INTERRUPT_TIME_HOUR)) - .collect(Collectors.toList()); - collectDetail.setFileProgressDetails(progressDetails); - } - return collectDetail; - }) - .filter(collectDetail -> CollectionUtils.isNotEmpty(collectDetail.getFileProgressDetails())) - .collect(Collectors.toList()); - return resultList; - } - - @Override - public List getFileProcessDetailByTail(Long tailId) { - List resultList = new ArrayList<>(); - if (tailId == null) { - return resultList; - } - try { - for (List details : tailProgressMap.values()) { - for (UpdateLogProcessCmd.CollectDetail detail : details) { - if (String.valueOf(tailId).equals(detail.getTailId())) { - resultList.addAll(detail.getFileProgressDetails()); - } - } - } - } catch (Throwable t) { - log.error("getFileProcessDetailByTail error : ", t); - } - return resultList; - } - - @Override - public List getAllCollectDetail(String ip) { - return tailProgressMap.get(ip); - } - - /** - * @param source 89% - * @param targetNum 0.98 - * @return - */ - private boolean lessThenRation(String source, Double targetNum) { - try { - double sourceOrigin = Double.parseDouble(StringUtils.substringBefore(source, PROCESS_SEPARATOR)); - double sourceNum = NumberUtil.div(sourceOrigin, 100d); - return Double.valueOf(sourceNum).compareTo(targetNum) < 0; - } catch (Exception e) { - log.error("lessThenRation error,source:{},target:{}", source, targetNum, e); - } - return true; - } - - - private List getTailLogProcessDTOS(List dtoList, List perOneIpProgressList) { - if (CollectionUtils.isNotEmpty(dtoList)) { - // Go to retrieve the latest one - Map> collect = dtoList.stream() - .collect(Collectors - .groupingBy(processDTO -> - String.format("%s-%s", processDTO.getIp(), processDTO.getPath())) - ); - perOneIpProgressList = collect.keySet().stream().map(s -> { - List tailLogProcessDTOS = collect.get(s); - return tailLogProcessDTOS.stream() - .sorted(Comparator.comparing(TailLogProcessDTO::getCollectTime).reversed()) - .findFirst().get(); - }).collect(Collectors.toList()); - return perOneIpProgressList; - } - return Lists.newArrayList(); - } - - private List filterExpireTimePath(List tailLogProcessDTOS) { - return tailLogProcessDTOS.stream() - .filter(processDTO -> Objects.nonNull(processDTO.getCollectTime()) && - Instant.now().toEpochMilli() - processDTO.getCollectTime() < TimeUnit.MINUTES.toMillis(MAX_INTERRUPT_TIME)) - .collect(Collectors.toList()); - } -} diff --git a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/service/DefaultPublishConfigService.java b/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/service/DefaultPublishConfigService.java deleted file mode 100644 index 3c1ef52af..000000000 --- a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/service/DefaultPublishConfigService.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.server.service; - -import com.google.common.base.Stopwatch; -import com.google.common.collect.Lists; -import com.xiaomi.data.push.context.AgentContext; -import com.xiaomi.data.push.rpc.RpcServer; -import com.xiaomi.data.push.rpc.netty.AgentChannel; -import com.xiaomi.data.push.rpc.protocol.RemotingCommand; -import com.xiaomi.mone.log.api.model.meta.LogCollectMeta; -import com.xiaomi.mone.log.api.model.vo.LogCmd; -import com.xiaomi.mone.log.api.service.PublishConfigService; -import com.xiaomi.mone.log.utils.NetUtil; -import com.xiaomi.youpin.docean.plugin.dubbo.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; -import java.util.*; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.GSON; -import static com.xiaomi.mone.log.common.Constant.SYMBOL_COLON; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/12/6 17:48 - */ -@Slf4j -@Service(interfaceClass = PublishConfigService.class, group = "$dubbo.group", timeout = 14000) -public class DefaultPublishConfigService implements PublishConfigService { - - private static final AtomicInteger COUNT_INCR = new AtomicInteger(0); - @Resource - private RpcServer rpcServer; - - /** - * dubbo interface, the timeout period cannot be too long - * - * @param agentIp - * @param logCollectMeta - */ - @Override - public void sengConfigToAgent(String agentIp, LogCollectMeta logCollectMeta) { - int count = 1; - while (count < 4) { - Map logAgentMap = getAgentChannelMap(); - String agentCurrentIp = queryCurrentDockerAgentIP(agentIp, logAgentMap); - if (logAgentMap.containsKey(agentCurrentIp)) { - String sendStr = GSON.toJson(logCollectMeta); - if (CollectionUtils.isNotEmpty(logCollectMeta.getAppLogMetaList())) { - RemotingCommand req = RemotingCommand.createRequestCommand(LogCmd.logReq); - req.setBody(sendStr.getBytes()); - log.info("Send the configuration,agent ip:{},Configuration information:{}", agentCurrentIp, sendStr); - Stopwatch started = Stopwatch.createStarted(); - RemotingCommand res = rpcServer.sendMessage(logAgentMap.get(agentCurrentIp), req, 10000); - started.stop(); - String response = new String(res.getBody()); - log.info("The configuration is sent successfully---->{},duration：{}s,agentIp:{}", response, started.elapsed().getSeconds(), agentCurrentIp); - if (Objects.equals(response, "ok")) { - break; - } - } - } else { - log.info("The current agent IP is not connected,ip:{},configuration data:{}", agentIp, GSON.toJson(logCollectMeta)); - } - //Retry policy - Retry 4 times, sleep 500 ms each time - try { - TimeUnit.MILLISECONDS.sleep(500L); - } catch (final InterruptedException ignored) { - } - count++; - } - } - - @Override - public List getAllAgentList() { - List remoteAddress = Lists.newArrayList(); - List ipAddress = Lists.newArrayList(); - AgentContext.ins().map.entrySet().forEach(agentChannelEntry -> { - String key = agentChannelEntry.getKey(); - remoteAddress.add(key); - ipAddress.add(StringUtils.substringBefore(key, SYMBOL_COLON)); - } - ); - if (COUNT_INCR.getAndIncrement() % 200 == 0) { - log.info("The set of remote addresses of the connected agent machine is:{}", GSON.toJson(remoteAddress)); - } - return remoteAddress; - } - - private Map getAgentChannelMap() { - Map logAgentMap = new HashMap<>(); - AgentContext.ins().map.forEach((k, v) -> logAgentMap.put(StringUtils.substringBefore(k, SYMBOL_COLON), v)); - return logAgentMap; - } - - private String queryCurrentDockerAgentIP(String agentIp, Map logAgentMap) { - if (Objects.equals(agentIp, NetUtil.getLocalIp())) { - //for Docker handles the agent on the current machine - final String tempIp = agentIp; - List ipList = getAgentChannelMap().keySet() - .stream().filter(ip -> ip.startsWith("172")) - .collect(Collectors.toList()); - Optional optionalS = ipList.stream() - .filter(ip -> Objects.equals(logAgentMap.get(ip).getIp(), tempIp)) - .findFirst(); - if (optionalS.isPresent()) { - String correctIp = optionalS.get(); - log.info("origin ip:{},set agent ip:{}", agentIp, correctIp); - agentIp = correctIp; - } - } - return agentIp; - } -} diff --git a/ozhera-log/log-agent-server/src/test/java/com/xiaomi/mone/log/server/DefaultAgentConfigAcquirerTest.java b/ozhera-log/log-agent-server/src/test/java/com/xiaomi/mone/log/server/DefaultAgentConfigAcquirerTest.java deleted file mode 100644 index 0affdef9e..000000000 --- a/ozhera-log/log-agent-server/src/test/java/com/xiaomi/mone/log/server/DefaultAgentConfigAcquirerTest.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -///* -// * Copyright (C) 2020 Xiaomi Corporation -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -//package com.xiaomi.mone.log.server; -// -//import com.google.gson.Gson; -//import com.google.gson.GsonBuilder; -//import com.xiaomi.mone.log.api.model.meta.LogCollectMeta; -//import com.xiaomi.mone.log.server.service.DefaultAgentConfigAcquirer; -//import com.xiaomi.youpin.docean.Ioc; -//import lombok.extern.slf4j.Slf4j; -//import org.junit.Before; -//import org.junit.Test; -// -///** -// * @author wtt -// * @version 1.0 -// * @description -// * @date 2022/12/21 10:56 -// */ -//@Slf4j -//public class DefaultAgentConfigAcquirerTest { -// -// DefaultAgentConfigAcquirer configAcquirer; -// Gson gson; -// -// @Before -// public void buildBean() { -// Ioc.ins().init("com.xiaomi"); -// configAcquirer = Ioc.ins().getBean(DefaultAgentConfigAcquirer.class); -// gson = new GsonBuilder().create(); -// } -// -// @Test -// public void testGetConfig() { -// String ip = "127.0.0.1:1"; -// LogCollectMeta logCollectMetaFromManager = configAcquirer.getLogCollectMetaFromManager(ip); -// log.info("config:{}", gson.toJson(logCollectMetaFromManager)); -// } -//} diff --git a/ozhera-log/log-agent-server/src/test/java/com/xiaomi/mone/log/server/DefaultLogProcessCollectorTest.java b/ozhera-log/log-agent-server/src/test/java/com/xiaomi/mone/log/server/DefaultLogProcessCollectorTest.java deleted file mode 100644 index b0c35feef..000000000 --- a/ozhera-log/log-agent-server/src/test/java/com/xiaomi/mone/log/server/DefaultLogProcessCollectorTest.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -///* -// * Copyright (C) 2020 Xiaomi Corporation -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -//package com.xiaomi.mone.log.server; -// -//import com.google.common.collect.Lists; -//import com.google.gson.Gson; -//import com.google.gson.GsonBuilder; -//import com.xiaomi.mone.log.api.model.vo.UpdateLogProcessCmd; -//import com.xiaomi.mone.log.server.service.DefaultLogProcessCollector; -//import com.xiaomi.youpin.docean.Ioc; -//import lombok.extern.slf4j.Slf4j; -//import org.junit.Before; -//import org.junit.Test; -// -//import java.util.List; -// -///** -// * @author wtt -// * @version 1.0 -// * @description -// * @date 2023/1/4 11:14 -// */ -//@Slf4j -//public class DefaultLogProcessCollectorTest { -// -// DefaultLogProcessCollector processCollector; -// Gson gson; -// -// @Before -// public void buildBean() { -// Ioc.ins().init("com.xiaomi"); -// processCollector = Ioc.ins().getBean(DefaultLogProcessCollector.class); -// gson = new GsonBuilder().create(); -// } -// -// @Test -// public void testCollectLogProcess() { -// UpdateLogProcessCmd updateLogProcessCmd = new UpdateLogProcessCmd(); -// List collectList = Lists.newArrayList(); -// UpdateLogProcessCmd.CollectDetail collectDetail = new UpdateLogProcessCmd.CollectDetail(); -// List fileProgressDetails = Lists.newArrayList(); -// UpdateLogProcessCmd.FileProgressDetail progressDetail = new UpdateLogProcessCmd.FileProgressDetail(); -// progressDetail.setPattern("/home/work/log/test/server.log"); -// progressDetail.setCollectPercentage("98%"); -// fileProgressDetails.add(progressDetail); -// collectDetail.setFileProgressDetails(fileProgressDetails); -// collectList.add(collectDetail); -// updateLogProcessCmd.setCollectList(collectList); -// updateLogProcessCmd.setIp("127.0.0.1"); -// processCollector.collectLogProcess(updateLogProcessCmd); -// List colProcessImperfect = processCollector.getColProcessImperfect(0.97); -// log.info("result:{}", gson.toJson(colProcessImperfect)); -// } -// -// @Test -// public void testGetColProcessImperfect() { -// List colProcessImperfect = processCollector.getColProcessImperfect(0.98); -// log.info("result:{}", gson.toJson(colProcessImperfect)); -// } -//} diff --git a/ozhera-log/log-agent-server/src/test/java/com/xiaomi/mone/log/server/DubboTest.java b/ozhera-log/log-agent-server/src/test/java/com/xiaomi/mone/log/server/DubboTest.java deleted file mode 100644 index 577f5e9dd..000000000 --- a/ozhera-log/log-agent-server/src/test/java/com/xiaomi/mone/log/server/DubboTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.server; - -import lombok.extern.slf4j.Slf4j; -import org.junit.Test; - -/** - * @author wtt - * @version 1.0 - * @description dubbo generalization call - * @date 2022/12/21 16:08 - */ -@Slf4j -public class DubboTest { - - @Test - public void test() { -// ApplicationConfig application = new ApplicationConfig(); -// application.setName("api-gateway-test"); -// application.setQosEnable(false); -// -// RegistryConfig registry = new RegistryConfig(); -// registry.setAddress("nacos://127.0.0.1:80"); -// registry.setRegister(false); -// -// ReferenceConfig reference = new ReferenceConfig<>(); -// reference.setInterface("com.xiaomi.mone.log.api.service.PublishConfigService"); -// reference.setVersion(""); -// reference.setGroup("staging"); -// reference.setGeneric("true"); -// -// reference.setApplication(application); -// reference.setRegistry(registry); -// -//// DubboBootstrap bootstrap = DubboBootstrap.getInstance(); -//// bootstrap.application(application) -//// .registry(registry) -//// .reference(reference) -//// .start(); -// -// ReferenceConfigCache cache = ReferenceConfigCache.getCache(); -// GenericService genericService = cache.get(reference); -// -// Object result = genericService.$invoke("getAllAgentList", new String[]{}, new Object[]{}); -// log.info("result:{}", GSON.toJson(result)); - } -} diff --git a/ozhera-log/log-agent/pom.xml b/ozhera-log/log-agent/pom.xml index ed87bb535..47044fbce 100644 --- a/ozhera-log/log-agent/pom.xml +++ b/ozhera-log/log-agent/pom.xml @@ -3,9 +3,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - run.mone + org.apache.ozhera ozhera-log - 1.4.0-jdk21 + 2.0.0-SNAPSHOT 4.0.0 @@ -206,7 +206,7 @@ false - com.xiaomi.mone.log.agent.bootstrap.MiLogAgentBootstrap + org.apache.ozhera.log.agent.bootstrap.MiLogAgentBootstrap diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/bootstrap/MiLogAgentBootstrap.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/bootstrap/MiLogAgentBootstrap.java deleted file mode 100644 index df94bbc62..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/bootstrap/MiLogAgentBootstrap.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.bootstrap; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.xiaomi.data.push.bo.ClientInfo; -import com.xiaomi.data.push.rpc.RpcClient; -import com.xiaomi.mone.log.agent.common.Version; -import com.xiaomi.mone.log.agent.rpc.task.PingTask; -import com.xiaomi.mone.log.common.Config; -import com.xiaomi.mone.log.utils.NetUtil; -import com.xiaomi.youpin.docean.Aop; -import com.xiaomi.youpin.docean.Ioc; -import lombok.extern.slf4j.Slf4j; - -import java.io.IOException; - -import static com.xiaomi.mone.log.utils.ConfigUtils.getConfigValue; -import static com.xiaomi.mone.log.utils.ConfigUtils.getDataHashKey; - -/** - * @Author goodjava@qq.com - * @Date 2021/6/22 11:22 - */ -@Slf4j -public class MiLogAgentBootstrap { - - public static void main(String[] args) throws IOException { - String nacosAddr = getConfigValue("nacosAddr"); - String serviceName = getConfigValue("serviceName"); - log.info("nacosAddr:{},serviceName:{},version:{}", nacosAddr, serviceName, new Version()); - String appName = Config.ins().get("app_name", "milog_agent"); - ClientInfo clientInfo = new ClientInfo( - String.format("%s_%d", appName, getDataHashKey(NetUtil.getLocalIp(), Integer.parseInt(Config.ins().get("app_max_index", "30")))), - NetUtil.getLocalIp(), - Integer.parseInt(Config.ins().get("port", "9799")), - new Version() + ":" + serviceName + ":" + nacosAddr); - final RpcClient client = new RpcClient(nacosAddr, serviceName); - //Even without service information, use the old registration information (fault tolerance processing). - client.setClearServerAddr(false); - client.setReconnection(false); - client.setClientInfo(clientInfo); - client.start(); - client.setTasks(Lists.newArrayList(new PingTask(client))); - client.init(); - client.waitStarted(); - log.info("create rpc client finish"); - Aop.ins().init(Maps.newLinkedHashMap()); - Ioc.ins().putBean(client).init("com.xiaomi.mone.log.agent", "com.xiaomi.youpin.docean"); - //Because the client life cycle is advanced, the processor needs to be re-registered here - client.registerProcessor(); - System.in.read(); - } - -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/AbstractChannelService.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/AbstractChannelService.java deleted file mode 100644 index dcd947963..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/AbstractChannelService.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel; - -import com.xiaomi.mone.file.ReadResult; -import com.xiaomi.mone.file.common.FileInfoCache; -import com.xiaomi.mone.log.agent.channel.memory.ChannelMemory; -import com.xiaomi.mone.log.agent.common.ChannelUtil; -import com.xiaomi.mone.log.agent.input.Input; -import com.xiaomi.mone.log.api.enums.LogTypeEnum; -import com.xiaomi.mone.log.api.model.meta.LogPattern; -import com.xiaomi.mone.log.api.model.msg.LineMessage; -import com.xiaomi.mone.log.utils.NetUtil; -import lombok.extern.slf4j.Slf4j; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.GSON; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/6/20 16:26 - */ -@Slf4j -public abstract class AbstractChannelService implements ChannelService { - - public String instanceId = UUID.randomUUID().toString(); - - @Override - public String instanceId() { - return instanceId; - } - - @Override - public ChannelState state() { - ChannelState channelState = new ChannelState(); - - ChannelDefine channelDefine = getChannelDefine(); - ChannelMemory channelMemory = getChannelMemory(); - - channelState.setTailId(channelDefine.getChannelId()); - channelState.setTailName(channelDefine.getTailName()); - channelState.setAppId(channelDefine.getAppId()); - channelState.setAppName(channelDefine.getAppName()); - channelState.setLogPattern(channelDefine.getInput().getLogPattern()); - channelState.setLogPatternCode(channelDefine.getInput().getPatternCode()); - - List distinctIpList = channelDefine.getIpDirectoryRel() - .stream() - .map(LogPattern.IPRel::getIp) - .distinct() - .collect(Collectors.toList()); - channelState.setIpList(distinctIpList); - - channelState.setCollectTime(channelMemory.getCurrentTime()); - - if (channelState.getStateProgressMap() == null) { - channelState.setStateProgressMap(new HashMap<>(256)); - } - channelMemory.getFileProgressMap().forEach((pattern, fileProcess) -> { - if (null != fileProcess.getFinished() && fileProcess.getFinished()) { - return; - } - ChannelState.StateProgress stateProgress = new ChannelState.StateProgress(); - stateProgress.setCurrentFile(pattern); - stateProgress.setIp(getTailPodIp(pattern)); - stateProgress.setCurrentRowNum(fileProcess.getCurrentRowNum()); - stateProgress.setPointer(fileProcess.getPointer()); - stateProgress.setFileMaxPointer(fileProcess.getFileMaxPointer()); - stateProgress.setCtTime(fileProcess.getCtTime()); - channelState.getStateProgressMap().put(pattern, stateProgress); - }); - - channelState.setTotalSendCnt(getLogCounts()); - return channelState; - } - - public abstract ChannelDefine getChannelDefine(); - - public abstract ChannelMemory getChannelMemory(); - - public abstract Map getExpireFileMap(); - - public abstract void cancelFile(String file); - - public abstract Long getLogCounts(); - - public LogTypeEnum getLogTypeEnum() { - Input input = getChannelDefine().getInput(); - return LogTypeEnum.name2enum(input.getType()); - } - - /** - * Query IP information based on the actual collection path. - * - * @param pattern - * @return - */ - protected String getTailPodIp(String pattern) { - ChannelDefine channelDefine = getChannelDefine(); - List ipDirectoryRel = channelDefine.getIpDirectoryRel(); - LogPattern.IPRel actualIpRel = ipDirectoryRel.stream().filter(ipRel -> pattern.contains(ipRel.getKey())).findFirst().orElse(null); - if (null != actualIpRel) { - return actualIpRel.getIp(); - } - return NetUtil.getLocalIp(); - } - - protected ChannelMemory initChannelMemory(Long channelId, Input input, List patterns, ChannelDefine channelDefine) { - ChannelMemory channelMemory = new ChannelMemory(); - channelMemory.setChannelId(channelId); - channelMemory.setInput(input); - channelMemory.setFileProgressMap(buildFileProgressMap(patterns, channelDefine)); - channelMemory.setCurrentTime(System.currentTimeMillis()); - channelMemory.setVersion(ChannelMemory.DEFAULT_VERSION); - return channelMemory; - } - - private Map buildFileProgressMap(List patterns, ChannelDefine channelDefine) { - Map fileProgressMap = new HashMap<>(); - for (String pattern : patterns) { - ChannelMemory.FileProgress fileProgress = new ChannelMemory.FileProgress(); - fileProgress.setPointer(0L); - fileProgress.setCurrentRowNum(0L); - fileProgress.setUnixFileNode(ChannelUtil.buildUnixFileNode(pattern)); - fileProgress.setPodType(channelDefine.getPodType()); - fileProgressMap.put(pattern, fileProgress); - } - return fileProgressMap; - } - - protected static void wildcardGraceShutdown(List directory, String matchExpress) { - // Add a shutdown hook to gracefully shutdown FileInfoCache - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - log.info("wildcardGraceShutdown Shutdown,directory:{},express:{}", GSON.toJson(directory), matchExpress); - FileInfoCache.ins().shutdown(); - })); - } - - protected LineMessage createLineMessage(String lineMsg, AtomicReference readResult, String pattern, String patternCode, String ip, long ct) { - LineMessage lineMessage = new LineMessage(); - lineMessage.setMsgBody(lineMsg); - lineMessage.setPointer(readResult.get().getPointer()); - lineMessage.setLineNumber(readResult.get().getLineNumber()); - lineMessage.setFileName(pattern); - lineMessage.setProperties(LineMessage.KEY_MQ_TOPIC_TAG, patternCode); - lineMessage.setProperties(LineMessage.KEY_IP, ip); - lineMessage.setProperties(LineMessage.KEY_COLLECT_TIMESTAMP, String.valueOf(ct)); - - String logType = getChannelDefine().getInput().getType(); - LogTypeEnum logTypeEnum = LogTypeEnum.name2enum(logType); - if (logTypeEnum != null) { - lineMessage.setProperties(LineMessage.KEY_MESSAGE_TYPE, logTypeEnum.getType().toString()); - } - - return lineMessage; - } - - protected void updateChannelMemory(ChannelMemory channelMemory, String fileName, LogTypeEnum logTypeEnum, - long ct, AtomicReference readResult) { - ChannelMemory.FileProgress fileProgress = channelMemory.getFileProgressMap().get(fileName); - ChannelDefine channelDefine = getChannelDefine(); - if (null == fileProgress) { - fileProgress = new ChannelMemory.FileProgress(); - channelMemory.getFileProgressMap().put(fileName, fileProgress); - channelMemory.getInput().setLogPattern(channelDefine.getInput().getLogPattern()); - channelMemory.getInput().setType(logTypeEnum.name()); - channelMemory.getInput().setLogSplitExpress(channelDefine.getInput().getLogSplitExpress()); - } - fileProgress.setCurrentRowNum(readResult.get().getLineNumber()); - fileProgress.setPointer(readResult.get().getPointer()); - if (null != readResult.get().getFileMaxPointer()) { - fileProgress.setFileMaxPointer(readResult.get().getFileMaxPointer()); - } - fileProgress.setUnixFileNode(ChannelUtil.buildUnixFileNode(fileName)); - fileProgress.setPodType(channelDefine.getPodType()); - fileProgress.setCtTime(ct); - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/ChannelDefine.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/ChannelDefine.java deleted file mode 100644 index 32df5b956..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/ChannelDefine.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel; - -import com.xiaomi.mone.log.agent.input.Input; -import com.xiaomi.mone.log.agent.output.Output; -import com.xiaomi.mone.log.api.enums.OperateEnum; -import com.xiaomi.mone.log.api.model.meta.FilterConf; -import com.xiaomi.mone.log.api.model.meta.LogPattern; -import lombok.Data; - -import java.io.Serializable; -import java.util.List; - -/** - * @author shanwb - * @date 2021-07-20 - */ -@Data -public class ChannelDefine implements Serializable { - - private Long channelId; - - private String tailName; - - private Long appId; - - private String appName; - - private Input input; - - private Output output; - - private OperateEnum operateEnum; - - private List ips; - /** - * Relationship between IP and directory - */ - private List ipDirectoryRel; - - /** - * filter and script configuration - */ - private List filters; - - /** - * Individual configuration data, default full configuration under this machine. - */ - private Boolean singleMetaData; - - private String podType; - /** - * The log collection in the directory that needs to be deleted when a machine goes offline only has a value when a machine of a certain application goes offline. - */ - private String delDirectory; - -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/ChannelEngine.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/ChannelEngine.java deleted file mode 100644 index 8fc9f01ac..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/ChannelEngine.java +++ /dev/null @@ -1,621 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel; - -import cn.hutool.core.lang.Pair; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.gson.Gson; -import com.xiaomi.data.push.common.SafeRun; -import com.xiaomi.data.push.rpc.RpcClient; -import com.xiaomi.data.push.rpc.protocol.RemotingCommand; -import com.xiaomi.mone.file.ILogFile; -import com.xiaomi.mone.log.agent.channel.comparator.*; -import com.xiaomi.mone.log.agent.channel.listener.DefaultFileMonitorListener; -import com.xiaomi.mone.log.agent.channel.listener.FileMonitorListener; -import com.xiaomi.mone.log.agent.channel.locator.ChannelDefineJsonLocator; -import com.xiaomi.mone.log.agent.channel.locator.ChannelDefineLocator; -import com.xiaomi.mone.log.agent.channel.locator.ChannelDefineRpcLocator; -import com.xiaomi.mone.log.agent.channel.memory.AgentMemoryService; -import com.xiaomi.mone.log.agent.channel.memory.AgentMemoryServiceImpl; -import com.xiaomi.mone.log.agent.common.ExecutorUtil; -import com.xiaomi.mone.log.agent.export.MsgExporter; -import com.xiaomi.mone.log.agent.factory.OutPutServiceFactory; -import com.xiaomi.mone.log.agent.filter.FilterChain; -import com.xiaomi.mone.log.agent.input.Input; -import com.xiaomi.mone.log.agent.output.Output; -import com.xiaomi.mone.log.api.enums.LogTypeEnum; -import com.xiaomi.mone.log.api.enums.OperateEnum; -import com.xiaomi.mone.log.api.model.vo.UpdateLogProcessCmd; -import com.xiaomi.mone.log.common.Constant; -import com.xiaomi.mone.log.utils.NetUtil; -import com.xiaomi.youpin.docean.Ioc; -import com.xiaomi.youpin.docean.anno.Lookup; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.plugin.config.Config; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.text.NumberFormat; -import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.GSON; -import static com.xiaomi.mone.log.common.PathUtils.PATH_WILDCARD; -import static com.xiaomi.mone.log.common.PathUtils.SEPARATOR; - -/** - * @author shanwb - * @date 2021-07-20 - */ -@Service -@Slf4j -public class ChannelEngine { - private AgentMemoryService agentMemoryService; - - private ChannelDefineLocator channelDefineLocator; - /** - * The configuration pulled in full when the service starts. - */ - private List channelDefineList = Lists.newArrayList(); - - private volatile List channelServiceList = Lists.newArrayList(); - /** - * File listener - */ - private FileMonitorListener fileMonitorListener; - - private String memoryBasePath; - - private Gson gson = GSON; - - @Getter - private volatile boolean initComplete; - - @Lookup("$logFile") - public ILogFile logFile() { - return null; - } - - public void init() { - List failedChannelId = Lists.newArrayList(); - try { - Config config = Ioc.ins().getBean(Config.class.getName()); - memoryBasePath = config.get("agent.memory.path", AgentMemoryService.DEFAULT_BASE_PATH); - //talosProducerMap = new ConcurrentHashMap<>(512); - - channelDefineLocator = getChannelDefineLocator(config); - channelDefineList = new CopyOnWriteArrayList<>(channelDefineLocator.getChannelDefine()); - log.info("current agent all config meta:{}", gson.toJson(channelDefineList)); - agentMemoryService = new AgentMemoryServiceImpl(memoryBasePath); - fileMonitorListener = new DefaultFileMonitorListener(); - - log.info("query channelDefineList:{}", gson.toJson(channelDefineList)); - channelServiceList = channelDefineList.stream() - .filter(channelDefine -> filterCollStart(channelDefine.getAppName())) - .map(channelDefine -> { - ChannelService channelService = this.channelServiceTrans(channelDefine); - if (null == channelService) { - failedChannelId.add(channelDefine.getChannelId()); - } - return channelService; - }).filter(Objects::nonNull).collect(Collectors.toList()); - // Delete failed channel - deleteFailedChannel(failedChannelId, this.channelDefineList, this.channelServiceList); - channelServiceList = new CopyOnWriteArrayList<>(channelServiceList); - // start channel - channelStart(channelServiceList); - //Shutdown - callback action - graceShutdown(); - //Report channel progress once every 10 seconds - exportChannelState(); - log.info("current channelDefineList:{},current channelServiceList:{}", gson.toJson(this.channelDefineList), gson.toJson(this.channelServiceList.stream().map(ChannelService::instanceId).collect(Collectors.toList()))); - monitorFilesClean(); - executorFileClean(); - } catch (Exception e) { - log.error("ChannelEngine init exception", e); - } finally { - initComplete = true; - } - } - - /** - * Thread pool cleaning, many wasted files don't need to keep wasting threads, they should be cleaned up directly. - */ - private void executorFileClean() { - ExecutorUtil.scheduleAtFixedRate(() -> { - SafeRun.run(() -> { - List>> serviceTimeList = Lists.newArrayList(); - for (ChannelService channelService : channelServiceList) { - AbstractChannelService service = (AbstractChannelService) channelService; - Map fileReadTime = service.getExpireFileMap(); - if (!fileReadTime.isEmpty()) { - for (Map.Entry entry : fileReadTime.entrySet()) { - serviceTimeList.add(Pair.of(service, Pair.of(entry.getKey(), entry.getValue()))); - } - } - } - if (serviceTimeList.size() > 500) { - serviceTimeList = serviceTimeList.stream().sorted(Comparator.comparing(o -> o.getValue().getValue())).collect(Collectors.toList()); - for (int i = 0; i < serviceTimeList.size(); i++) { - if (i < 100) { - serviceTimeList.get(i).getKey().cancelFile(serviceTimeList.get(i).getValue().getKey()); - } - } - } - }); - }, 1, 10, TimeUnit.MINUTES); - } - - /** - * Clean up deleted file events - */ - private void monitorFilesClean() { - ExecutorUtil.scheduleAtFixedRate(() -> { - for (ChannelService channelService : channelServiceList) { - try { - channelService.cleanCollectFiles(); - } catch (Exception e) { - log.error("monitorFilesClean error", e); - } - } - }, 1, 1, TimeUnit.MINUTES); - } - - private ChannelDefineLocator getChannelDefineLocator(Config config) { - String locatorType = config.get("agent.channel.locator", "rpc"); - log.warn("locatorType: {}", locatorType); - switch (locatorType) { - case "json": - return new ChannelDefineJsonLocator(); - default: - return new ChannelDefineRpcLocator(); - } - } - - private void exportChannelState() { - ExecutorUtil.scheduleAtFixedRate(() -> { - SafeRun.run(() -> { - List channelStateList = channelServiceList.stream().map(c -> c.state()).collect(Collectors.toList()); - // Send the collection progress - sendCollectionProgress(channelStateList); - }); - }, 10, 10, TimeUnit.SECONDS); - } - - private List channelStart(List channelServiceList) { - List failedChannelIds = Lists.newArrayList(); - List successChannelIds = Lists.newArrayList(); - for (ChannelService channelService : channelServiceList) { - AbstractChannelService abstractChannelService = (AbstractChannelService) channelService; - Long channelId = abstractChannelService.getChannelDefine().getChannelId(); - log.info("realChannelService,id:{}", channelId); - try { - channelService.start(); - fileMonitorListener.addChannelService(channelService); - successChannelIds.add(channelId); - } catch (RejectedExecutionException e) { - log.error("The thread pool is full.id:{}", channelId, e); - } catch (Exception e) { - failedChannelIds.add(channelId); - log.error("start channel exception,channelId:{}", channelId, e); - } - } - deleteFailedChannel(failedChannelIds, this.channelDefineList, this.channelServiceList); - return successChannelIds; - } - - private void deleteFailedChannel(List failedChannelId, List defineList, List serviceList) { - if (CollectionUtils.isNotEmpty(failedChannelId)) { - //Processing is removed from the current queue - for (Long delChannelId : failedChannelId) { - defineList.removeIf(channelDefine -> Objects.equals(delChannelId, channelDefine.getChannelId())); - serviceList.removeIf(channelService -> Objects.equals(delChannelId, ((AbstractChannelService) channelService).getChannelDefine().getChannelId())); - } - } - } - - private void graceShutdown() { - //Close operation - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - log.info("shutdown hook begin!"); - for (ChannelService c : channelServiceList) { - try { - c.close(); - } catch (Exception e) { - log.error("shutdown channel exception:{}", e); - } - } - log.info("shutdown hook end!"); - })); - } - - private ChannelService channelServiceTrans(ChannelDefine channelDefine) { - try { - preCheckChannelDefine(channelDefine); - Output output = channelDefine.getOutput(); - MsgExporter exporter = exporterTrans(output); - if (null == exporter) { - throw new IllegalArgumentException("cant not trans to MsgExporter, output:" + gson.toJson(output)); - } - FilterChain filterChain = new FilterChain(); - filterChain.loadFilterList(channelDefine.getFilters()); - filterChain.reset(); - if (null == agentMemoryService) { - agentMemoryService = new AgentMemoryServiceImpl(com.xiaomi.mone.log.common.Config.ins().get("agent.memory.path", AgentMemoryService.DEFAULT_BASE_PATH)); - } - ChannelService channelService; - Input input = channelDefine.getInput(); - boolean matchWildcard = Arrays.stream(input.getLogPattern().split(",")).anyMatch(data -> StringUtils.substringAfterLast(data, SEPARATOR).contains(PATH_WILDCARD)); - if (matchWildcard) { - channelService = new WildcardChannelServiceImpl(exporter, agentMemoryService, channelDefine, filterChain, memoryBasePath); - } else { - channelService = new ChannelServiceImpl(exporter, agentMemoryService, channelDefine, filterChain); - } - return channelService; - } catch (Throwable e) { - log.error("channelServiceTrans exception, channelDefine:{}, exception:{}", gson.toJson(channelDefine), e); - } - return null; - } - - private void preCheckChannelDefine(ChannelDefine channelDefine) { - Preconditions.checkArgument(null != channelDefine, "channelDefine can not be null"); - Preconditions.checkArgument(null != channelDefine.getInput(), "channelDefine.input can not be null"); - Preconditions.checkArgument(null != channelDefine.getOutput(), "channelDefine.output can not be null"); - Preconditions.checkArgument(null != channelDefine.getChannelId(), "channelDefine.channelId can not be null"); - preCheckOutput(channelDefine.getOutput()); - - Input input = channelDefine.getInput(); - String logPattern = input.getLogPattern(); - Preconditions.checkArgument(null != logPattern, "channelDefine.logPattern can not be null"); - - } - - private void preCheckOutput(Output output) { - Preconditions.checkArgument(StringUtils.isNotBlank(output.getOutputType()), "outputType can not be null"); - OutPutServiceFactory.getOutPutService(output.getServiceName()).preCheckOutput(output); - } - - private MsgExporter exporterTrans(Output output) throws Exception { - if (null == output) { - return null; - } - return OutPutServiceFactory.getOutPutService(output.getServiceName()).exporterTrans(output); - } - - - /** - * Refresh configuration (refresh existing configuration when incremental configuration and full configuration come) - * There are deletion events, indicating that it is not a full configuration, and it goes directly to the stop event. - * - * @param channelDefines - */ - public void refresh(List channelDefines) { - log.info("[config change],changed data:{},origin data:{}", gson.toJson(channelDefines), gson.toJson(channelDefineList)); - try { - if (CollectionUtils.isNotEmpty(channelDefines) && !CollectionUtils.isEqualCollection(channelDefines, channelDefineList)) { - if (channelDefines.stream().allMatch(channelDefine -> null != channelDefine.getOperateEnum() && - channelDefine.getOperateEnum().getCode().equals(OperateEnum.STOP_OPERATE.getCode()))) { - // Collect and delete files in the specified directory - log.info("stopSpecialFileColl,config:{}", gson.toJson(channelDefines)); - delSpecialFileColl(channelDefines); - return; - } - - if (channelDefines.stream().allMatch(channelDefine -> null != channelDefine.getOperateEnum() && - channelDefine.getOperateEnum().getCode().equals(OperateEnum.DELETE_OPERATE.getCode()))) { - log.info("delSpecialFileColl,config:{}", gson.toJson(channelDefines)); - deleteConfig(channelDefines, false); - return; - } - - log.info("refresh,config:{}", gson.toJson(channelDefines)); - // add config - addConfig(channelDefines, false); - // update config - updateConfig(channelDefines); - /** - * Single configuration processing without deletion. - */ - if (channelDefines.size() == 1 && channelDefines.get(0).getSingleMetaData() != null && channelDefines.get(0).getSingleMetaData()) { - return; - } - // delete config - deleteConfig(channelDefines, false); - } - } catch (Exception e) { - log.error("refresh error,[config change],changed data:{},origin data:{}", gson.toJson(channelDefines), gson.toJson(channelDefineList), e); - } - } - - /** - * New configuration - * - * @param channelDefines - */ - private void addConfig(List channelDefines, boolean directAdd) { - try { - // Newly added, initialize - List channelDefinesDifference = differenceSet(channelDefines, channelDefineList); - if (directAdd) { - channelDefinesDifference = channelDefines; - } - if (directAdd || CollectionUtils.isNotEmpty(channelDefinesDifference)) { - log.info("[add config]data:{}", gson.toJson(channelDefinesDifference)); - initIncrement(channelDefinesDifference); - } - } catch (Exception e) { - log.error("addConfig error,source channelDefines:{},origin channelDefines:{},directAdd:{}", gson.toJson(channelDefines), gson.toJson(channelDefineList), directAdd, e); - } - } - - /** - * Update configuration( - * 1. Find the changed configuration - * 2. Delete the original configuration - * 3. Add the configuration again - * ) - * - * @param channelDefines - */ - private void updateConfig(List channelDefines) { - List channelDefinesIntersection = intersection(channelDefines, channelDefineList); - if (CollectionUtils.isNotEmpty(channelDefinesIntersection)) { - List changedDefines = Lists.newArrayList(); - log.info("have exist config:{}", GSON.toJson(channelDefineList)); - Iterator iterator = channelDefinesIntersection.iterator(); - while (iterator.hasNext()) { - ChannelDefine newChannelDefine = iterator.next(); - // old channelDefine - Long channelId = newChannelDefine.getChannelId(); - ChannelDefine oldChannelDefine = channelDefineList.stream().filter(channelDefine -> channelDefine.getChannelId().equals(channelId)).findFirst().orElse(null); - if (null != oldChannelDefine) { - // Comparator - SimilarComparator appSimilarComparator = new AppSimilarComparator(oldChannelDefine.getAppId()); - SimilarComparator inputSimilarComparator = new InputSimilarComparator(oldChannelDefine.getInput()); - SimilarComparator outputSimilarComparator = new OutputSimilarComparator(oldChannelDefine.getOutput()); - FilterSimilarComparator filterSimilarComparator = new FilterSimilarComparator(oldChannelDefine.getFilters()); - if (appSimilarComparator.compare(newChannelDefine.getAppId()) && inputSimilarComparator.compare(newChannelDefine.getInput()) && outputSimilarComparator.compare(newChannelDefine.getOutput())) { - if (!filterSimilarComparator.compare(newChannelDefine.getFilters())) { - channelServiceList.stream().filter(channelService -> ((AbstractChannelService) channelService).getChannelDefine().getChannelId().equals(channelId)).findFirst().ifPresent(channelService -> channelService.filterRefresh(newChannelDefine.getFilters())); - } - } else { - log.info("config changed,old:{},new:{}", gson.toJson(oldChannelDefine), gson.toJson(newChannelDefine)); - changedDefines.add(newChannelDefine); - deleteConfig(Arrays.asList(newChannelDefine), true); - addConfig(Arrays.asList(newChannelDefine), true); - } - } - } - if (CollectionUtils.isNotEmpty(changedDefines)) { - log.info("[update config]data:{}", gson.toJson(changedDefines)); - } - } - } - - /** - * Delete configuration - * - * @param channelDefines - */ - private void deleteConfig(List channelDefines, boolean directDel) { - // The entire file is collected and deleted. - delTailFileColl(channelDefines, directDel); - } - - private void delTailFileColl(List channelDefines, boolean directDel) { - List channelDels = channelDefines.stream().filter(channelDefine -> null != channelDefine.getOperateEnum() && channelDefine.getOperateEnum().getCode().equals(OperateEnum.DELETE_OPERATE.getCode()) && StringUtils.isEmpty(channelDefine.getDelDirectory())).collect(Collectors.toList()); - if (directDel) { - channelDels = channelDefines; - } - try { - if (directDel || CollectionUtils.isNotEmpty(channelDels)) { - log.info("[delete config]data:{}", gson.toJson(channelDels)); - List channelIdDels = channelDels.stream().map(ChannelDefine::getChannelId).collect(Collectors.toList()); - List tempChannelServiceList = Lists.newArrayList(); - channelServiceList.forEach(channelService -> { - Long channelId = ((AbstractChannelService) channelService).getChannelDefine().getChannelId(); - if (channelIdDels.contains(channelId)) { - log.info("[delete config]channelService:{}", channelId); - channelService.close(); - fileMonitorListener.removeChannelService(channelService); - tempChannelServiceList.add(channelService); - this.channelDefineList.removeIf(channelDefine -> { - if (channelDefine.getChannelId().equals(channelId)) { - //delete mq - Output output = channelDefine.getOutput(); - OutPutServiceFactory.getOutPutService(output.getServiceName()).removeMQ(output); - return true; - } - return false; - }); - } - }); - if (CollectionUtils.isNotEmpty(tempChannelServiceList)) { - channelServiceList.removeAll(tempChannelServiceList); - } - } - } catch (Exception e) { - log.error(String.format("delete config exception,config:%s", gson.toJson(channelDels)), e); - } - } - - /** - * Delete log collection under a specific directory. - * - * @param channelDefines - */ - private void delSpecialFileColl(List channelDefines) { - //Find out the pods that need to be deleted when a machine goes offline - List delSpecialFiles = channelDefines.stream().filter(channelDefine -> null != channelDefine.getOperateEnum() && channelDefine.getOperateEnum().getCode().equals(OperateEnum.DELETE_OPERATE.getCode()) && StringUtils.isNotEmpty(channelDefine.getDelDirectory())).collect(Collectors.toList()); - if (CollectionUtils.isNotEmpty(delSpecialFiles)) { - try { - for (ChannelService channelService : channelServiceList) { - CompletableFuture.runAsync(() -> { - AbstractChannelService abstractChannelService = (AbstractChannelService) channelService; - Long channelId = abstractChannelService.getChannelDefine().getChannelId(); - - List defineList = delSpecialFiles.stream().filter(channelDefine -> Objects.equals(channelDefine.getChannelId(), channelId)).collect(Collectors.toList()); - - for (ChannelDefine channelDefine : defineList) { - log.info("deleteConfig,deleteCollFile,channelDefine:{}", gson.toJson(channelDefine)); - channelService.deleteCollFile(channelDefine.getDelDirectory()); - } - //Also need to delete opentelemetry logs. - if (LogTypeEnum.OPENTELEMETRY == abstractChannelService.getLogTypeEnum()) { - for (ChannelDefine channelDefine : delSpecialFiles) { - log.info("deleteConfig OPENTELEMETRY,deleteCollFile,channelDefine:{}", gson.toJson(channelDefine)); - channelService.deleteCollFile(channelDefine.getDelDirectory()); - } - } - }); - } - } catch (Exception e) { - log.error("delSpecialFileColl error,delSpecialFiles:{}", gson.toJson(channelDefines), e); - } - } - } - - /** - * Difference - * - * @param origin New and old configuration - * @param source Old configuration - * @return - */ - private List differenceSet(List origin, List source) { - if (CollectionUtils.isEmpty(source)) { - return origin; - } - List sourceIds = source.stream().map(ChannelDefine::getChannelId).collect(Collectors.toList()); - return origin.stream().filter(channelDefine -> !sourceIds.contains(channelDefine.getChannelId()) && OperateEnum.DELETE_OPERATE != channelDefine.getOperateEnum()).collect(Collectors.toList()); - } - - - /** - * Intersection - * - * @param origin - * @param source - * @return - */ - private List intersection(List origin, List source) { - List sourceIds = Lists.newArrayList(); - if (CollectionUtils.isNotEmpty(source)) { - sourceIds = source.stream().map(ChannelDefine::getChannelId).collect(Collectors.toList()); - } - List finalSourceIds = sourceIds; - return origin.stream().filter(channelDefine -> finalSourceIds.contains(channelDefine.getChannelId()) && OperateEnum.DELETE_OPERATE != channelDefine.getOperateEnum()).collect(Collectors.toList()); - } - - /** - * New configuration initialization - * - * @param definesIncrement - */ - public void initIncrement(List definesIncrement) { - List failedChannelId = Lists.newArrayList(); - List channelServices = definesIncrement.stream() - .filter(Objects::nonNull) - .filter(channelDefine -> filterCollStart(channelDefine.getAppName())) - .map(channelDefine -> { - ChannelService channelService = channelServiceTrans(channelDefine); - if (null == channelService) { - failedChannelId.add(channelDefine.getChannelId()); - } - return channelService; - }).filter(Objects::nonNull).collect(Collectors.toList()); - deleteFailedChannel(failedChannelId, definesIncrement, channelServices); - List successChannelIds = channelStart(channelServices); - if (CollectionUtils.isNotEmpty(successChannelIds)) { - this.channelServiceList.addAll(channelServices.stream().filter(channelService -> successChannelIds.contains(((AbstractChannelService) channelService).getChannelDefine().getChannelId())).collect(Collectors.toList())); - this.channelDefineList.addAll(definesIncrement.stream().filter(channelDefine -> successChannelIds.contains(channelDefine.getChannelId())).collect(Collectors.toList())); - } - log.info("[add config] after current channelDefineList:{},channelServiceList:{}", gson.toJson(this.channelDefineList), gson.toJson(gson.toJson(channelServiceList.stream().map(ChannelService::instanceId).collect(Collectors.toList())))); - } - - private boolean filterCollStart(String appName) { - String serviceName = System.getenv("K8S_SERVICE"); - if (StringUtils.isNotEmpty(serviceName) && StringUtils.isNotEmpty(appName)) { - return serviceName.contains(appName); - } - return true; - } - - - /** - * Send collection progress. - * - * @param - */ - private void sendCollectionProgress(List channelStateList) { - if (CollectionUtils.isEmpty(channelStateList)) { - return; - } - UpdateLogProcessCmd processCmd = assembleLogProcessData(channelStateList); - RpcClient rpcClient = Ioc.ins().getBean(RpcClient.class); - RemotingCommand req = RemotingCommand.createRequestCommand(Constant.RPCCMD_AGENT_CODE); - req.setBody(GSON.toJson(processCmd).getBytes()); - rpcClient.sendToAllMessage(req); - log.debug("send collect progress,data:{}", gson.toJson(processCmd)); - } - - private UpdateLogProcessCmd assembleLogProcessData(List channelStateList) { - UpdateLogProcessCmd cmd = new UpdateLogProcessCmd(); - try { - cmd.setIp(NetUtil.getLocalIp()); - List collects = Lists.newArrayList(); - List finalCollects = collects; - channelStateList.forEach(channelState -> { - - UpdateLogProcessCmd.CollectDetail collectDetail = new UpdateLogProcessCmd.CollectDetail(); - collectDetail.setTailId(channelState.getTailId().toString()); - collectDetail.setAppId(channelState.getAppId()); - collectDetail.setTailName(channelState.getTailName()); - collectDetail.setAppName(channelState.getAppName()); - collectDetail.setIpList(channelState.getIpList()); - collectDetail.setPath(channelState.getLogPattern()); - - List progressDetails = channelState.getStateProgressMap().entrySet().stream().map(entry -> UpdateLogProcessCmd.FileProgressDetail.builder().fileRowNumber(entry.getValue().getCurrentRowNum()).collectTime(entry.getValue().getCtTime()).pointer(entry.getValue().getPointer()).fileMaxPointer(entry.getValue().getFileMaxPointer()).collectPercentage(getPercent(entry.getValue().getPointer(), entry.getValue().getFileMaxPointer())).configIp(entry.getValue().getIp()).pattern(entry.getKey()).build()).collect(Collectors.toList()); - collectDetail.setFileProgressDetails(progressDetails); - finalCollects.add(collectDetail); - }); - //Progress deduplication - collects = collects.stream().distinct().collect(Collectors.toList()); - cmd.setCollectList(collects); - return cmd; - } catch (Exception e) { - log.error("send collect data progress wrap data error", e); - } - return cmd; - } - - private String getPercent(Long pointer, Long maxPointer) { - if (null == pointer || pointer == 0 || null == maxPointer || maxPointer == 0) { - return 0 + "%"; - } - NumberFormat numberFormat = NumberFormat.getInstance(); - numberFormat.setMaximumFractionDigits(2); - return numberFormat.format(((float) pointer / (float) maxPointer) * 100) + "%"; - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/ChannelService.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/ChannelService.java deleted file mode 100644 index dfd93cf9b..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/ChannelService.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel; - -import com.xiaomi.mone.log.agent.channel.file.MonitorFile; -import com.xiaomi.mone.log.agent.export.MsgExporter; -import com.xiaomi.mone.log.api.model.meta.FilterConf; - -import java.util.List; - -/** - * @author shanwb - * @date 2021-07-19 - */ -public interface ChannelService extends Closeable { - /** - * Start channel task. - */ - void start(); - - /** - * Dynamic refresh channel configuration - * - * @param channelDefine - * @param msgExporter - */ - void refresh(ChannelDefine channelDefine, MsgExporter msgExporter); - - /** - * Stop specified file collection. - * - * @param filePrefixList - */ - void stopFile(List filePrefixList); - - /** - * Get the current latest status of Chanel. - * - * @return - */ - ChannelState state(); - - /** - * channel instance id - * - * @return - */ - String instanceId(); - - /** - * There have been changes in the filter configuration. - * - * @param confs - */ - void filterRefresh(List confs); - - /** - * Listening for changes and restarting file collection. - * - * @param filePath - */ - void reOpen(String filePath); - - /** - * List of files to be monitored - * - * @return - */ - List getMonitorPathList(); - - /** - * File cleanup needed - */ - void cleanCollectFiles(); - - /** - * Delete the file collection of a certain directory, applicable when using the demonset deployment method in k8s, when a certain node goes offline, it needs to delete its collection and release resource occupation. - * - * @param directory - */ - void deleteCollFile(String directory); -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/ChannelServiceImpl.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/ChannelServiceImpl.java deleted file mode 100644 index b4771b92a..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/ChannelServiceImpl.java +++ /dev/null @@ -1,718 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel; - -import cn.hutool.core.io.FileUtil; -import cn.hutool.core.lang.Pair; -import com.google.common.collect.Lists; -import com.google.gson.Gson; -import com.xiaomi.data.push.common.SafeRun; -import com.xiaomi.mone.file.*; -import com.xiaomi.mone.log.agent.channel.file.InodeFileComparator; -import com.xiaomi.mone.log.agent.channel.file.MonitorFile; -import com.xiaomi.mone.log.agent.channel.memory.AgentMemoryService; -import com.xiaomi.mone.log.agent.channel.memory.ChannelMemory; -import com.xiaomi.mone.log.agent.common.ChannelUtil; -import com.xiaomi.mone.log.agent.common.ExecutorUtil; -import com.xiaomi.mone.log.agent.export.MsgExporter; -import com.xiaomi.mone.log.agent.filter.FilterChain; -import com.xiaomi.mone.log.agent.input.Input; -import com.xiaomi.mone.log.api.enums.K8sPodTypeEnum; -import com.xiaomi.mone.log.api.enums.LogTypeEnum; -import com.xiaomi.mone.log.api.model.meta.FilterConf; -import com.xiaomi.mone.log.api.model.msg.LineMessage; -import com.xiaomi.mone.log.common.Constant; -import com.xiaomi.mone.log.common.PathUtils; -import com.xiaomi.mone.log.utils.NetUtil; -import com.xiaomi.youpin.docean.Ioc; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.time.Instant; -import java.util.*; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.SYMBOL_COMMA; -import static com.xiaomi.mone.log.common.PathUtils.PATH_WILDCARD; -import static com.xiaomi.mone.log.common.PathUtils.SEPARATOR; - -/** - * @author shanwb - * @date 2021-07-20 - */ -@Slf4j -public class ChannelServiceImpl extends AbstractChannelService { - - private AgentMemoryService memoryService; - - private MsgExporter msgExporter; - - private ChannelDefine channelDefine; - - private ChannelMemory channelMemory; - - @Getter - private final ConcurrentHashMap logFileMap = new ConcurrentHashMap<>(); - - @Getter - private final ConcurrentHashMap futureMap = new ConcurrentHashMap<>(); - - private Set delFileCollList = new CopyOnWriteArraySet<>(); - - private final Map reOpenMap = new HashMap<>(); - private final Map fileReadMap = new ConcurrentHashMap<>(); - - private final Map>> resultMap = new ConcurrentHashMap<>(); - - private ScheduledFuture lastFileLineScheduledFuture; - - private Gson gson = Constant.GSON; - - private List lineMessageList = new ArrayList<>(); - - private ReentrantLock fileColLock = new ReentrantLock(); - - private ReentrantLock fileReopenLock = new ReentrantLock(); - - private volatile long lastSendTime = System.currentTimeMillis(); - - private volatile long logCounts = 0; - - private ScheduledFuture scheduledFuture; - - /** - * collect once flag - */ - private boolean collectOnce; - - private FilterChain chain; - - /** - * The file path to monitor - */ - private List monitorFileList; - - private LogTypeEnum logTypeEnum; - - private String logPattern; - - private String logSplitExpress; - - private String linePrefix; - - public ChannelServiceImpl(MsgExporter msgExporter, AgentMemoryService memoryService, ChannelDefine channelDefine, FilterChain chain) { - this.memoryService = memoryService; - this.msgExporter = msgExporter; - this.channelDefine = channelDefine; - this.chain = chain; - this.monitorFileList = Lists.newArrayList(); - } - - @Override - public void refresh(ChannelDefine channelDefine, MsgExporter msgExporter) { - this.channelDefine = channelDefine; - if (null != msgExporter) { - this.msgExporter.close(); - this.msgExporter = msgExporter; - } - } - - @Override - public void stopFile(List filePrefixList) { - Map fileProgressMap = channelMemory.getFileProgressMap(); - if (null == fileProgressMap) { - fileProgressMap = new HashMap<>(); - } - - for (Iterator> it = logFileMap.entrySet().iterator(); it.hasNext(); ) { - Map.Entry entry = it.next(); - String filePath = entry.getKey(); - for (String filePrefix : filePrefixList) { - if (filePath.startsWith(filePrefix)) { - entry.getValue().setStop(true); - futureMap.get(filePath).cancel(false); - log.warn("channel:{} stop file:{} success", channelDefine.getChannelId(), filePath); - ChannelMemory.FileProgress fileProgress = fileProgressMap.get(filePath); - //Refresh the memory record to prevent the agent from restarting and recollect the file. - if (null != fileProgress) { - fileProgress.setFinished(true); - } - it.remove(); - } - } - } - } - - @Override - public void start() { - Long channelId = channelDefine.getChannelId(); - Input input = channelDefine.getInput(); - - this.logPattern = input.getLogPattern(); - this.logSplitExpress = input.getLogSplitExpress(); - this.linePrefix = input.getLinePrefix(); - - String logType = channelDefine.getInput().getType(); - logTypeEnum = LogTypeEnum.name2enum(logType); - collectOnce = StringUtils.substringAfterLast(logPattern, SEPARATOR).contains(PATH_WILDCARD); - - List patterns = PathUtils.parseLevel5Directory(logPattern); - if (CollectionUtils.isEmpty(patterns)) { - log.info("config pattern:{},current files not exist", logPattern); - } - log.info("channel start, logPattern:{}，fileList:{}, channelId:{}, instanceId:{}", logPattern, patterns, channelId, instanceId()); - // disassembly monitor file - logMonitorPathDisassembled(logSplitExpress, patterns, logPattern); - - channelMemory = memoryService.getMemory(channelId); - if (null == channelMemory) { - channelMemory = initChannelMemory(channelId, input, patterns, channelDefine); - } - memoryService.cleanChannelMemoryContent(channelId, patterns); - - startCollectFile(channelId, input, patterns); - - startExportQueueDataThread(); - memoryService.refreshMemory(channelMemory); - log.warn("channelId:{}, channelInstanceId:{} start success! channelDefine:{}", channelId, instanceId(), gson.toJson(this.channelDefine)); - } - - @Override - public void cleanCollectFiles() { - for (String path : delFileCollList) { - delCollFile(path); - } - } - - @Override - public void deleteCollFile(String directory) { - log.info("channelId:{},deleteCollFile,directory:{}", channelDefine.getChannelId(), directory); - for (Map.Entry logFileEntry : logFileMap.entrySet()) { - if (logFileEntry.getKey().contains(directory)) { - delFileCollList.add(logFileEntry.getKey()); - log.info("channelId:{},delFileCollList:{}", channelDefine.getChannelId(), gson.toJson(delFileCollList)); - } - } - } - - private void startExportQueueDataThread() { - scheduledFuture = ExecutorUtil.scheduleAtFixedRate(() -> { - // If the mq message is not sent for more than 10 seconds, it will be sent asynchronously. - if (System.currentTimeMillis() - lastSendTime < 10 * 1000 || CollectionUtils.isEmpty(lineMessageList)) { - return; - } - if (CollectionUtils.isNotEmpty(lineMessageList) && fileColLock.tryLock()) { - try { - this.doExport(lineMessageList); - } finally { - fileColLock.unlock(); - } - } - }, 10, 7, TimeUnit.SECONDS); - } - - private void startCollectFile(Long channelId, Input input, List patterns) { - for (int i = 0; i < patterns.size(); i++) { - log.info("startCollectFile,total file:{},start:{},remain:{}", patterns.size(), i + 1, patterns.size() - (i + 1)); - readFile(input.getPatternCode(), getTailPodIp(patterns.get(i)), patterns.get(i), channelId); - InodeFileComparator.addFile(patterns.get(i)); - } - lastLineRemainSendSchedule(input.getPatternCode()); - } - - - private void handleAllFileCollectMonitor(String patternCode, String newFilePath, Long channelId) { - String ip = getTailPodIp(newFilePath); - - if (logFileMap.keySet().stream().anyMatch(key -> Objects.equals(newFilePath, key))) { - log.info("collectOnce open file:{}", newFilePath); - logFileMap.get(newFilePath).setReOpen(true); - } else { - readFile(patternCode, ip, newFilePath, channelId); - } - } - - /** - * 1.logSplitExpress:/home/work/log/log-agent/server.log.* realFilePaths: ["/home/work/log/log-agent/server.log"] - * 2.logSplitExpress:/home/work/log/log-agent/(server.log.*|error.log.*) realFilePaths: ["/home/work/log/log-agent/server.log","/home/work/log/log-agent/server.log"] - * 2.logSplitExpress:/home/work/log/(log-agent|log-stream)/server.log.* realFilePaths: ["/home/work/log/log-agent/server.log","/home/work/log/log-stream/server.log"] - * The real file does not exist, it should also listen - * - * @param logSplitExpress - * @param realFilePaths - */ - private void logMonitorPathDisassembled(String logSplitExpress, List realFilePaths, String configPath) { - List cleanedPathList = Lists.newArrayList(); - if (StringUtils.isNotBlank(logSplitExpress)) { - PathUtils.dismantlingStrWithSymbol(logSplitExpress, cleanedPathList); - } - if (LogTypeEnum.OPENTELEMETRY == logTypeEnum || realFilePaths.isEmpty()) { - opentelemetryMonitor(configPath); - return; - } - if (collectOnce) { - collectOnceFileMonitor(configPath); - return; - } - for (int i = 0; i < realFilePaths.size(); i++) { - String perFilePathExpress; - try { - perFilePathExpress = cleanedPathList.get(i); - /** - * Compatible with the current file, it can be monitored when it is created. - */ - perFilePathExpress = String.format("(%s|%s)", perFilePathExpress, String.format("%s.*", realFilePaths.get(i))); - } catch (Exception e) { - perFilePathExpress = String.format("%s.*", realFilePaths.get(i)); - } - monitorFileList.add(MonitorFile.of(realFilePaths.get(i), perFilePathExpress, logTypeEnum, collectOnce)); - } - } - - private void collectOnceFileMonitor(String configPath) { - String singleTimeExpress = ChannelUtil.buildSingleTimeExpress(configPath); - monitorFileList.add(MonitorFile.of(configPath, singleTimeExpress, logTypeEnum, collectOnce)); - } - - private void opentelemetryMonitor(String configPath) { - List cleanedPathList = ChannelUtil.buildLogExpressList(configPath); - monitorFileList.add(MonitorFile.of(configPath, cleanedPathList.get(0), logTypeEnum, collectOnce)); - } - - private ReadListener initFileReadListener(MLog mLog, String patternCode, String ip, String pattern) { - AtomicReference readResult = new AtomicReference<>(); - ReadListener listener = new DefaultReadListener(event -> { - readResult.set(event.getReadResult()); - if (null == readResult.get()) { - log.info("empty data"); - return; - } - long ct = System.currentTimeMillis(); - readResult.get().getLines().stream().forEach(l -> { - String logType = channelDefine.getInput().getType(); - LogTypeEnum logTypeEnum = LogTypeEnum.name2enum(logType); - // Multi-line application log type and opentelemetry type are used to determine the exception stack - if (LogTypeEnum.APP_LOG_MULTI == logTypeEnum || LogTypeEnum.OPENTELEMETRY == logTypeEnum) { - l = mLog.append2(l); - } else { - // tail single line mode - } - if (null != l) { - try { - fileColLock.lock(); - wrapDataToSend(l, readResult, pattern, patternCode, ip, ct); - } finally { - fileColLock.unlock(); - } - } else { - log.debug("biz log channelId:{}, not new line:{}", channelDefine.getChannelId(), l); - } - }); - - }); - resultMap.put(pattern, Pair.of(mLog, readResult)); - return listener; - } - - private void lastLineRemainSendSchedule(String patternCode) { - /** - * Collect all data in the last row of data that has not been sent for more than 10 seconds. - */ - lastFileLineScheduledFuture = ExecutorUtil.scheduleAtFixedRate(() -> SafeRun.run(() -> { - for (Map.Entry>> referenceEntry : resultMap.entrySet()) { - MLog mLog = referenceEntry.getValue().getKey(); - String pattern = referenceEntry.getKey(); - Long appendTime = mLog.getAppendTime(); - if (null != appendTime && Instant.now().toEpochMilli() - appendTime > 10 * 1000) { - if (fileColLock.tryLock()) { - try { - String remainMsg = mLog.takeRemainMsg2(); - if (null != remainMsg) { - log.info("start send last line,pattern:{},patternCode:{},data:{}", pattern, patternCode, remainMsg); - wrapDataToSend(remainMsg, referenceEntry.getValue().getValue(), pattern, patternCode, getTailPodIp(pattern), appendTime); - } - } finally { - fileColLock.unlock(); - } - } - } - } - }), 30, 30, TimeUnit.SECONDS); - } - - private void wrapDataToSend(String lineMsg, AtomicReference readResult, String pattern, String patternCode, String ip, long ct) { - LineMessage lineMessage = createLineMessage(lineMsg, readResult, pattern, patternCode, ip, ct); - - updateChannelMemory(channelMemory, pattern, logTypeEnum, ct, readResult); - lineMessageList.add(lineMessage); - - fileReadMap.put(pattern, ct); - int batchSize = msgExporter.batchExportSize(); - if (lineMessageList.size() > batchSize) { - List subList = lineMessageList.subList(0, batchSize); - doExport(subList); - } - } - - private void readFile(String patternCode, String ip, String filePath, Long channelId) { - MLog mLog = new MLog(); - if (StringUtils.isNotBlank(this.linePrefix)) { - mLog.setCustomLinePattern(this.linePrefix); - } - String usedIp = StringUtils.isBlank(ip) ? NetUtil.getLocalIp() : ip; - - ReadListener listener = initFileReadListener(mLog, patternCode, usedIp, filePath); - Map fileProgressMap = channelMemory.getFileProgressMap(); - printMapToJson(fileProgressMap, collectOnce); - - ILogFile logFile = getLogFile(filePath, listener, fileProgressMap); - if (null == logFile) { - log.warn("file:{} marked stop to collect", filePath); - return; - } - //Determine whether the file exists - if (FileUtil.exist(filePath)) { - stopOldCurrentFileThread(filePath); - log.info("start to collect file,channelId:{},fileName:{}", channelId, filePath); - logFileMap.put(filePath, logFile); - Future future = ExecutorUtil.submit(() -> { - try { - log.info("thread {} {}", Thread.currentThread().isVirtual(), Thread.currentThread()); - logFile.readLine(); - } catch (Exception e) { - logFile.setExceptionFinish(); - log.error("logFile read line err,channelId:{},localIp:{},file:{},patternCode:{}", channelId, usedIp, fileProgressMap, patternCode, e); - } - }); - futureMap.put(filePath, future); - } else { - log.info("file not exist,file:{}", filePath); - } - } - - private void stopOldCurrentFileThread(String filePath) { - ILogFile logFile = logFileMap.get(filePath); - if (null != logFile) { - logFile.setStop(true); - } - Future future = futureMap.get(filePath); - if (null != future) { - future.cancel(false); - } - } - - private void printMapToJson(Map map, boolean collectOnce) { - if (map == null || map.isEmpty()) { - return; - } - - Map snapshot; - try { - snapshot = new HashMap<>(map); - } catch (ConcurrentModificationException e) { - log.error("Failed to create snapshot of fileProgressMap", e); - return; - } - - if (!collectOnce && !snapshot.isEmpty()) { - String jsonMap = gson.toJson(snapshot); - log.info("fileProgressMap: {}", jsonMap); - } - } - - - private ILogFile getLogFile(String filePath, ReadListener listener, Map fileProgressMap) { - - ChannelMemory.FileProgress progressInfo = fileProgressMap.get(filePath); - - if (progressInfo == null || (progressInfo.getFinished() != null && progressInfo.getFinished())) { - // Stateful pods in k8s do not need to be judged by finished - if (StringUtils.isNotBlank(channelDefine.getPodType()) && - K8sPodTypeEnum.valueOf(channelDefine.getPodType().toUpperCase()) != K8sPodTypeEnum.STATEFUL) { - return null; - } - } - long pointer = progressInfo != null ? progressInfo.getPointer() : 0L; - long lineNumber = progressInfo != null ? progressInfo.getCurrentRowNum() : 0L; - if (progressInfo != null) { - ChannelMemory.UnixFileNode memoryUnixFileNode = progressInfo.getUnixFileNode(); - if (memoryUnixFileNode != null && memoryUnixFileNode.getSt_ino() != null) { - log.info("memory file inode info, filePath:{},:{}", filePath, gson.toJson(memoryUnixFileNode)); - ChannelMemory.UnixFileNode currentUnixFileNode = ChannelUtil.buildUnixFileNode(filePath); - if (currentUnixFileNode != null && currentUnixFileNode.getSt_ino() != null && - !Objects.equals(memoryUnixFileNode.getSt_ino(), currentUnixFileNode.getSt_ino())) { - pointer = 0L; - lineNumber = 0L; - log.info("read file start from head, filePath:{}, memory:{}, current:{}", - filePath, gson.toJson(memoryUnixFileNode), gson.toJson(currentUnixFileNode)); - } - } - } - ChannelEngine channelEngine = Ioc.ins().getBean(ChannelEngine.class); - ILogFile logFile = channelEngine.logFile(); - logFile.initLogFile(filePath, listener, pointer, lineNumber); - return logFile; - } - - private void doExport(List subList) { - try { - if (CollectionUtils.isEmpty(subList)) { - return; - } - //Current limiting processing - chain.doFilter(); - - long current = System.currentTimeMillis(); - msgExporter.export(subList); - logCounts += subList.size(); - lastSendTime = System.currentTimeMillis(); - channelMemory.setCurrentTime(lastSendTime); - - log.info("doExport channelId:{}, send {} message, cost:{}, total send:{}, instanceId:{},", channelDefine.getChannelId(), subList.size(), lastSendTime - current, logCounts, instanceId()); - } catch (Exception e) { - log.error("doExport Exception:{}", e); - } finally { - subList.clear(); - } - } - - @Override - public void close() { - log.info("Delete the current collection task,channelId:{}", getChannelId()); - //1.Stop log capture - for (Map.Entry fileEntry : logFileMap.entrySet()) { - fileEntry.getValue().setStop(true); - InodeFileComparator.removeFile(fileEntry.getKey()); - } - //2. stop exporting - this.msgExporter.close(); - //3. refresh cache - memoryService.refreshMemory(channelMemory); - // stop task - if (null != scheduledFuture) { - scheduledFuture.cancel(false); - } - if (null != lastFileLineScheduledFuture) { - lastFileLineScheduledFuture.cancel(false); - } - for (Future future : futureMap.values()) { - future.cancel(false); - } - log.info("stop file monitor,fileName:", logFileMap.keySet().stream().collect(Collectors.joining(SYMBOL_COMMA))); - lineMessageList.clear(); - reOpenMap.clear(); - fileReadMap.clear(); - resultMap.clear(); - } - - public Long getChannelId() { - return channelDefine.getChannelId(); - } - - public MsgExporter getMsgExporter() { - return msgExporter; - } - - @Override - public void filterRefresh(List confs) { - try { - this.chain.loadFilterList(confs); - this.chain.reset(); - } catch (Exception e) { - log.error("filter refresh err,new conf:{}", confs, e); - } - } - - @Override - public void reOpen(String filePath) { - fileReopenLock.lock(); - try { - //Judging the number of openings, it can only be reopened once within 10 seconds. - final long REOPEN_THRESHOLD = 10 * 1000; - - if (reOpenMap.containsKey(filePath) && Instant.now().toEpochMilli() - reOpenMap.get(filePath) < REOPEN_THRESHOLD) { - log.info("The file has been opened too frequently.Please try again in 10 seconds.fileName:{}," + - "last time opening time.:{}", filePath, reOpenMap.get(filePath)); - return; - } - - reOpenMap.put(filePath, Instant.now().toEpochMilli()); - log.info("reOpen file:{}", filePath); - - if (collectOnce) { - handleAllFileCollectMonitor(channelDefine.getInput().getPatternCode(), filePath, getChannelId()); - return; - } - - ILogFile logFile = logFileMap.get(filePath); - String tailPodIp = getTailPodIp(filePath); - String ip = StringUtils.isBlank(tailPodIp) ? NetUtil.getLocalIp() : tailPodIp; - if (null == logFile || logFile.getExceptionFinish()) { - // Add new log file - readFile(channelDefine.getInput().getPatternCode(), ip, filePath, getChannelId()); - log.info("watch new file create for channelId:{},ip:{},path:{}", getChannelId(), filePath, ip); - } else { - handleExistingLogFileWithRetry(logFile, filePath, ip); - } - } finally { - fileReopenLock.unlock(); - } - } - - private void handleExistingLogFileWithRetry(ILogFile logFile, String filePath, String ip) { - LogFile file = (LogFile) logFile; - - int maxRetries = 60; - int currentRetries = 0; - - while (currentRetries < maxRetries) { - if (file.getPointer() < file.getMaxPointer()) { - // Normal log segmentation - try { - TimeUnit.SECONDS.sleep(1); - } catch (InterruptedException e) { - log.error("handleExistingLogFileWithRetry Sleep error", e); - } - - currentRetries++; - } else { - logFile.setReOpen(true); - log.info("file reOpen: channelId:{},ip:{},path:{}", getChannelId(), ip, filePath, file.getFile(), file.getBeforePointerHashCode()); - break; - } - } - } - - @Override - public List getMonitorPathList() { - return monitorFileList; - } - - public ChannelDefine getChannelDefine() { - return channelDefine; - } - - public ChannelMemory getChannelMemory() { - return channelMemory; - } - - /** - * A file that has not been written to for more than 10 minutes. - * - * @return - */ - @Override - public Map getExpireFileMap() { - Map expireMap = new HashMap(); - for (Map.Entry entry : fileReadMap.entrySet()) { - if (Instant.now().toEpochMilli() - entry.getValue() > TimeUnit.MINUTES.toMillis(10)) { - expireMap.put(entry.getKey(), entry.getValue()); - } - } - return expireMap; - } - - @Override - public void cancelFile(String file) { - log.info("cancelFile,file:{}", file); - for (Map.Entry logFileEntry : logFileMap.entrySet()) { - if (file.equals(logFileEntry.getKey())) { - delFileCollList.add(logFileEntry.getKey()); - } - } - } - - /** - * Delete the specified directory collection, receive the delete event and no data is written in for more than 1 minute. - * - * @param path - */ - private void delCollFile(String path) { - boolean shouldRemovePath = false; - if (logFileMap.containsKey(path) && fileReadMap.containsKey(path)) { - if ((Instant.now().toEpochMilli() - fileReadMap.get(path)) > TimeUnit.MINUTES.toMillis(1)) { - cleanFile(path::equals); - shouldRemovePath = true; - log.info("stop coll file:{}", path); - } - } else { - shouldRemovePath = true; - } - if (shouldRemovePath) { - log.info("channelId:{},delCollFile remove file:{}", channelDefine.getChannelId(), path); - delFileCollList.removeIf(data -> StringUtils.equals(data, path)); - } - } - - private void cleanFile(Predicate filter) { - List delFiles = Lists.newArrayList(); - for (Map.Entry logFileEntry : logFileMap.entrySet()) { - if (filter.test(logFileEntry.getKey())) { - InodeFileComparator.removeFile(logFileEntry.getKey()); - logFileEntry.getValue().setStop(true); - delFiles.add(logFileEntry.getKey()); - log.info("cleanFile,stop file:{}", logFileEntry.getKey()); - } - } - for (String delFile : delFiles) { - logFileMap.remove(delFile); - } - delFiles.clear(); - for (Map.Entry futureEntry : futureMap.entrySet()) { - if (filter.test(futureEntry.getKey())) { - futureEntry.getValue().cancel(false); - delFiles.add(futureEntry.getKey()); - } - } - for (String delFile : delFiles) { - futureMap.remove(delFile); - } - delFiles.clear(); - delFiles = reOpenMap.keySet().stream() - .filter(filePath -> filter.test(filePath)) - .collect(Collectors.toList()); - for (String delFile : delFiles) { - reOpenMap.remove(delFile); - } - - delFiles = fileReadMap.keySet().stream() - .filter(filePath -> filter.test(filePath)) - .collect(Collectors.toList()); - for (String delFile : delFiles) { - fileReadMap.remove(delFile); - } - - delFiles = resultMap.keySet().stream() - .filter(filePath -> filter.test(filePath)) - .collect(Collectors.toList()); - for (String delFile : delFiles) { - resultMap.remove(delFile); - } - } - - @Override - public Long getLogCounts() { - return this.logCounts; - } - - -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/ChannelState.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/ChannelState.java deleted file mode 100644 index f45e0164a..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/ChannelState.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel; - -import lombok.Data; - -import java.io.Serializable; -import java.util.List; -import java.util.Map; - -/** - * @author shanwb - * @date 2021-08-26 - */ -@Data -public class ChannelState implements Serializable { - - private Long tailId; - - private String tailName; - - private Long appId; - - private String appName; - - private String logPattern; - /** - * Generated by appId + logPattern - */ - private String logPatternCode; - /** - * Total number of collected and sent rows. - */ - private Long totalSendCnt; - - private List ipList; - - private Long collectTime; - - private Map stateProgressMap; - - @Data - public static class StateProgress implements Serializable { - /** - * ip - */ - private String ip; - /** - * Current collection file - */ - private String currentFile; - /** - * The latest line number currently being collected. - */ - private Long currentRowNum; - /** - * The latest character symbol currently being collected. - */ - private Long pointer; - - /** - * The maximum character count of the current file. - */ - private Long fileMaxPointer; - - /** - * Collection time - */ - private Long ctTime; - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/Closeable.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/Closeable.java deleted file mode 100644 index 9cd872f48..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/Closeable.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel; - -/** - * @author shanwb - * @date 2021-07-29 - */ -public interface Closeable { - /** - * Shut down - */ - void close(); -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/WildcardChannelServiceImpl.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/WildcardChannelServiceImpl.java deleted file mode 100644 index 77cf99d2f..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/WildcardChannelServiceImpl.java +++ /dev/null @@ -1,461 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.xiaomi.data.push.common.SafeRun; -import com.xiaomi.mone.file.MLog; -import com.xiaomi.mone.file.ReadListener; -import com.xiaomi.mone.file.ReadResult; -import com.xiaomi.mone.file.common.FileInfo; -import com.xiaomi.mone.file.common.FileInfoCache; -import com.xiaomi.mone.file.listener.DefaultMonitorListener; -import com.xiaomi.mone.file.ozhera.HeraFileMonitor; -import com.xiaomi.mone.log.agent.channel.file.MonitorFile; -import com.xiaomi.mone.log.agent.channel.memory.AgentMemoryService; -import com.xiaomi.mone.log.agent.channel.memory.ChannelMemory; -import com.xiaomi.mone.log.agent.common.ChannelUtil; -import com.xiaomi.mone.log.agent.common.ExecutorUtil; -import com.xiaomi.mone.log.agent.export.MsgExporter; -import com.xiaomi.mone.log.agent.filter.FilterChain; -import com.xiaomi.mone.log.agent.input.Input; -import com.xiaomi.mone.log.api.enums.LogTypeEnum; -import com.xiaomi.mone.log.api.model.meta.FilterConf; -import com.xiaomi.mone.log.api.model.msg.LineMessage; -import com.xiaomi.mone.log.common.PathUtils; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.io.File; -import java.io.IOException; -import java.time.Instant; -import java.util.*; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Future; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.ReentrantLock; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.agent.channel.memory.AgentMemoryService.MEMORY_DIR; -import static com.xiaomi.mone.log.common.Constant.GSON; -import static com.xiaomi.mone.log.common.Constant.SYMBOL_MULTI; -import static com.xiaomi.mone.log.common.PathUtils.*; - -/** - * @author wtt - * @version 1.0 - * @description Wildcard log collection implementation - * @date 2023/9/27 11:26 - */ -@Slf4j -public class WildcardChannelServiceImpl extends AbstractChannelService { - - private AgentMemoryService memoryService; - - private MsgExporter msgExporter; - - private ChannelDefine channelDefine; - - private ChannelMemory channelMemory; - - private FilterChain chain; - - private String logPattern; - - private String linePrefix; - - private String memoryBasePath; - - private static final String POINTER_FILENAME_PREFIX = ".ozhera_pointer"; - - private List lineMessageList = new ArrayList<>(); - - private ScheduledFuture scheduledFuture; - - private ScheduledFuture lastFileLineScheduledFuture; - - private List> fileCollFutures = Lists.newArrayList(); - - private volatile long lastSendTime = System.currentTimeMillis(); - - private volatile long logCounts = 0; - - private ReentrantLock reentrantLock = new ReentrantLock(); - - private DefaultMonitorListener defaultMonitorListener; - - - public WildcardChannelServiceImpl(MsgExporter msgExporter, AgentMemoryService memoryService, - ChannelDefine channelDefine, FilterChain chain, String memoryBasePath) { - this.memoryService = memoryService; - this.msgExporter = msgExporter; - this.channelDefine = channelDefine; - this.chain = chain; - this.memoryBasePath = memoryBasePath; - } - - @Override - public void start() { - Long channelId = channelDefine.getChannelId(); - Input input = channelDefine.getInput(); - - this.logPattern = input.getLogPattern(); - this.linePrefix = input.getLinePrefix(); - - List patterns = PathUtils.parseLevel5Directory(logPattern); - log.info("channel start, logPattern:{}，fileList:{}, channelId:{}, instanceId:{}", logPattern, GSON.toJson(patterns), channelId, instanceId()); - - channelMemory = memoryService.getMemory(channelId); - if (null == channelMemory) { - channelMemory = initChannelMemory(channelId, input, patterns, channelDefine); - } - memoryService.cleanChannelMemoryContent(channelId, patterns); - - startCollectFile(channelId, input, getTailPodIp(logPattern)); - - startExportQueueDataThread(); - memoryService.refreshMemory(channelMemory); - log.warn("channelId:{}, channelInstanceId:{} start success! channelDefine:{}", channelId, instanceId(), GSON.toJson(this.channelDefine)); - - } - - private void startCollectFile(Long channelId, Input input, String ip) { - try { - // Load the restart file - String restartFile = buildRestartFilePath(); - FileInfoCache.ins().load(restartFile); - - HeraFileMonitor monitor = createFileMonitor(input.getPatternCode(), ip); - - String fileExpression = buildFileExpression(input.getLogPattern()); - - List monitorPaths = buildMonitorPaths(input.getLogPattern()); - - wildcardGraceShutdown(monitorPaths, fileExpression); - - saveCollProgress(); - - log.info("fileExpression:{}", fileExpression); - // Compile the file expression pattern - Pattern pattern = Pattern.compile(fileExpression); - for (String monitorPath : monitorPaths) { - fileCollFutures.add(ExecutorUtil.submit(() -> monitorFileChanges(monitor, monitorPath, pattern))); - } - } catch (Exception e) { - log.error("startCollectFile error, channelId: {}, input: {}, ip: {}", channelId, GSON.toJson(input), ip, e); - } - } - - private void saveCollProgress() { - ExecutorUtil.scheduleAtFixedRate(() -> SafeRun.run(() -> { - try { - for (ReadListener readListener : defaultMonitorListener.getReadListenerList()) { - readListener.saveProgress(); - } - cleanUpInvalidFileInfos(); - FileInfoCache.ins().shutdown(); - } catch (Exception e) { - log.error("saveCollProgress error", e); - } - }), 60, 30, TimeUnit.SECONDS); - } - - // 清理无效的文件信息的方法 - private void cleanUpInvalidFileInfos() { - ConcurrentMap caches = FileInfoCache.ins().caches(); - - for (Iterator> iterator = caches.entrySet().iterator(); iterator.hasNext(); ) { - Map.Entry entry = iterator.next(); - FileInfo fileInfo = entry.getValue(); - File file = new File(fileInfo.getFileName()); - - if (StringUtils.isEmpty(fileInfo.getFileName())) { - continue; - } - - if (!file.exists()) { - FileInfoCache.ins().remove(entry.getKey()); - } - } - } - - private String buildRestartFilePath() { - return String.format("%s%s%s", memoryBasePath, MEMORY_DIR, POINTER_FILENAME_PREFIX); - } - - private String buildFileExpression(String logPattern) { - String[] expressSplit = logPattern.split(","); - if (expressSplit.length == 1) { - return ChannelUtil.buildSingleTimeExpress(logPattern); - } - List expressions = Arrays.stream(expressSplit) - .map(ChannelUtil::buildSingleTimeExpress) - .map(s -> { - String multipleFileName = StringUtils.substringAfterLast(s, SEPARATOR); - return multipleFileName.contains(PATH_WILDCARD) ? s : s + SYMBOL_MULTI; - }) - .distinct() - .toList(); - return expressions.size() == 1 ? - expressions.get(0) : - expressions.stream().collect(Collectors.joining("|", MULTI_FILE_PREFIX, MULTI_FILE_SUFFIX)); - } - - private void monitorFileChanges(HeraFileMonitor monitor, String monitorPath, Pattern pattern) { - try { - log.info("monitorFileChanges,directory:{}", monitorPath); - monitor.reg(monitorPath, filePath -> { - boolean matches = pattern.matcher(filePath).matches(); - log.debug("file: {}, matches: {}", filePath, matches); - return matches; - }); - } catch (IOException | InterruptedException e) { - log.error("Error while monitoring files, monitorPath: {}", monitorPath, e); - } - } - - private List buildMonitorPaths(String filePathExpressName) { - String[] pathExpress = filePathExpressName.split(","); - - List monitorPaths = Arrays.stream(pathExpress) - .map(express -> { - String monitorPath = StringUtils.substringBeforeLast(express, SEPARATOR); - return monitorPath.endsWith(SEPARATOR) ? monitorPath : monitorPath + SEPARATOR; - }) - .flatMap(monitorPath -> PathUtils.buildMultipleDirectories(monitorPath).stream()) - .distinct() - .collect(Collectors.toList()); - - return monitorPaths; - } - - - private HeraFileMonitor createFileMonitor(String patternCode, String ip) { - MLog mLog = new MLog(); - if (StringUtils.isNotBlank(this.linePrefix)) { - mLog.setCustomLinePattern(this.linePrefix); - } - - HeraFileMonitor monitor = new HeraFileMonitor(); - AtomicReference readResult = new AtomicReference<>(); - - defaultMonitorListener = new DefaultMonitorListener(monitor, event -> { - readResult.set(event.getReadResult()); - if (readResult.get() == null) { - log.info("Empty data"); - return; - } - processLogLines(readResult, patternCode, ip, mLog); - }); - - monitor.setListener(defaultMonitorListener); - - /** - * Collect all data in the last row of data that has not been sent for more than 10 seconds. - */ - scheduleLastLineSender(mLog, readResult, patternCode, ip); - return monitor; - } - - private void processLogLines(AtomicReference readResult, String patternCode, String ip, MLog mLog) { - long currentTime = System.currentTimeMillis(); - ReadResult result = readResult.get(); - - LogTypeEnum logTypeEnum = getLogTypeEnum(); - result.getLines().forEach(line -> { - if (LogTypeEnum.APP_LOG_MULTI == logTypeEnum || LogTypeEnum.OPENTELEMETRY == logTypeEnum) { - line = mLog.append2(line); - } - - if (line != null) { - try { - reentrantLock.lock(); - wrapDataToSend(line, readResult, patternCode, ip, currentTime); - } finally { - reentrantLock.unlock(); - } - } else { - log.debug("Biz log channelId:{}, not a new line", channelDefine.getChannelId()); - } - }); - } - - private void scheduleLastLineSender(MLog mLog, AtomicReference readResult, String patternCode, String ip) { - lastFileLineScheduledFuture = ExecutorUtil.scheduleAtFixedRate(() -> { - Long appendTime = mLog.getAppendTime(); - if (appendTime != null && Instant.now().toEpochMilli() - appendTime > 10 * 1000) { - if (reentrantLock.tryLock()) { - try { - String remainMsg = mLog.takeRemainMsg2(); - if (null != remainMsg) { - log.info("start send last line, fileName:{}, patternCode:{}, data:{}", readResult.get().getFilePathName(), patternCode, remainMsg); - wrapDataToSend(remainMsg, readResult, patternCode, ip, Instant.now().toEpochMilli()); - } - } finally { - reentrantLock.unlock(); - } - } - } - }, 30, 30, TimeUnit.SECONDS); - } - - private void wrapDataToSend(String lineMsg, AtomicReference readResult, String patternCode, String localIp, long ct) { - String filePathName = readResult.get().getFilePathName(); - LineMessage lineMessage = createLineMessage(lineMsg, readResult, filePathName, patternCode, localIp, ct); - updateChannelMemory(channelMemory, filePathName, getLogTypeEnum(), ct, readResult); - - lineMessageList.add(lineMessage); - - int batchSize = msgExporter.batchExportSize(); - if (lineMessageList.size() > batchSize) { - List subList = lineMessageList.subList(0, batchSize); - doExport(subList); - } - } - - - private void doExport(List subList) { - try { - if (CollectionUtils.isEmpty(subList)) { - return; - } - //Current limiting processing - chain.doFilter(); - - long current = System.currentTimeMillis(); - msgExporter.export(subList); - logCounts += subList.size(); - lastSendTime = System.currentTimeMillis(); - channelMemory.setCurrentTime(lastSendTime); - - log.info("doExport channelId:{}, send {} message, cost:{}, total send:{}, instanceId:{},", channelDefine.getChannelId(), subList.size(), lastSendTime - current, logCounts, instanceId()); - } catch (Exception e) { - log.error("doExport Exception", e); - } finally { - subList.clear(); - } - } - - private void startExportQueueDataThread() { - scheduledFuture = ExecutorUtil.scheduleAtFixedRate(() -> { - // If the mq message is not sent for more than 10 seconds, it will be sent asynchronously. - if (System.currentTimeMillis() - lastSendTime < 10 * 1000 || CollectionUtils.isEmpty(lineMessageList)) { - return; - } - if (CollectionUtils.isNotEmpty(lineMessageList) && reentrantLock.tryLock()) { - try { - this.doExport(lineMessageList); - } finally { - reentrantLock.unlock(); - } - } - }, 10, 7, TimeUnit.SECONDS); - } - - @Override - public ChannelDefine getChannelDefine() { - return channelDefine; - } - - @Override - public ChannelMemory getChannelMemory() { - return channelMemory; - } - - @Override - public Map getExpireFileMap() { - return Maps.newHashMap(); - } - - @Override - public void cancelFile(String file) { - - } - - @Override - public Long getLogCounts() { - return logCounts; - } - - @Override - public void refresh(ChannelDefine channelDefine, MsgExporter msgExporter) { - this.channelDefine = channelDefine; - if (null != msgExporter) { - this.msgExporter.close(); - this.msgExporter = msgExporter; - } - } - - @Override - public void stopFile(List filePrefixList) { - - } - - @Override - public void filterRefresh(List confs) { - try { - this.chain.loadFilterList(confs); - this.chain.reset(); - } catch (Exception e) { - log.error("filter refresh err,new conf:{}", confs, e); - } - } - - @Override - public void reOpen(String filePath) { - - } - - @Override - public List getMonitorPathList() { - return Lists.newArrayList(); - } - - @Override - public void cleanCollectFiles() { - - } - - @Override - public void deleteCollFile(String directory) { - - } - - @Override - public void close() { - log.info("Delete the current collection task,channelId:{}", channelDefine.getChannelId()); - //2. stop exporting - this.msgExporter.close(); - //3. refresh cache - memoryService.refreshMemory(channelMemory); - // stop task - if (null != scheduledFuture) { - scheduledFuture.cancel(false); - } - if (null != lastFileLineScheduledFuture) { - lastFileLineScheduledFuture.cancel(false); - } - for (Future fileCollFuture : fileCollFutures) { - fileCollFuture.cancel(false); - } - lineMessageList.clear(); - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/comparator/AppSimilarComparator.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/comparator/AppSimilarComparator.java deleted file mode 100644 index 4b0cb58c9..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/comparator/AppSimilarComparator.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel.comparator; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/15 10:04 - */ -public class AppSimilarComparator implements SimilarComparator { - - private Long oldAppId; - - public AppSimilarComparator(Long oldAppId) { - this.oldAppId = oldAppId; - } - - @Override - public boolean compare(Long newAppId) { - return Long.compare(oldAppId, newAppId) == 0; - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/comparator/FilterSimilarComparator.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/comparator/FilterSimilarComparator.java deleted file mode 100644 index 3b3f84103..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/comparator/FilterSimilarComparator.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel.comparator; - -import com.xiaomi.mone.log.api.model.meta.FilterConf; - -import java.util.List; - -public class FilterSimilarComparator implements SimilarComparator> { - private List filterConf; - - public FilterSimilarComparator(List confs) { - this.filterConf = confs; - } - - @Override - public boolean compare(List confs) { - if (confs == null && filterConf == null) { - return true; - } else if (confs != null && filterConf != null) { - if (confs.size() != filterConf.size()) { - return false; - } - return confs.equals(filterConf); - } - return false; - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/comparator/InputSimilarComparator.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/comparator/InputSimilarComparator.java deleted file mode 100644 index 90e1943f5..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/comparator/InputSimilarComparator.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel.comparator; - -import com.xiaomi.mone.log.agent.input.Input; -import lombok.extern.slf4j.Slf4j; - -import static com.xiaomi.mone.log.common.Constant.GSON; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/15 10:08 - */ -@Slf4j -public class InputSimilarComparator implements SimilarComparator { - - private Input oldInput; - - public InputSimilarComparator(Input oldInput) { - this.oldInput = oldInput; - } - - @Override - public boolean compare(Input newInput) { - if (null == oldInput) { - return false; - } - if (oldInput == newInput) { - return true; - } - return baseSimilarCompare(newInput); - } - - private boolean baseSimilarCompare(Input newInput) { - try { - if (oldInput.equals(newInput)) { - return true; - } - } catch (Exception e) { - log.error("input compare error:new input:{},oldInput:{}", GSON.toJson(newInput), GSON.toJson(oldInput), e); - } - return false; - } - - -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/comparator/OutputSimilarComparator.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/comparator/OutputSimilarComparator.java deleted file mode 100644 index 7965d128e..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/comparator/OutputSimilarComparator.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel.comparator; - -import com.xiaomi.mone.log.agent.factory.OutPutServiceFactory; -import com.xiaomi.mone.log.agent.output.Output; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/15 10:08 - */ -public class OutputSimilarComparator implements SimilarComparator { - - private Output oldOutput; - - public OutputSimilarComparator(Output oldOutput) { - this.oldOutput = oldOutput; - } - - @Override - public boolean compare(Output newOutput) { - if (null == oldOutput) { - return false; - } - if (oldOutput == newOutput) { - return true; - } - return OutPutServiceFactory.getOutPutService(newOutput.getServiceName()).compare(oldOutput, newOutput); - } - -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/comparator/SimilarComparator.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/comparator/SimilarComparator.java deleted file mode 100644 index 3c3a79dd4..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/comparator/SimilarComparator.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel.comparator; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/15 10:02 - */ -public interface SimilarComparator { - - boolean compare(T t); -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/conf/AgentTailConf.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/conf/AgentTailConf.java deleted file mode 100644 index b04628ab4..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/conf/AgentTailConf.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel.conf; - -import com.xiaomi.mone.log.agent.channel.ChannelDefine; -import com.xiaomi.mone.log.api.model.meta.AgentDefine; -import lombok.Data; - -import java.util.List; - -/** - * @author milog - */ -@Data -public class AgentTailConf { - private List channelDefine; - private AgentDefine agentDefine; -} \ No newline at end of file diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/file/FileListener.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/file/FileListener.java deleted file mode 100644 index 70682da66..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/file/FileListener.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel.file; - - -import com.xiaomi.mone.log.agent.common.ExecutorUtil; -import lombok.AllArgsConstructor; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.monitor.FileAlterationListener; -import org.apache.commons.io.monitor.FileAlterationObserver; - -import java.io.File; -import java.util.function.Consumer; - -@NoArgsConstructor -@AllArgsConstructor -@Slf4j -public class FileListener implements FileAlterationListener { - - private Consumer consumer; - - @Override - public void onStart(FileAlterationObserver fileAlterationObserver) { - log.debug("FileListener onStart ,filepath:{}", fileAlterationObserver.getDirectory()); - } - - @Override - public void onDirectoryCreate(File file) { - } - - @Override - public void onDirectoryChange(File file) { - } - - @Override - public void onDirectoryDelete(File file) { - } - - @Override - public void onFileCreate(File file) { - log.info("onFileCreate:" + file.getAbsolutePath()); - ExecutorUtil.submit(() -> consumer.accept(file.getAbsolutePath())); - } - - @Override - public void onFileChange(File file) { - - } - - @Override - public void onFileDelete(File file) { - - } - - @Override - public void onStop(FileAlterationObserver fileAlterationObserver) { - log.debug("FileListener onStop ,filepath:{}", fileAlterationObserver.getDirectory()); - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/file/FileMonitor.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/file/FileMonitor.java deleted file mode 100644 index f3b0c2be1..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/file/FileMonitor.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel.file; - -import com.google.common.collect.Lists; -import com.xiaomi.mone.file.LogFile; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.monitor.FileAlterationMonitor; -import org.apache.commons.io.monitor.FileAlterationObserver; - -import java.io.File; -import java.util.List; -import java.util.function.Consumer; - -import static com.xiaomi.mone.log.common.Constant.GSON; - -@Slf4j -@Service -public class FileMonitor implements FileWatcher { - - @Override - public void watch(String filePattern, List monitorList, Consumer consumer) { - List watchList = Lists.newArrayList(filePattern); - //Default traverse file interval time is 10s. - FileAlterationMonitor monitor = new FileAlterationMonitor(10000); - log.info("agent monitor files:{}", GSON.toJson(watchList)); - for (String watch : watchList) { - FileAlterationObserver observer = new LogFileAlterationObserver(new File(watch)); - observer.addListener(new FileListener(consumer)); - log.info("## agent monitor file:{}, filePattern:{}", watch, filePattern); - monitor.addObserver(observer); - } - try { - monitor.start(); - log.info("## agent monitor filePattern:{} started", filePattern); - monitorList.add(monitor); - } catch (Exception e) { - log.error(String.format("agent file monitor start err,monitor filePattern:%s", filePattern), e); - } - } - - @Override - public void watch(LogFile logFile) { - - } - - @Override - public void onChange() { - - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/file/FileWatcher.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/file/FileWatcher.java deleted file mode 100644 index 24e23d617..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/file/FileWatcher.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel.file; - -import com.xiaomi.mone.file.LogFile; -import org.apache.commons.io.monitor.FileAlterationMonitor; - -import java.util.List; -import java.util.function.Consumer; - -/** - * @author shanwb - * @date 2021-07-19 - */ -public interface FileWatcher { - /** - * Watch directory file changes. - * - * @param logFile - */ - void watch(LogFile logFile); - - /** - * watch directory file changes - * - * @param filePattern - * @param monitorList - * @param consumer - */ - void watch(String filePattern, List monitorList, Consumer consumer); - - /** - * - */ - void onChange(); - -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/file/InodeFileComparator.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/file/InodeFileComparator.java deleted file mode 100644 index 0482cf1ca..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/file/InodeFileComparator.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel.file; - -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.agent.channel.memory.ChannelMemory; -import com.xiaomi.mone.log.agent.common.ChannelUtil; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.comparator.DefaultFileComparator; -import org.apache.commons.io.comparator.NameFileComparator; - -import java.io.File; -import java.util.*; - -/** - * @author wtt - * @version 1.0 - * @description File comparator, when the file names of two files are the same, - * continue to compare whether the file inodes are the same. If they are not the same, - * it means that the file has changed. - * @date 2023/7/14 11:19 - */ -@Slf4j -public class InodeFileComparator extends DefaultFileComparator { - - public static final Comparator INODE_COMPARATOR = new InodeFileComparator(); - - private static final Comparator fileComparator = NameFileComparator.NAME_SYSTEM_COMPARATOR; - - private static final Map INODE_MAP = new HashMap<>(); - - private static final List filePaths = Lists.newArrayList(); - - @Override - public int compare(File file1, File file2) { - int sort = fileComparator.compare(file1, file2); - if (file1.isDirectory() || file2.isDirectory()) { - return sort; - } - if (sort == 0 && filePaths.contains(file1.getAbsolutePath())) { - //The file name is the same - Long oldInode; - if (INODE_MAP.containsKey(file1.getAbsolutePath())) { - oldInode = INODE_MAP.get(file1.getAbsolutePath()); - } else { - oldInode = ChannelUtil.buildUnixFileNode(file1.getAbsolutePath()).getSt_ino(); - INODE_MAP.put(file1.getAbsolutePath(), oldInode); - } - ChannelMemory.UnixFileNode unixFileNode2 = ChannelUtil.buildUnixFileNode(file2.getAbsolutePath()); - if (!Objects.equals(oldInode, unixFileNode2.getSt_ino())) { - INODE_MAP.put(file2.getAbsolutePath(), unixFileNode2.getSt_ino()); - return 1; - } - } - return sort; - } - - public static void addFile(String filePath) { - filePaths.add(filePath); - } - - public static void removeFile(String filePath) { - filePaths.remove(filePath); - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/file/LogFileAlterationObserver.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/file/LogFileAlterationObserver.java deleted file mode 100644 index 40b2d7422..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/file/LogFileAlterationObserver.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel.file; - -import cn.hutool.core.util.ReflectUtil; -import com.xiaomi.data.push.rpc.common.RemotingUtil; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOCase; -import org.apache.commons.io.comparator.NameFileComparator; -import org.apache.commons.io.monitor.FileAlterationListener; -import org.apache.commons.io.monitor.FileAlterationObserver; -import org.apache.commons.io.monitor.FileEntry; - -import java.io.File; -import java.io.FileFilter; -import java.util.Arrays; -import java.util.Comparator; - -import static com.xiaomi.mone.log.agent.channel.file.InodeFileComparator.INODE_COMPARATOR; - -/** - * @author wtt - * @version 1.0 - * @description Rewritten file listener, removes events for modifications and deletions, - * only triggers events for new file additions. - * @date 2023/7/14 10:51 - */ -@Slf4j -public class LogFileAlterationObserver extends FileAlterationObserver { - - private static final FileEntry[] EMPTY_ENTRIES = new FileEntry[0]; - - private FileEntry childRootEntry; - private Comparator childComparator; - - public LogFileAlterationObserver(String directoryName) { - this(new File(directoryName)); - } - - public LogFileAlterationObserver(String directoryName, FileFilter fileFilter) { - this(new File(directoryName), fileFilter); - } - - public LogFileAlterationObserver(String directoryName, FileFilter fileFilter, IOCase caseSensitivity) { - this(new File(directoryName), fileFilter, caseSensitivity); - } - - public LogFileAlterationObserver(File directory) { - this(directory, null); - } - - public LogFileAlterationObserver(File directory, FileFilter fileFilter) { - this(directory, fileFilter, null); - } - - public LogFileAlterationObserver(File directory, FileFilter fileFilter, IOCase caseSensitivity) { - this(new FileEntry(directory), fileFilter, caseSensitivity); - } - - protected LogFileAlterationObserver(FileEntry rootEntry, FileFilter fileFilter, IOCase caseSensitivity) { - super(rootEntry, fileFilter, caseSensitivity); - if (!RemotingUtil.isWindowsPlatform()) { - log.info("LogFileAlterationObserver set comparator:{}", INODE_COMPARATOR); - ReflectUtil.setFieldValue(this, "comparator", INODE_COMPARATOR); - } - } - - @Override - public void checkAndNotify() { - - /* fire onStart() */ - for (FileAlterationListener listener : getListeners()) { - listener.onStart(this); - } - - /* fire directory/file events */ - File rootFile = getDirectory(); - if (null == childRootEntry) { - childRootEntry = (FileEntry) ReflectUtil.getFieldValue(this, "rootEntry"); - } - if (rootFile.exists()) { - checkAndNotify(childRootEntry, childRootEntry.getChildren(), listFiles(rootFile)); - } else if (childRootEntry.isExists()) { - checkAndNotify(childRootEntry, childRootEntry.getChildren(), FileUtils.EMPTY_FILE_ARRAY); - } - - /* fire onStop() */ - for (FileAlterationListener listener : getListeners()) { - listener.onStop(this); - } - } - - private void checkAndNotify(FileEntry parent, FileEntry[] previous, File[] files) { - int c = 0; - if (null == childComparator) { - childComparator = (Comparator) ReflectUtil.getFieldValue(this, "comparator"); - } - FileEntry[] current = files.length > 0 ? new FileEntry[files.length] : EMPTY_ENTRIES; - for (FileEntry entry : previous) { - while (c < files.length && childComparator.compare(entry.getFile(), files[c]) > 0) { - current[c] = createFileEntry(parent, files[c]); - doCreate(current[c]); - c++; - } - if (c < files.length && childComparator.compare(entry.getFile(), files[c]) == 0) { - checkAndNotify(entry, entry.getChildren(), listFiles(files[c])); - current[c] = entry; - c++; - } else { - checkAndNotify(entry, entry.getChildren(), FileUtils.EMPTY_FILE_ARRAY); - } - } - for (; c < files.length; c++) { - current[c] = createFileEntry(parent, files[c]); - doCreate(current[c]); - } - parent.setChildren(current); - } - - private File[] listFiles(File file) { - File[] children = null; - if (file.isDirectory()) { - children = getFileFilter() == null ? file.listFiles() : file.listFiles(getFileFilter()); - } - if (children == null) { - children = FileUtils.EMPTY_FILE_ARRAY; - } - if (children.length > 1) { - Arrays.sort(children, NameFileComparator.NAME_SYSTEM_COMPARATOR); - } - return children; - } - - private FileEntry createFileEntry(FileEntry parent, File file) { - FileEntry entry = parent.newChildInstance(file); - entry.refresh(file); - File[] files = listFiles(file); - FileEntry[] children = files.length > 0 ? new FileEntry[files.length] : EMPTY_ENTRIES; - for (int i = 0; i < files.length; i++) { - children[i] = createFileEntry(entry, files[i]); - } - entry.setChildren(children); - return entry; - } - - private void doCreate(FileEntry entry) { - for (FileAlterationListener listener : getListeners()) { - if (entry.isDirectory()) { - listener.onDirectoryCreate(entry.getFile()); - } else { - listener.onFileCreate(entry.getFile()); - } - } - FileEntry[] children = entry.getChildren(); - for (FileEntry aChildren : children) { - doCreate(aChildren); - } - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/file/MonitorFile.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/file/MonitorFile.java deleted file mode 100644 index 380f28b81..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/file/MonitorFile.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel.file; - -import com.xiaomi.mone.log.api.enums.LogTypeEnum; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.regex.Pattern; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/8/4 16:18 - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -public class MonitorFile { - /** - * Real file path address - */ - private String realFilePath; - /** - * Expression for monitoring file changes, eg: /home/work/log/server.log.* - */ - private String monitorFileExpress; - /** - * Generate a regular expression interpreter based on the expression for subsequent matching. - */ - private Pattern filePattern; - /** - * The collection of individual files is completed and then ends. - */ - private boolean collectOnce; - - /** - * Log type, due to the special nature of OpenTelemetry logs, requires special handling when listening. - */ - private LogTypeEnum logTypeEnum; - - public MonitorFile(String realFilePath, String monitorFileExpress, LogTypeEnum logTypeEnum, boolean collectOnce) { - this.realFilePath = realFilePath; - this.monitorFileExpress = monitorFileExpress; - this.filePattern = Pattern.compile(monitorFileExpress); - this.logTypeEnum = logTypeEnum; - this.collectOnce = collectOnce; - } - - public static MonitorFile of(String realFilePath, String monitorFileExpress, LogTypeEnum logTypeEnum, boolean collectOnce) { - return new MonitorFile(realFilePath, monitorFileExpress, logTypeEnum, collectOnce); - } - -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/listener/DefaultFileMonitorListener.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/listener/DefaultFileMonitorListener.java deleted file mode 100644 index 4953584a7..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/listener/DefaultFileMonitorListener.java +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel.listener; - -import cn.hutool.core.io.FileUtil; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import com.google.gson.Gson; -import com.xiaomi.mone.log.agent.channel.ChannelService; -import com.xiaomi.mone.log.agent.channel.file.FileMonitor; -import com.xiaomi.mone.log.agent.channel.file.MonitorFile; -import com.xiaomi.mone.log.agent.common.ChannelUtil; -import com.xiaomi.mone.log.agent.common.ExecutorUtil; -import com.xiaomi.mone.log.api.enums.LogTypeEnum; -import com.xiaomi.mone.log.common.PathUtils; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.io.monitor.FileAlterationMonitor; -import org.apache.commons.lang3.StringUtils; - -import java.io.File; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.*; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.SYMBOL_COMMA; -import static com.xiaomi.mone.log.common.PathUtils.*; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/8/4 15:09 - */ -@Slf4j -public class DefaultFileMonitorListener implements FileMonitorListener { - - private static Gson gson = new Gson(); - - /** - * Default listening folder - */ - private String defaultMonitorPath = "/home/work/log/"; - /** - * List of folders actually monitored - */ - List pathList = new CopyOnWriteArrayList<>(); - /** - * Actual listener list - */ - private List monitorList = new CopyOnWriteArrayList(); - /** - * Each listening thread - */ - private Map> scheduledFutureMap = new ConcurrentHashMap<>(); - /** - * Each ChannelService corresponds to the monitored file - */ - Map, ChannelService> pathChannelServiceMap = new ConcurrentHashMap<>(); - - private final List specialFileNameSuffixList = Lists.newArrayList("wf"); - - private static final int DEFAULT_FILE_SIZE = 100000; - - public DefaultFileMonitorListener() { - //Check if there are too many files, if there are more than 50,000 files, - // then it cannot be monitored. - long size = getDefaultFileSize(); - log.info("defaultMonitorPath:{} file size:{}", defaultMonitorPath, size); - if (size < DEFAULT_FILE_SIZE) { - this.startFileMonitor(defaultMonitorPath); - pathList.add(defaultMonitorPath); - } - } - - private long getDefaultFileSize() { - ExecutorService executor = Executors.newSingleThreadExecutor(); - Future fileSizeFuture = executor.submit(() -> - ChannelUtil.countFilesRecursive(new File(defaultMonitorPath))); - try { - // set the timeout to 1 seconds - return fileSizeFuture.get(1, TimeUnit.SECONDS); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - log.info("getDefaultFileSize error", e); - // cancel the task and close the thread pool - fileSizeFuture.cancel(true); - } finally { - executor.shutdown(); - } - return DEFAULT_FILE_SIZE * 2; - } - - @Override - public void addChannelService(ChannelService channelService) { - List monitorPathList = channelService.getMonitorPathList(); - if (CollectionUtils.isEmpty(monitorPathList)) { - return; - } - List newMonitorDirectories = newMonitorDirectories(monitorPathList); - for (String watchDirectory : newMonitorDirectories) { - if (isValidWatch(watchDirectory)) { - startFileMonitor(watchDirectory); - pathList.add(watchDirectory); - } - } - pathChannelServiceMap.put(monitorPathList, channelService); - } - - private boolean isValidWatch(String watchDirectory) { - if (pathList.contains(watchDirectory)) { - return false; - } - for (String path : pathList) { - if (watchDirectory.startsWith(path)) { - return false; - } - } - return true; - } - - private List newMonitorDirectories(List monitorPathList) { - log.info("start newMonitorDirectories:{}", gson.toJson(monitorPathList)); - List newMonitorDirectories = Lists.newArrayList(); - Set expressList = monitorPathList.stream().map(MonitorFile::getMonitorFileExpress).collect(Collectors.toSet()); - Set realExpressList = Sets.newHashSet(); - /** - * Handle multiple paths spliced together, such as：(/home/work/data/logs/mishop-oscar/mishop-oscar-.*|/home/work/data/logs/mishop-oscar/mishop-oscar-current.log.*) - */ - for (String express : expressList) { - if (express.startsWith(MULTI_FILE_PREFIX) && express.endsWith(MULTI_FILE_SUFFIX) - && express.contains("|")) { - for (String perExpress : StringUtils.substringBetween(express, MULTI_FILE_PREFIX, MULTI_FILE_SUFFIX).split(SPLIT_VERTICAL_LINE)) { - realExpressList.add(perExpress); - } - } else { - realExpressList.add(express); - } - } - for (String perExpress : realExpressList) { - if (pathList.stream().noneMatch(perExpress::startsWith)) { - List watchDList = PathUtils.parseWatchDirectory(perExpress); - /** - * It is already the cleanest directory, there will only be one - */ - String monitorDirectory = watchDList.get(0); - if (monitorDirectory.contains(".*")) { - monitorDirectory = StringUtils.substringBefore(monitorDirectory, ".*"); - } - if (pathList.stream().noneMatch(monitorDirectory::startsWith)) { - newMonitorDirectories.add(monitorDirectory); - } - } - } - newMonitorDirectories = newMonitorDirectories.stream().distinct().collect(Collectors.toList()); - log.info("end newMonitorDirectories:", gson.toJson(newMonitorDirectories)); - return newMonitorDirectories; - } - - @Override - public void removeChannelService(ChannelService channelService) { - try { - pathChannelServiceMap.remove(channelService.getMonitorPathList()); - List monitorPathList = channelService.getMonitorPathList(); - List newMonitorDirectories = newMonitorDirectories(monitorPathList); - for (String watchDirectory : newMonitorDirectories) { - pathList.remove(watchDirectory); - if (scheduledFutureMap.containsKey(watchDirectory)) { - scheduledFutureMap.get(watchDirectory).cancel(true); - } - } - } catch (Exception e) { - log.error("removeChannelService file listener,monitorPathList:{}", gson.toJson(channelService.getMonitorPathList()), e); - } - } - - public void startFileMonitor(String monitorFilePath) { - log.debug("startFileMonitor,monitorFilePath:{}", monitorFilePath); - if (pathList.stream().anyMatch(monitorFilePath::startsWith)) { - log.info("current path has started,monitorFilePath:{},pathList:{}", monitorFilePath, String.join(SYMBOL_COMMA, pathList)); - return; - } - Future fileMonitorFuture = ExecutorUtil.submit(() -> { - new FileMonitor().watch(monitorFilePath, monitorList, changedFilePath -> { - try { - if (FileUtil.isDirectory(changedFilePath)) { - return; - } - log.info("monitor changedFilePath：{}", changedFilePath); - List filterSuffixList = judgeSpecialFileNameSuffix(changedFilePath); - if (CollectionUtils.isNotEmpty(filterSuffixList)) { - specialFileSuffixChanged(changedFilePath, filterSuffixList); - return; - } - ordinaryFileChanged(changedFilePath); - } catch (Exception e) { - log.error("FileMonitor error,monitorFilePath:{},changedFilePath:{}", monitorFilePath, changedFilePath, e); - } - }); - }); - scheduledFutureMap.put(monitorFilePath, fileMonitorFuture); - } - - /** - * Normal file change event handling - * - * @param changedFilePath - */ - private void ordinaryFileChanged(String changedFilePath) { - for (Map.Entry, ChannelService> channelServiceEntry : pathChannelServiceMap.entrySet()) { - for (MonitorFile monitorFile : channelServiceEntry.getKey()) { - if (monitorFile.getFilePattern().matcher(changedFilePath).matches()) { - String reOpenFilePath = monitorFile.getRealFilePath(); - /** - * OPENTELEMETRY Special processing of logs - */ - if (LogTypeEnum.OPENTELEMETRY == monitorFile.getLogTypeEnum() || reOpenFilePath.contains(PATH_WILDCARD)) { - reOpenFilePath = String.format("%s%s%s", StringUtils.substringBeforeLast(changedFilePath, SEPARATOR), - SEPARATOR, StringUtils.substringAfterLast(reOpenFilePath, SEPARATOR)); - } - if (monitorFile.isCollectOnce()) { - reOpenFilePath = changedFilePath; - } - log.info("【change file path reopen】started,changedFilePath:{},realFilePath:{},monitorFileExpress:{}", - changedFilePath, reOpenFilePath, monitorFile.getMonitorFileExpress()); - channelServiceEntry.getValue().reOpen(reOpenFilePath); - log.info("【end change file path】 end,changedFilePath:{},realFilePath:{},monitorFileExpress:{},InstanceId:{}", - changedFilePath, reOpenFilePath, monitorFile.getMonitorFileExpress(), channelServiceEntry.getValue().instanceId()); - } - } - } - } - - /** - * Special file suffix change event handling Through actual observation, - * the go project found that the error log file of the log is server.log.wf, - * which conflicts with the normal server.log, and will receive restart information, - * so for compatibility For something so special, we need to separate out the WF for judgment. - */ - private void specialFileSuffixChanged(String changedFilePath, List filterSuffixList) { - Map serviceMap = new HashMap<>(); - for (Map.Entry, ChannelService> channelServiceEntry : pathChannelServiceMap.entrySet()) { - for (MonitorFile monitorFile : channelServiceEntry.getKey()) { - if (filterSuffixList.stream() - .filter(s -> monitorFile.getRealFilePath().contains(s)).findAny().isPresent() - && monitorFile.getFilePattern().matcher(changedFilePath).matches()) { - serviceMap.put(monitorFile.getRealFilePath(), channelServiceEntry.getValue()); - } - } - } - for (Map.Entry serviceEntry : serviceMap.entrySet()) { - serviceEntry.getValue().reOpen(serviceEntry.getKey()); - } - } - - private List judgeSpecialFileNameSuffix(String changedFilePath) { - String changedFileName = StringUtils.substringAfterLast(changedFilePath, SEPARATOR); - return specialFileNameSuffixList.stream() - .filter(s -> changedFileName.contains(s)) - .collect(Collectors.toList()); - } - - -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/listener/FileMonitorListener.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/listener/FileMonitorListener.java deleted file mode 100644 index 9c40455a4..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/listener/FileMonitorListener.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel.listener; - -import com.xiaomi.mone.log.agent.channel.ChannelService; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/8/4 15:06 - */ -public interface FileMonitorListener { - /** - * Add - */ - void addChannelService(ChannelService channelService); - - /** - * Delete - */ - void removeChannelService(ChannelService channelService); -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/locator/ChannelDefineJsonLocator.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/locator/ChannelDefineJsonLocator.java deleted file mode 100644 index 76680d310..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/locator/ChannelDefineJsonLocator.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel.locator; - -import com.google.common.collect.Lists; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.reflect.TypeToken; -import com.xiaomi.mone.log.agent.channel.ChannelDefine; -import com.xiaomi.mone.log.agent.channel.conf.AgentTailConf; -import com.xiaomi.mone.log.agent.common.AbstractElementAdapter; -import com.xiaomi.mone.log.agent.input.Input; -import com.xiaomi.mone.log.agent.output.Output; -import lombok.extern.slf4j.Slf4j; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.List; - -/** - * Local json configuration method to obtain channel metadata - * - * @author shanwb - * @date 2021-07-21 - */ -@Slf4j -public class ChannelDefineJsonLocator implements ChannelDefineLocator { - private static Gson gson; - - static { - GsonBuilder gsonBuilder = new GsonBuilder(); - gsonBuilder.registerTypeAdapter(Input.class, new AbstractElementAdapter()); - gsonBuilder.registerTypeAdapter(Output.class, new AbstractElementAdapter()); - gson = gsonBuilder.create(); - } - - @Override - public List getChannelDefine() { - try { - AgentTailConf agentTailConf = gson.fromJson(readConfigJson(), new TypeToken() { - }.getType()); - return ChannelDefineRpcLocator.agentTail2ChannelDefine(agentTailConf); - } catch (Exception e) { - log.error("ChannelDefineJsonLocator getChannelDefine exception:{}", e); - } - return Lists.newArrayList(); - } - - @Override - public List getChannelDefine(String ip) { - return null; - } - - private String readConfigJson() { - BufferedReader in = null; - try { - in = new BufferedReader( - new InputStreamReader(ChannelDefineJsonLocator.class.getClassLoader().getResourceAsStream("agent_channel_config.json"))); - StringBuffer sb = new StringBuffer(); - String line; - while (null != (line = in.readLine())) { - sb.append("\n" + line); - } - log.warn("ChannelDefineJsonLocator:{}", sb.toString()); - String str = sb.toString().replaceAll("\r|\n|\\s", ""); - return str; - } catch (IOException e) { - log.error("readConfigJson IOException:{}", e); - } finally { - if (null != in) { - try { - in.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - return null; - } - -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/locator/ChannelDefineLocator.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/locator/ChannelDefineLocator.java deleted file mode 100644 index 00b24f836..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/locator/ChannelDefineLocator.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel.locator; - -import com.xiaomi.mone.log.agent.channel.ChannelDefine; - -import java.util.List; - -/** - * @author shanwb - * @date 2021-07-20 - */ -public interface ChannelDefineLocator { - - /** - * Get channel configuration metadata - * - * @return - */ - List getChannelDefine(); - - /** - * get metadata config with ip - * - * @param ip - * @return - */ - List getChannelDefine(String ip); - -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/locator/ChannelDefineRpcLocator.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/locator/ChannelDefineRpcLocator.java deleted file mode 100644 index 94f626f3c..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/locator/ChannelDefineRpcLocator.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel.locator; - -import cn.hutool.core.util.ObjectUtil; -import com.google.common.collect.Lists; -import com.xiaomi.data.push.rpc.RpcClient; -import com.xiaomi.data.push.rpc.protocol.RemotingCommand; -import com.xiaomi.mone.log.agent.channel.ChannelDefine; -import com.xiaomi.mone.log.agent.channel.conf.AgentTailConf; -import com.xiaomi.mone.log.agent.factory.OutPutServiceFactory; -import com.xiaomi.mone.log.agent.filter.FilterTrans; -import com.xiaomi.mone.log.agent.input.AppLogInput; -import com.xiaomi.mone.log.agent.output.Output; -import com.xiaomi.mone.log.api.enums.LogTypeEnum; -import com.xiaomi.mone.log.api.model.meta.*; -import com.xiaomi.mone.log.common.Constant; -import com.xiaomi.mone.log.utils.NetUtil; -import com.xiaomi.youpin.docean.Ioc; -import com.xiaomi.youpin.docean.anno.Component; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.GSON; -import static com.xiaomi.mone.log.common.Constant.SYMBOL_COLON; - -/** - * RPC method to obtain channel metadata from log-manager - * - * @author shanwb - * @date 2021-07-20 - */ -@Slf4j -@Component -public class ChannelDefineRpcLocator implements ChannelDefineLocator { - - @Override - public List getChannelDefine() { - String localIp = NetUtil.getLocalIp(); - String heraK8sEnv = NetUtil.getHeraK8sEnv(); - if (StringUtils.isNotEmpty(heraK8sEnv)) { - localIp = String.format("%s%s%s", localIp, SYMBOL_COLON, heraK8sEnv); - } - log.info("ChannelDefineRpcLocator,localIp:{},heraK8sEnv:{},ipUnique:{}", NetUtil.getLocalIp(), heraK8sEnv, localIp); - return getChannelDefine(localIp); - } - - @Override - public List getChannelDefine(String ip) { - try { - LogCollectMeta meta; - int count = 0; - while (true) { - meta = getLogCollectMeta(ip); - if (null == meta || CollectionUtils.isEmpty(meta.getAppLogMetaList())) { - //Retry 10 times, stopping for 1 s each time - TimeUnit.SECONDS.sleep(1); - count++; - } else { - break; - } - if (count > 11) { - break; - } - } - return agentTail2ChannelDefine(logCollectMeta2ChannelDefines(meta)); - } catch (Throwable ex) { - log.error(String.format("【agent pull metadata config error】ip:%s", ip), ex); - } - return Lists.newArrayList(); - } - - private LogCollectMeta getLogCollectMeta(String ip) { - try { - RpcClient rpcClient = Ioc.ins().getBean(RpcClient.class); - RemotingCommand res = rpcClient.sendMessage(rpcClient.getServerAddrs(), Constant.RPCCMD_AGENT_CONFIG_CODE, ip, 30000); - String str = new String(res.getBody(), StandardCharsets.UTF_8); - LogCollectMeta meta = GSON.fromJson(str, LogCollectMeta.class); - log.info("agent getChannelDefine finish:{}", str); - return meta; - } catch (Throwable e) { - log.error("getLogCollectMeta error,ip:{}", ip, e); - } - return null; - } - - public static AgentTailConf logCollectMeta2ChannelDefines(LogCollectMeta logCollectMeta) { - AgentTailConf agentTailConf = new AgentTailConf(); - List channelDefines = Lists.newArrayList(); - logCollectMeta.getAppLogMetaList().forEach(appLogMeta -> { - ChannelDefine channelDefine = new ChannelDefine(); - channelDefine.setAppId(appLogMeta.getAppId()); - channelDefine.setAppName(appLogMeta.getAppName()); - channelDefine.setSingleMetaData(logCollectMeta.getSingleMetaData()); - channelDefine.setPodType(logCollectMeta.getPodType()); - channelDefine.setDelDirectory(logCollectMeta.getDelDirectory()); - - List logPatternList = appLogMeta.getLogPatternList(); - for (LogPattern logPattern : logPatternList) { - ChannelDefine cloneChannelDefine = ObjectUtil.clone(channelDefine); - cloneChannelDefine.setChannelId(logPattern.getLogtailId()); - cloneChannelDefine.setTailName(logPattern.getTailName()); - //input - AppLogInput input = new AppLogInput(); - input.setType(logPattern.getLogType() != null ? LogTypeEnum.getLogTypeName(logPattern.getLogType()) : ""); - input.setLogPattern(logPattern.getLogPattern()); - input.setPatternCode(logPattern.getPatternCode()); - input.setLogSplitExpress(logPattern.getLogSplitExpress()); - input.setLinePrefix(logPattern.getFirstLineReg()); - - //output - Output output = OutPutServiceFactory.getChannelDefineLocatorExtension().getOutPutByMQConfigType(logPattern); - // filter - List filterDefines = logPattern.getFilters(); - List filters = Lists.newArrayList(); - if (filterDefines != null) { - filters = filterDefines.stream().map(FilterTrans::filterConfTrans).collect(Collectors.toList()); - } - cloneChannelDefine.setIps(logPattern.getIps()); - cloneChannelDefine.setIpDirectoryRel(logPattern.getIpDirectoryRel()); - cloneChannelDefine.setFilters(filters); - cloneChannelDefine.setInput(input); - cloneChannelDefine.setOutput(output); - cloneChannelDefine.setOperateEnum(logPattern.getOperateEnum()); - channelDefines.add(cloneChannelDefine); - } - }); - - agentTailConf.setChannelDefine(channelDefines); - agentTailConf.setAgentDefine(logCollectMeta.getAgentDefine()); - log.info("agent build metadata config:{}", GSON.toJson(agentTailConf)); - return agentTailConf; - } - - /** - * Add the global filter to the filterlist of channelDefine - * - * @param conf - * @return - */ - public static List agentTail2ChannelDefine(AgentTailConf conf) { - if (conf != null && conf.getAgentDefine() != null && CollectionUtils.isNotEmpty(conf.getAgentDefine().getFilters())) { - List filters = conf.getAgentDefine().getFilters() - .stream() - .filter(f -> f != null && f.getType().equals(FilterType.GLOBAL)) - .collect(Collectors.toList()); - if (conf.getChannelDefine() != null && CollectionUtils.isNotEmpty(filters)) { - conf.getChannelDefine().forEach(c -> { - c.getFilters().addAll(filters); - }); - } - } - return conf.getChannelDefine(); - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/memory/AgentMemoryService.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/memory/AgentMemoryService.java deleted file mode 100644 index ed71ba3f7..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/memory/AgentMemoryService.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel.memory; - -import java.util.List; - -/** - * @author shanwb - * @date 2021-07-19 - */ -public interface AgentMemoryService { - - String MEMORY_DIR = "/milog/memory/"; - String CHANNEL_FILE_PREFIX = "channel_"; - String DEFAULT_BASE_PATH = "/tmp/"; - - /** - * Refresh memory progress - * - * @param channelMemory - */ - void refreshMemory(ChannelMemory channelMemory); - - /** - * Get cache information - * - * @return - */ - List getMemory(); - - /** - * Get cache information - * - * @param channelId - * @return - */ - ChannelMemory getMemory(Long channelId); - - /** - * Restore memory from disk - * - * @return - */ - List restoreFromDisk(); - - /** - * Flush to disk - * - * @param channelMemoryList - */ - void flush2disk(List channelMemoryList); - - /** - * Compare files to prevent them from becoming too large - * - * @param filePaths All files that actually exist - * @param channelId - */ - void cleanChannelMemoryContent(Long channelId, List filePaths); - - /** - * Manage deleted tail memory files - * - * @param channelIds Full collection of chanel IDs - */ - void cleanMemoryHistoryFile(List channelIds); - -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/memory/AgentMemoryServiceImpl.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/memory/AgentMemoryServiceImpl.java deleted file mode 100644 index 6e8a53cd1..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/memory/AgentMemoryServiceImpl.java +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel.memory; - -import cn.hutool.core.io.FileUtil; -import com.google.common.collect.Lists; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.xiaomi.mone.file.FileUtils; -import com.xiaomi.mone.file.ReadResult; -import com.xiaomi.mone.log.agent.common.AbstractElementAdapter; -import com.xiaomi.mone.log.agent.common.ExecutorUtil; -import com.xiaomi.mone.log.agent.exception.AgentException; -import com.xiaomi.mone.log.agent.input.Input; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.stream.Collectors; - -/** - * @author shanwb - * @date 2021-07-19 - */ -@Slf4j -public class AgentMemoryServiceImpl implements AgentMemoryService { - - private static ConcurrentHashMap memoryFileList = new ConcurrentHashMap<>(); - private ConcurrentHashMap channelMemoryMap = new ConcurrentHashMap<>(); - - private String basePath; - private static Gson gson; - - static { - GsonBuilder gsonBuilder = new GsonBuilder(); - gsonBuilder.registerTypeAdapter(Input.class, new AbstractElementAdapter()); - gson = gsonBuilder.create(); - } - - public AgentMemoryServiceImpl(String basePath) { - this.basePath = basePath; - initFolder(this.basePath + MEMORY_DIR); - initChannelMemory(); - //Brush the disc regularly for 30 s - initFlushTask(this); - } - - private static void initFolder(String dir) { - File file = new File(dir); - if (!file.exists()) { - try { - Files.createDirectories(file.toPath()); - } catch (IOException e) { - log.error("initFolder:{} exception:{}", dir, e); - throw new AgentException(dir + " create exception", e); - } - } else { - log.info("dir:{} is exists,no need to create", dir); - } - } - - private void initChannelMemory() { - List channelMemoryList = this.restoreFromDisk(); - if (CollectionUtils.isEmpty(channelMemoryList)) { - return; - } - - channelMemoryList.forEach(c -> { - if (c != null && ChannelMemory.DEFAULT_VERSION.equals(c.getVersion())) { - channelMemoryMap.put(c.getChannelId(), c); - } - }); - } - - private void initFlushTask(AgentMemoryService agentMemoryService) { - ExecutorUtil.scheduleAtFixedRate(new Runnable() { - @Override - public void run() { - agentMemoryService.flush2disk(agentMemoryService.getMemory()); - } - }, 10, 30, TimeUnit.SECONDS); - } - - - @Override - public void refreshMemory(ChannelMemory channelMemory) { - //refresh memory - channelMemoryMap.put(channelMemory.getChannelId(), channelMemory); - //refresh disk - this.flush2disk(Lists.newArrayList(channelMemory)); - } - - @Override - public List getMemory() { - return new ArrayList<>(channelMemoryMap.values()); - } - - @Override - public ChannelMemory getMemory(Long channelId) { - return channelMemoryMap.get(channelId); - } - - @Override - public List restoreFromDisk() { - List channelMemoryList = new ArrayList<>(); - File file = new File(this.basePath + MEMORY_DIR); - File[] fs = file.listFiles(); - if (null == fs || fs.length == 0) { - return Lists.newArrayList(); - } - - for (File f : fs) { - if (!f.isDirectory()) { - try { - if (f.getName().startsWith(CHANNEL_FILE_PREFIX)) { - ReadResult readResult = FileUtils.readFile(f.getAbsolutePath(), 0, 0); - List list = readResult.getLines(); - if (CollectionUtils.isNotEmpty(list)) { - String channel = list.get(0); - ChannelMemory channelMemory = gson.fromJson(channel, ChannelMemory.class); - - - if (channelMemory != null && ChannelMemory.DEFAULT_VERSION.equals(channelMemory.getVersion())) { - channelMemoryList.add(channelMemory); - } - log.warn("restoreFromDisk channel:{},channelMemory:{}", channel, channelMemory); - } - } - } catch (Exception e) { - log.error("restoreFromDisk error,file:{}", f.getName(), e); - } - } - } - - return channelMemoryList; - } - - @Override - public void flush2disk(List channelMemoryList) { - for (ChannelMemory channelMemory : channelMemoryList) { - FileWriter writer = null; - try { - Long channelId = channelMemory.getChannelId(); - File memoryFile = memoryFileList.get(channelId); - if (null == memoryFile) { - memoryFile = new File(this.basePath + MEMORY_DIR + CHANNEL_FILE_PREFIX + channelId); - if (!memoryFile.exists()) { - try { - memoryFile.createNewFile(); - } catch (IOException e) { - e.printStackTrace(); - } - } - // put - memoryFileList.put(channelId, memoryFile); - } - - writer = new FileWriter(memoryFile, false); - writer.append(gson.toJson(channelMemory)); - writer.flush(); - } catch (Throwable e) { - log.error("flush2disk error, channelId:{}", channelMemory.getChannelId(), e); - } - } - } - - @Override - public void cleanChannelMemoryContent(Long channelId, List filePaths) { - ChannelMemory channelMemory = channelMemoryMap.get(channelId); - if (null != channelMemory) { - Map fileProgressMap = channelMemory.getFileProgressMap(); - List expireFilePaths = fileProgressMap.keySet().stream() - .filter(filePath -> !filePaths.contains(filePath)) - .collect(Collectors.toList()); - expireFilePaths.forEach(expireFilePath -> { - fileProgressMap.remove(expireFilePath); - }); - flush2disk((Lists.newArrayList(channelMemory))); - } - } - - @Override - public void cleanMemoryHistoryFile(List channelIds) { - try { - log.info("all channelIds:{}", gson.toJson(channelIds)); - List channelIdList = channelIds.stream().map(String::valueOf).collect(Collectors.toList()); - List files = FileUtil.loopFiles(this.basePath + MEMORY_DIR); - Map fileMap = files.stream() - .collect(Collectors - .toMap(File::getName, Function.identity(), (file, file2) -> file2)); - for (String fileName : fileMap.keySet()) { - if (!channelIdList.contains(StringUtils.substringAfter(fileName, CHANNEL_FILE_PREFIX))) { - fileMap.get(fileName).delete(); - } - } - } catch (Exception e) { - log.error("cleanMemoryHistoryFile error,channelIds:{}", gson.toJson(channelIds), e); - } - } - - /** - * Repair data usage, this does not need to be used - */ - private void fixedData() { - channelMemoryMap.entrySet().stream().forEach(memoryEntry -> { - FileWriter writer; - ChannelMemory channelMemory = memoryEntry.getValue(); - channelMemory.getFileProgressMap().entrySet().forEach(progressEntry -> { - progressEntry.getValue().setFinished(Boolean.FALSE); - }); - try { - File memoryFile = new File(this.basePath + MEMORY_DIR + CHANNEL_FILE_PREFIX + memoryEntry.getKey()); - writer = new FileWriter(memoryFile, false); - writer.append(gson.toJson(channelMemory)); - writer.flush(); - } catch (Throwable e) { - log.error("flush2disk error, channelId:{}", channelMemory.getChannelId(), e); - } - }); - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/memory/ChannelMemory.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/memory/ChannelMemory.java deleted file mode 100644 index 4db2cc1a3..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/channel/memory/ChannelMemory.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel.memory; - -import com.xiaomi.mone.log.agent.input.Input; -import com.xiaomi.mone.log.api.model.msg.LineMessage; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.io.Serializable; -import java.util.List; -import java.util.Map; - -/** - * @author shanwb - * @date 2021-07-21 - */ -@Data -public class ChannelMemory implements Serializable { - - public transient static final String DEFAULT_VERSION = "2.0"; - - private Long channelId; - - private Input input; - - private String version; - - private Map fileProgressMap; - - private Long currentTime; - - /** - * Message cache list, changed to file address, loaded again when needed - */ - private List messageList; - - @Data - public static class FileProgress implements Serializable { - - private String currentFile; - /** - * collection time - */ - private Long ctTime; - - /** - * The index number of the current file list is not the largest, indicating that the collection is delayed - */ - private Integer currentFileIdx; - /** - * Identify the uniqueness of files in unix - */ - private UnixFileNode unixFileNode; - - /** - * The latest line number currently collected - */ - private Long currentRowNum; - /** - * The latest character symbol currently collected - */ - private Long pointer; - /** - * The latest character symbol of the current file - */ - private Long fileMaxPointer; - /** - * File list. Generally, logs will be equipped with split rules and split into multiple files. - */ - private List fileList; - - /** - * Whether to stop the collection - * - if the collection is stopped, the agent will ignore the collection when it restarts - * - if the collection is stopped, if the file does not exist, the memory record will also be deleted - */ - private Boolean finished; - /** - * Only has value when deployed on k8s - */ - private String podType; - - } - - @Data - @EqualsAndHashCode - public static class UnixFileNode { - private Long st_dev; - private Long st_ino; - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/common/AbstractElementAdapter.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/common/AbstractElementAdapter.java deleted file mode 100644 index 645a34fe1..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/common/AbstractElementAdapter.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.common; - -import com.google.gson.*; -import com.xiaomi.mone.log.agent.input.AppLogInput; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; - -import java.io.InputStream; -import java.io.InputStreamReader; -import java.lang.reflect.Type; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; - -/** - * @author shanwb - * @date 2021-08-02 - */ -@Slf4j -public class AbstractElementAdapter implements - JsonSerializer, JsonDeserializer { - - private static final String KEY_TYPE = "type"; - - private Map typeToClassMap; - - public AbstractElementAdapter() { - this.typeToClassMap = loadTypeToClassMappingFromConfig(); - } - - - @SneakyThrows - @Override - public Object deserialize(JsonElement jsonElement, Type type, - JsonDeserializationContext jsonDeserializationContext) - throws JsonParseException { - JsonObject jsonObj = jsonElement.getAsJsonObject(); - String implType = jsonObj.get(KEY_TYPE).getAsString(); - - // Get the corresponding class object - Class clz = getClassForImplType(implType); - - return jsonDeserializationContext.deserialize(jsonElement, clz); - } - - private Class getClassForImplType(String implType) throws ClassNotFoundException { - - // Get the corresponding class name based on implType - String className = this.typeToClassMap.get(implType); - - if (className != null && !className.isEmpty()) { - // Loading classes using reflection - return Class.forName(className); - } else { - // If the corresponding class name cannot be found, you can throw an exception or provide a default value - return AppLogInput.class; - } - } - - private static Map loadTypeToClassMappingFromConfig() { - Map typeToClassMap = new HashMap<>(); - - String configFile = "log_impl_type.json"; - - try (InputStream input = AbstractElementAdapter.class.getClassLoader().getResourceAsStream(configFile)) { - // Parse JSON files using Gson - JsonParser parser = new JsonParser(); - JsonObject json = parser.parse(new InputStreamReader(input, StandardCharsets.UTF_8)).getAsJsonObject(); - - // Parse JSON and store the mapping in typeToClassMap - for (Map.Entry entry : json.entrySet()) { - typeToClassMap.put(entry.getKey(), entry.getValue().getAsString()); - } - } catch (Exception e) { - log.error("loadTypeToClassMappingFromConfig error,fileName:{}", configFile, e); - } - - return typeToClassMap; - } - - @Override - public JsonElement serialize(Object object, Type type, - JsonSerializationContext jsonSerializationContext) { - JsonElement jsonEle = jsonSerializationContext.serialize(object, object.getClass()); - return jsonEle; - } -} \ No newline at end of file diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/common/ChannelUtil.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/common/ChannelUtil.java deleted file mode 100644 index ba5199a09..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/common/ChannelUtil.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.common; - -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.agent.channel.memory.ChannelMemory; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.io.File; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Paths; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.*; -import static com.xiaomi.mone.log.common.PathUtils.PATH_WILDCARD; -import static com.xiaomi.mone.log.common.PathUtils.SEPARATOR; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/6/23 14:55 - */ -@Slf4j -public class ChannelUtil { - - private ChannelUtil() { - - } - - public static List buildLogExpressList(String logPattern) { - List pathList = Lists.newArrayList(); - for (String filePath : logPattern.split(SYMBOL_COMMA)) { - String filePrefix = StringUtils.substringBeforeLast(filePath, SEPARATOR); - String multipleFileNames = StringUtils.substringAfterLast(filePath, SEPARATOR); - if (filePath.contains(PATH_WILDCARD) && !filePath.contains(SYMBOL_MULTI)) { - logPattern = logPattern.replaceAll("\\*", SYMBOL_MULTI); - } else { - logPattern = Arrays.stream(multipleFileNames.split("\\|")) - .map(s -> filePrefix + SEPARATOR + s + SYMBOL_MULTI) - .collect(Collectors.joining(DEFAULT_TAIL_SEPARATOR)); - } - if (!logPattern.endsWith(SYMBOL_MULTI)) { - logPattern = logPattern + SYMBOL_MULTI; - } - pathList.add(logPattern); - } - return pathList; - } - - public static String buildSingleTimeExpress(String filePath) { - String filePrefix = StringUtils.substringBeforeLast(filePath, SEPARATOR); - String multipleFileName = StringUtils.substringAfterLast(filePath, SEPARATOR); - if (!multipleFileName.contains(SYMBOL_MULTI)) { - multipleFileName = multipleFileName.replaceAll("\\*", SYMBOL_MULTI); - } - if (multipleFileName.startsWith(PATH_WILDCARD)) { - multipleFileName = multipleFileName.replaceFirst("\\*", SYMBOL_MULTI); - } - return String.format("%s%s%s", filePrefix, SEPARATOR, multipleFileName); - } - - /** - * The value can only be obtained from unix system files, otherwise it is an empty object. - * - * @param filePath - * @return - */ - public static ChannelMemory.UnixFileNode buildUnixFileNode(String filePath) { - try { - BasicFileAttributes fileAttributes = Files.readAttributes(Paths.get(filePath), BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS); - if (null != fileAttributes.fileKey()) { - ChannelMemory.UnixFileNode unixFileNode = GSON.fromJson(GSON.toJson(fileAttributes.fileKey()), ChannelMemory.UnixFileNode.class); - log.debug("origin file path:{},fileNode unixFileNode:{}", filePath, GSON.toJson(unixFileNode)); - return unixFileNode; - } - } catch (Exception e) { - log.info("buildUnixFileNode error,filePath:{}", filePath, e); - } - return new ChannelMemory.UnixFileNode(); - } - - public static long countFilesRecursive(File directory) { - long count = 0; - File[] files = directory.listFiles(); - if (files != null) { - for (File file : files) { - if (file.isDirectory()) { - count += countFilesRecursive(file); - } else { - count++; - } - } - } - return count; - } - -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/common/ExecutorUtil.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/common/ExecutorUtil.java deleted file mode 100644 index 38da5b7a2..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/common/ExecutorUtil.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.common; - -import lombok.extern.slf4j.Slf4j; - -import java.util.concurrent.*; - -/** - * @author shanwb - * @date 2021-08-05 - */ -@Slf4j -public class ExecutorUtil { - - public static ScheduledThreadPoolExecutor STP_EXECUTOR = new ScheduledThreadPoolExecutor(20, - Thread.ofVirtual().name("ExecutorUtil-STP-Virtual-Thread", 0) - .uncaughtExceptionHandler((t, e) -> { - log.error("ExecutorUtil-STP-Virtual-Thread uncaughtException:{}", e.getMessage(), e); - }).factory()); - - public static ExecutorService TP_EXECUTOR = createPool(); - - public static ScheduledFuture scheduleAtFixedRate(Runnable command, - long initialDelay, - long period, - TimeUnit unit) { - return STP_EXECUTOR.scheduleAtFixedRate(command, initialDelay, period, unit); - } - - public static ExecutorService createPool() { - System.setProperty("jdk.virtualThreadScheduler.parallelism", String.valueOf(Runtime.getRuntime().availableProcessors() + 1)); - ThreadFactory factory = Thread.ofVirtual().name("ExecutorUtil-TP-Virtual-Thread", 0) - .uncaughtExceptionHandler((t, e) -> { - log.error("ExecutorUtil-TP-Virtual-Thread uncaughtException:{}", e.getMessage(), e); - }).factory(); - return Executors.newThreadPerTaskExecutor(factory); - } - - public static Future submit(Runnable task) { - log.warn("TP_EXECUTOR submit task:{}", task.toString()); - return TP_EXECUTOR.submit(task); - } - - static { - //Regularly print thread pool information - STP_EXECUTOR.scheduleAtFixedRate(() -> { - log.warn("Executor statistic TP_EXECUTOR:{}", TP_EXECUTOR.toString()); - log.warn("Executor statistic STP_EXECUTOR:{}", STP_EXECUTOR.toString()); - }, 17, 30, TimeUnit.SECONDS); - } - -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/common/HashUtil.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/common/HashUtil.java deleted file mode 100644 index 931ad4a3d..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/common/HashUtil.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.common; - -import com.google.common.hash.HashCode; -import com.google.common.hash.Hashing; - -import java.nio.charset.StandardCharsets; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/9/21 16:07 - */ -public class HashUtil { - - private HashUtil() { - } - - public static int consistentHash(String partitionKey, int buckets) { - HashCode hasCode = Hashing.murmur3_32().hashString(partitionKey, StandardCharsets.UTF_8); - return Hashing.consistentHash(hasCode, buckets); - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/common/Version.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/common/Version.java deleted file mode 100644 index c0121c910..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/common/Version.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.common; - -/** - * @Author goodjava@qq.com - * @Date 2021/6/22 13:46 - */ -public class Version { - - @Override - public String toString() { - return "0.0.1:20240528"; - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/common/trace/MessageUtil.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/common/trace/MessageUtil.java deleted file mode 100644 index bbcdcbc36..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/common/trace/MessageUtil.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.common.trace; - -public class MessageUtil { - - // Message body delimiter - public static final String SPLIT = " ### "; - // rocksdb message delimiter - public static final String ROCKS_SPLIT = " #### "; - public static final String ERROR_CODE = "ERROR"; - // The corner labels of each field are easy to modify - public static final int START_TIME = 0; - public static final int DURATION = 1; - public static final int IP = 2; - public static final int APPLICATION_NAME = 3; - public static final int SPAN_NAME = 4; - public static final int STATUS_CODE = 5; - public static final int TRACE_ID = 6; - public static final int SPAN_ID = 7; - public static final int TAGS = 8; - public static final int EVENTS = 9; - public static final int REOUSCES = 10; - public static final int REFERERNCES = 11; - // The total number of messages after splitting - public static final int COUNT = 12; - - // Redis keys used to cache serviceName and operationName - public static final String TRACE_SERVICE_REDIS_KEY = "trace_service_"; - // Cache the expiration time of the redis key of serviceName and operationName - public static final int TRACE_SERVICE_REDIS_KEY_EXPIRE = 60 * 60 * 24; -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/common/trace/TraceUtil.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/common/trace/TraceUtil.java deleted file mode 100644 index dc7aaf42c..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/common/trace/TraceUtil.java +++ /dev/null @@ -1,403 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.common.trace; - -import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONObject; -import com.google.common.collect.Sets; -import org.apache.ozhera.tspandata.TAttributeKey; -import org.apache.ozhera.tspandata.TAttributeType; -import org.apache.ozhera.tspandata.TAttributes; -import org.apache.ozhera.tspandata.TEvent; -import org.apache.ozhera.tspandata.TExtra; -import org.apache.ozhera.tspandata.TInstrumentationLibraryInfo; -import org.apache.ozhera.tspandata.TKind; -import org.apache.ozhera.tspandata.TLink; -import org.apache.ozhera.tspandata.TResource; -import org.apache.ozhera.tspandata.TSpanContext; -import org.apache.ozhera.tspandata.TSpanData; -import org.apache.ozhera.tspandata.TStatus; -import org.apache.ozhera.tspandata.TValue; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.thrift.TException; -import org.apache.thrift.TSerializer; -import org.apache.thrift.protocol.TCompactProtocol; -import org.apache.thrift.protocol.TProtocolFactory; -import org.apache.thrift.transport.TTransportException; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicReference; -import java.util.regex.Pattern; - -@Slf4j -public class TraceUtil { - - private static final TProtocolFactory PROTOCOL_FACTORY = new TCompactProtocol.Factory(); - private static final String TAG_KEY_SPAN_KIND = "span.kind"; - private static final String TAG_KEY_SERVICE_NAME = "service.name"; - private static final String SPAN_KIND_INTERNAL = "INTERNAL"; - private static final String TAG_KEY_IP = "ip"; - private static final String TAG_KEY_HOST = "host.name"; - private static final Set SPECIAL_TAG_KEYS = Sets.newHashSet(TAG_KEY_SPAN_KIND, - TAG_KEY_SERVICE_NAME, TAG_KEY_IP, TAG_KEY_HOST); - private static Pattern EMPTY_PATTERN = Pattern.compile("\\r\\n"); - - public static byte[] toBytes(String spanStr) { - try { - TSpanData tSpanData = toTSpanData(spanStr); - return toBytes(tSpanData); - } catch (Throwable ex) { - log.error("Failed to convert span to thrift,spanStr={}", spanStr, ex); - } - return null; - } - - public static byte[] toBytes(TSpanData tSpanData) { - if (tSpanData == null) { - return null; - } - - try { - TSerializer serializer = new TSerializer(PROTOCOL_FACTORY); - return serializer.serialize(tSpanData); - } catch (TTransportException e) { - log.error("Failed to convert span to thrift TTransportException", e); - } catch (TException e) { - log.error("Failed to convert span to thrift TException", e); - } - - return null; - } - - public static TSpanData toTSpanData(String spanStr) { - String message; - spanStr = EMPTY_PATTERN.matcher(spanStr).replaceAll(""); - if (spanStr.contains(" ||| ")) { - String[] messages = spanStr.split(" \\|\\|\\| "); - message = messages[1]; - } else { - message = spanStr.split(" \\| ")[1]; - } - String[] messageArray = message.split(MessageUtil.SPLIT); - // Bit check - if (messageArray.length != MessageUtil.COUNT) { - log.error("message count illegal : " + spanStr); - return null; - } - - return toTSpanData(messageArray); - } - - private static TSpanData toTSpanData(String[] array) { - TSpanData span = new TSpanData(); - span.setTraceId(array[MessageUtil.TRACE_ID]); - span.setSpanId(array[MessageUtil.SPAN_ID]); - span.setName(array[MessageUtil.SPAN_NAME]); - span.setStatus(toTStatus(array[MessageUtil.STATUS_CODE])); - span.setStartEpochNanos(Long.parseLong(array[MessageUtil.START_TIME])); - span.setEndEpochNanos(span.getStartEpochNanos() + Long.parseLong(array[MessageUtil.DURATION])); - Map specialAttrMap = new HashMap<>(); - span.setAttributes(toTAttributes(JSONArray.parseArray(decodeLineBreak(array[MessageUtil.TAGS])), - specialAttrMap)); - span.setTotalAttributeCount(span.getAttributes().getKeysSize()); - // using tags["span.kind"] as span kind - String spanKind = specialAttrMap.get(TAG_KEY_SPAN_KIND) == null ? SPAN_KIND_INTERNAL : specialAttrMap.get(TAG_KEY_SPAN_KIND).getStringValue(); - span.setKind(toTKind(spanKind)); - span.setEvents(toTEventList(JSONArray.parseArray(decodeLineBreak(array[MessageUtil.EVENTS])))); - span.setTotalRecordedEvents(span.getEventsSize()); - span.setResouce( - toTResource(JSONObject.parseObject(array[MessageUtil.REOUSCES]), specialAttrMap)); - span.setExtra(toTExtra(specialAttrMap)); - // using links["ref_type=CHILD_OF"] as parent span context and using left as links - AtomicReference parentSpanContextRef = new AtomicReference<>(); - span.setLinks( - toTLinkList(JSONArray.parseArray(array[MessageUtil.REFERERNCES]), parentSpanContextRef)); - span.setParentSpanContext(parentSpanContextRef.get()); - span.setTotalRecordedLinks(span.getLinksSize()); - return span; - } - - private static String decodeLineBreak(String value) { - if (StringUtils.isNotEmpty(value)) { - return value.replaceAll("\\\\","\\\\\\\\").replaceAll("##r'", "\\\\\"") - .replaceAll("##n", "\\\\n").replaceAll("##r", "\\\\r").replaceAll("##t", "\\\\t") - .replaceAll("##tat", "\\\\tat").replaceAll("##'", "\\\\\""); - } - return value; - } - - /** - * convert links. - */ - private static List toTLinkList(JSONArray links, - AtomicReference spanContextAtomicReference) { - if (links == null) { - return null; - } - List ret = new ArrayList<>(links.size()); - links.forEach(link -> { - JSONObject linkJson = (JSONObject) link; - if ("CHILD_OF".equals(linkJson.getString("refType"))) { - spanContextAtomicReference.set(toTSpanContext(linkJson)); - } else { - ret.add(toTLink(linkJson)); - } - }); - return ret; - } - - /** - * convert events. - */ - private static List toTEventList(JSONArray events) { - if (events == null) { - return null; - } - List ret = new ArrayList<>(events.size()); - events.forEach(event -> { - JSONObject eventJson = (JSONObject) event; - ret.add(toTEvent(eventJson)); - }); - return ret; - } - - /** - * convert span context. - */ - private static TSpanContext toTSpanContext(JSONObject ctx) { - if (ctx == null) { - return null; - } - TSpanContext spanContext = new TSpanContext(); - spanContext.setTraceId(ctx.getString("traceID")); - spanContext.setSpanId(ctx.getString("spanID")); - return spanContext; - } - - /** - * convert instrumentation lib info. - */ - private static TInstrumentationLibraryInfo toTInstrumentationLibraryInfo(String name, - String version) { - if (name == null && version == null) { - return null; - } - TInstrumentationLibraryInfo ret = new TInstrumentationLibraryInfo(); - ret.setName(name); - ret.setVersion(version); - return ret; - } - - /** - * convert resource. - */ - private static TResource toTResource(JSONObject resource, Map specialAttrMap) { - if (resource == null || !resource.containsKey("tags")) { - return null; - } - TResource ret = new TResource(); - ret.setAttributes(toTAttributes(resource.getJSONArray("tags"), specialAttrMap)); - return ret; - } - - /** - * convert attributes. - */ - private static TAttributes toTAttributes(JSONArray attributes) { - return toTAttributes(attributes, null); - } - - /** - * convert attributes. - */ - private static TAttributes toTAttributes(JSONArray attributes, - Map specialAttrMap) { - if (attributes == null) { - return null; - } - TAttributes ret = new TAttributes(); - ret.setKeys(new ArrayList<>()); - ret.setValues(new ArrayList<>()); - // attribute key put type in - attributes.forEach(attr -> { - JSONObject attrJson = (JSONObject) attr; - TAttributeKey attributeKey = toTAttributeKey(attrJson); - Object value = attrJson.get("value"); -// try { - TValue attributeValue = new TValue(); - switch (attributeKey.getType()) { - case LONG: - if(value instanceof String){ - attributeValue.setLongValue(Long.valueOf((String) value)); - } else { - attributeValue.setLongValue((Long) value); - } - break; - case DOUBLE: - if(value instanceof String){ - attributeValue.setDoubleValue(Double.valueOf((String) value)); - } else { - attributeValue.setDoubleValue((Double) value); - } - break; - case STRING: - attributeValue.setStringValue(String.valueOf(value)); - break; - case BOOLEAN: - if(value instanceof String){ - attributeValue.setBoolValue(Boolean.valueOf((String) value)); - } else { - attributeValue.setBoolValue((Boolean) value); - } - break; - } - if (specialAttrMap != null && SPECIAL_TAG_KEYS.contains(attributeKey.getValue())) { - specialAttrMap.put(attributeKey.getValue(), attributeValue); - } - ret.getKeys().add(attributeKey); - ret.getValues().add(attributeValue); -// } catch (Exception e) { -// log.error("Failed to add key '{}' value '{}' to attributes", attributeKey, value, e); -// } - }); - return ret; - } - - - /** - * convert attribute key. - */ - private static TAttributeKey toTAttributeKey(JSONObject attrJson) { - if (attrJson == null) { - return null; - } - TAttributeKey ret = new TAttributeKey(); - ret.setType(toTAttributeType(attrJson.getString("type"))); - ret.setValue(attrJson.getString("key")); - return ret; - } - - /** - * convert attribute type. - */ - private static TAttributeType toTAttributeType(String type) { - if (type == null) { - return null; - } - type = type.toUpperCase(); - switch (type) { - case "INT64": - return TAttributeType.LONG; - case "FLOAT64": - return TAttributeType.DOUBLE; - case "STRING": - return TAttributeType.STRING; - case "BOOL": - return TAttributeType.BOOLEAN; - } - return null; - } - - /** - * convert span status. - */ - private static TStatus toTStatus(String status) { - if (status == null) { - return null; - } - status = status.toUpperCase(); - switch (status) { - case "OK": - return TStatus.OK; - case "ERROR": - return TStatus.ERROR; - case "UNSET": - return TStatus.UNSET; - } - return null; - } - - /** - * convert span event. - */ - private static TEvent toTEvent(JSONObject event) { - if (event == null) { - return null; - } - TEvent ret = new TEvent(); - ret.setName(event.getString("name")); - if (event.containsKey("fields")) { - TAttributes attributes = toTAttributes(event.getJSONArray("fields")); - ret.setAttributes(attributes); - ret.setTotalAttributeCount(attributes.getKeysSize()); - } - ret.setEpochNanos(event.getLong("timestamp")); - return ret; - } - - /** - * convert span kind. - */ - private static TKind toTKind(String kind) { - if (kind == null) { - return null; - } - kind = kind.toUpperCase(); - switch (kind) { - case "CLIENT": - return TKind.CLIENT; - case "SERVER": - return TKind.SERVER; - case "CONSUMER": - return TKind.CONSUMER; - case "PRODUCER": - return TKind.PRODUCER; - default: - return TKind.INTERNAL; - } - } - - /** - * convert span link. - */ - private static TLink toTLink(JSONObject link) { - if (link == null) { - return null; - } - TLink ret = new TLink(); - ret.setSpanContext(toTSpanContext(link)); - return ret; - } - - private static TExtra toTExtra(Map specialAttrMap) { - TExtra tExtra = new TExtra(); - if (specialAttrMap.containsKey(TAG_KEY_SERVICE_NAME)) { - tExtra.setServiceName(specialAttrMap.get(TAG_KEY_SERVICE_NAME).getStringValue()); - } - if (specialAttrMap.containsKey(TAG_KEY_IP)) { - tExtra.setIp(specialAttrMap.get(TAG_KEY_IP).getStringValue()); - } - if (specialAttrMap.containsKey(TAG_KEY_HOST)) { - tExtra.setHostname(specialAttrMap.get(TAG_KEY_HOST).getStringValue()); - } - return tExtra; - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/exception/AgentException.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/exception/AgentException.java deleted file mode 100644 index 828c4cfc0..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/exception/AgentException.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.exception; - -/** - * @author shanwb - * @date 2021-07-20 - */ -public class AgentException extends RuntimeException { - public AgentException(String message, Throwable cause) { - super(message, cause); - } - - public AgentException(Throwable cause) { - super(cause); - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/export/MsgExporter.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/export/MsgExporter.java deleted file mode 100644 index beb43bac9..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/export/MsgExporter.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.export; - -import com.google.gson.Gson; -import com.xiaomi.mone.log.agent.channel.Closeable; -import com.xiaomi.mone.log.api.enums.LogTypeEnum; -import com.xiaomi.mone.log.api.model.msg.LineMessage; - -import java.util.List; - -/** - * @author shanwb - * @date 2021-07-19 - * @describe Exposed encapsulation of log messages - */ -public interface MsgExporter extends Closeable { - - String OPENTELEMETRY_TYPE = String.valueOf(LogTypeEnum.OPENTELEMETRY.getType()); - - Gson gson = new Gson(); - - int BATCH_EXPORT_SIZE = 200; - - /** - * Single export - * - * @param message - */ - void export(LineMessage message); - - /** - * Multiple exports - * - * @param messageList - */ - void export(List messageList); - - /** - * Batch export quantity - * - * @return - */ - default int batchExportSize() { - return BATCH_EXPORT_SIZE; - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/extension/KafkaExporter.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/extension/KafkaExporter.java deleted file mode 100644 index 38c202b6b..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/extension/KafkaExporter.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.extension; - -import com.google.common.collect.Lists; -import org.apache.ozhera.tspandata.TSpanData; -import com.xiaomi.mone.log.agent.common.HashUtil; -import com.xiaomi.mone.log.agent.common.trace.TraceUtil; -import com.xiaomi.mone.log.agent.export.MsgExporter; -import com.xiaomi.mone.log.api.enums.LogTypeEnum; -import com.xiaomi.mone.log.api.model.msg.LineMessage; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.PartitionInfo; - -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.concurrent.ThreadLocalRandom; - -import static com.xiaomi.mone.log.common.Constant.COMMON_MQ_PREFIX; - -@Slf4j -public class KafkaExporter implements MsgExporter { - - private Producer producer; - - private String topic; - - private String tag; - - private boolean isCommonTag; - - private Integer batchSize; - - private Integer maxPartitionPer = 4; - - public KafkaExporter(Producer mqProducer, String tag) { - this.producer = mqProducer; - this.tag = tag; - } - - @Override - public void close() { - - } - - @Override - public void export(LineMessage message) { - this.export(Lists.newArrayList(message)); - } - - @Override - public void export(List messageList) { - if (messageList.isEmpty()) { - return; - } - - List partitions = producer.partitionsFor(topic); - - for (LineMessage message : messageList) { - ProducerRecord record = buildProducerRecord(message, partitions); - if (record != null) { - producer.send(record, (RecordMetadata metadata, Exception e) -> { - if (null != e) { - log.error("send message to kafka error", e); - } - }); - } - } - } - - private ProducerRecord buildProducerRecord(LineMessage message, List partitions) { - String messageType = message.getProperties(LineMessage.KEY_MESSAGE_TYPE); - ProducerRecord record = null; - - if (String.valueOf(LogTypeEnum.ORIGIN_LOG.getType()).equals(messageType)) { - record = new ProducerRecord<>(topic, tag, gson.toJson(message.getMsgBody())); - } else if (OPENTELEMETRY_TYPE.equals(messageType)) { - String msgBody = message.getMsgBody(); - TSpanData tSpanData = TraceUtil.toTSpanData(msgBody); - - if (tSpanData != null) { - byte[] bytes = TraceUtil.toBytes(tSpanData); - if (bytes != null) { - String spanMessage = new String(bytes, StandardCharsets.ISO_8859_1); - record = new ProducerRecord<>(topic, tag, spanMessage); - - String appName = tSpanData.getExtra().getServiceName(); - if (appName != null) { - int key = ThreadLocalRandom.current().nextInt(maxPartitionPer); - appName = String.format("p%s%s", key, appName); - int hash = HashUtil.consistentHash(appName, partitions.size()); - int partition = partitions.get(hash).partition(); - record = new ProducerRecord<>(topic, partition, tag, spanMessage); - } - } - } - } else { - record = new ProducerRecord<>(topic, tag, gson.toJson(message)); - } - - return record; - } - - public void setTopic(String topic) { - this.topic = topic; - if (topic.startsWith(COMMON_MQ_PREFIX)) { - this.isCommonTag = true; - } - } - - public Integer getBatchSize() { - return batchSize; - } - - public void setBatchSize(Integer batchSize) { - this.batchSize = batchSize; - } - - @Override - public int batchExportSize() { - if (null == batchSize || batchSize < 0) { - return BATCH_EXPORT_SIZE; - } - - return batchSize; - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/extension/KafkaOutput.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/extension/KafkaOutput.java deleted file mode 100644 index de0f8da8d..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/extension/KafkaOutput.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.extension; - -import com.xiaomi.mone.log.agent.output.Output; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.io.Serializable; - - -@Data -@EqualsAndHashCode -public class KafkaOutput extends Output implements Serializable { - - public static final String OUTPUT_KAFKAMQ = "kafka"; - - private String serviceName = "KafkaService"; - - /** - * mq fill：namesrv_addr - */ - private String clusterInfo; - - private String producerGroup; - - private String orgId; - - private String ak; - - private String sk; - - private String topic; - - private Integer partitionCnt; - - private Integer batchExportSize; - - @Override - public String getEndpoint() { - return clusterInfo; - } - - @Override - public String getServiceName() { - return serviceName; - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/extension/KafkaService.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/extension/KafkaService.java deleted file mode 100644 index a9dcc062a..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/extension/KafkaService.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.extension; - -import com.google.common.base.Preconditions; -import com.xiaomi.mone.log.agent.export.MsgExporter; -import com.xiaomi.mone.log.agent.output.Output; -import com.xiaomi.mone.log.agent.service.OutPutService; -import com.xiaomi.mone.log.api.model.meta.LogPattern; -import com.xiaomi.mone.log.api.model.meta.MQConfig; -import com.xiaomi.mone.log.utils.KafkaUtils; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.plugin.config.anno.Value; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerConfig; - -import java.util.Objects; -import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; - -import static com.xiaomi.mone.log.common.Constant.DEFAULT_CONSUMER_GROUP; - - -@Service(name = "KafkaService") -@Slf4j -public class KafkaService implements OutPutService { - - private ConcurrentHashMap producerMap; - - @Value("$kafka.use.ssl") - private String kafkaUseSsl; - - @Value("$kafka.sll.location") - private String kafkaSllLocation; - - public void init() { - producerMap = new ConcurrentHashMap<>(128); - } - - @Override - public boolean compare(Output oldOutPut, Output newOutPut) { - if (!Objects.equals(oldOutPut.getOutputType(), newOutPut.getOutputType())) { - return false; - } - - // Assuming the OutputType is the discriminant property - if (oldOutPut instanceof KafkaOutput && newOutPut instanceof KafkaOutput) { - KafkaOutput oldRmqOutput = (KafkaOutput) oldOutPut; - KafkaOutput newRmqOutput = (KafkaOutput) newOutPut; - return oldRmqOutput.equals(newRmqOutput); - } - - // If not KafkaOutput, use general equals comparison - return Objects.equals(oldOutPut, newOutPut); - } - - @Override - public void preCheckOutput(Output output) { - KafkaOutput rmqOutput = (KafkaOutput) output; - Preconditions.checkArgument(null != rmqOutput.getClusterInfo(), "rmqOutput.getClusterInfo can not be null"); - Preconditions.checkArgument(null != rmqOutput.getTopic(), "rmqOutput.getTopic can not be null"); - } - - @Override - public MsgExporter exporterTrans(Output output) throws Exception { - KafkaOutput kafkaOutput = (KafkaOutput) output; - String nameSrvAddr = kafkaOutput.getClusterInfo(); - String key = getKey(nameSrvAddr, kafkaOutput.getTopic(), kafkaOutput.getTag()); - Producer mqProducer = producerMap.get(key); - if (null == mqProducer) { - mqProducer = initMqProducer(kafkaOutput); - producerMap.put(key, mqProducer); - } - - KafkaExporter rmqExporter = new KafkaExporter(mqProducer, output.getTag()); - rmqExporter.setTopic(kafkaOutput.getTopic()); - rmqExporter.setBatchSize(kafkaOutput.getBatchExportSize()); - - return rmqExporter; - } - - private String getKey(String nameSrvAddr, String topic, String tag) { - return String.format("%s-%s", nameSrvAddr, topic, tag); - } - - private Producer initMqProducer(KafkaOutput output) { - Properties properties = new Properties(); - - String clusterInfo = output.getClusterInfo(); - String ak = output.getAk(); - String sk = output.getSk(); - - if (StringUtils.isNotEmpty(ak) && StringUtils.isNotEmpty(sk) && Objects.equals("true", kafkaUseSsl)) { - properties.putAll(KafkaUtils.getSslKafkaProperties(clusterInfo, ak, sk, kafkaSllLocation)); - } else if (StringUtils.isNotEmpty(ak) && StringUtils.isNotEmpty(sk)) { - properties.putAll(KafkaUtils.getVpc9094KafkaProperties(clusterInfo, ak, sk)); - } else { - properties.putAll(KafkaUtils.getDefaultKafkaProperties(clusterInfo)); - } - - // set other properties - properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, clusterInfo); - properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); - properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); - properties.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, "lz4"); - return new KafkaProducer<>(properties); - } - - @Override - public void removeMQ(Output output) { - KafkaOutput kafkaOutput = (KafkaOutput) output; - String key = getKey(kafkaOutput.getClusterInfo(), kafkaOutput.getTopic(), kafkaOutput.getTag()); - if (null != producerMap.get(key)) { - producerMap.get(key).close(); - } - producerMap.remove(key); - } - - @Override - public Output configOutPut(LogPattern logPattern) { - - MQConfig mqConfig = logPattern.getMQConfig(); - KafkaOutput output = new KafkaOutput(); - output.setOutputType(KafkaOutput.OUTPUT_KAFKAMQ); - output.setClusterInfo(mqConfig.getClusterInfo()); - output.setProducerGroup(mqConfig.getProducerGroup()); - output.setAk(mqConfig.getAk()); - output.setSk(mqConfig.getSk()); - output.setTopic(mqConfig.getTopic()); - output.setPartitionCnt(mqConfig.getPartitionCnt()); - output.setTag(mqConfig.getTag()); - output.setProducerGroup(DEFAULT_CONSUMER_GROUP + (null == logPattern.getPatternCode() ? "" : logPattern.getPatternCode())); - return output; - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/extension/RmqExporter.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/extension/RmqExporter.java deleted file mode 100644 index 48e31fe0f..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/extension/RmqExporter.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.extension; - -import com.google.common.collect.Lists; -import com.google.gson.Gson; -import org.apache.ozhera.tspandata.TSpanData; -import com.xiaomi.mone.log.agent.common.HashUtil; -import com.xiaomi.mone.log.agent.common.trace.TraceUtil; -import com.xiaomi.mone.log.agent.export.MsgExporter; -import com.xiaomi.mone.log.api.model.msg.LineMessage; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.rocketmq.client.exception.MQBrokerException; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.producer.DefaultMQProducer; -import org.apache.rocketmq.common.message.Message; -import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.remoting.exception.RemotingException; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ThreadLocalRandom; - -/** - * Rocketmq Default message sending implementation class - * - * @author shanwb - * @date 2021-07-19 - */ -@Slf4j -public class RmqExporter implements MsgExporter { - - private DefaultMQProducer mqProducer; - - private String rmqTopic; - - private Integer batchSize; - - private Gson gson = new Gson(); - - - private List messageQueueList; - - public RmqExporter(DefaultMQProducer mqProducer) { - this.mqProducer = mqProducer; - } - - @Override - public void export(LineMessage message) { - this.export(Lists.newArrayList(message)); - } - - @Override - public void export(List messageList) { - if (messageList.isEmpty()) { - return; - } - this.messageQueueList = fetchMessageQueue(this.rmqTopic); - List logMessages = new ArrayList<>(); - Map> messageQueueListMap = new HashMap<>(); - - for (LineMessage lineMessage : messageList) { - if (OPENTELEMETRY_TYPE.equals(lineMessage.getProperties(LineMessage.KEY_MESSAGE_TYPE))) { - processOpenTelemetryMessage(lineMessage, messageQueueListMap); - } else { - processLogMessage(lineMessage, logMessages); - } - } - - sendMessagesToQueues(messageQueueListMap); - sendMessagesToProducer(logMessages); - } - - private void processOpenTelemetryMessage(LineMessage lineMessage, Map> messageQueueListMap) { - byte[] bytes = TraceUtil.toBytes(lineMessage.getMsgBody()); - if (bytes != null) { - TSpanData tSpanData = TraceUtil.toTSpanData(lineMessage.getMsgBody()); - String appName = tSpanData.getExtra().getServiceName(); - Message message = new Message(); - message.setBody(bytes); - message.setTopic(this.rmqTopic); - - // Calculate the message queue based on the appName - MessageQueue messageQueue = calculateMessageQueue(appName); - - messageQueueListMap.putIfAbsent(messageQueue, new ArrayList<>()); - messageQueueListMap.get(messageQueue).add(message); - } - } - - private void processLogMessage(LineMessage lineMessage, List logMessages) { - Message message = new Message(); - message.setTags(lineMessage.getProperties(LineMessage.KEY_MQ_TOPIC_TAG)); - message.setBody(gson.toJson(lineMessage).getBytes(StandardCharsets.UTF_8)); - message.setTopic(this.rmqTopic); - logMessages.add(message); - } - - private MessageQueue calculateMessageQueue(String appName) { - Integer partitionNumber = 2; - appName = String.format("p%s%s", ThreadLocalRandom.current().nextInt(partitionNumber), appName); - return messageQueueList.get(HashUtil.consistentHash(appName, messageQueueList.size())); - } - - private void sendMessagesToQueues(Map> messageQueueListMap) { - try { - for (Map.Entry> queueListEntry : messageQueueListMap.entrySet()) { - List messages = queueListEntry.getValue(); - if (CollectionUtils.isNotEmpty(messages)) { - mqProducer.send(messages, queueListEntry.getKey()); - } - } - } catch (MQClientException | RemotingException | MQBrokerException | InterruptedException e) { - log.error("OPENTELEMETRY log rocketMQ export error", e); - } - } - - private void sendMessagesToProducer(List logMessages) { - try { - if (CollectionUtils.isNotEmpty(logMessages)) { - mqProducer.send(logMessages); - } - } catch (MQClientException | RemotingException | MQBrokerException | InterruptedException e) { - log.error("normal rocketMQ export error", e); - } - } - - public String getRmqTopic() { - return rmqTopic; - } - - public void setRmqTopic(String rmqTopic) { - this.messageQueueList = fetchMessageQueue(rmqTopic); - this.rmqTopic = rmqTopic; - } - - public void setBatchSize(Integer batchSize) { - this.batchSize = batchSize; - } - - @Override - public int batchExportSize() { - if (null == batchSize || batchSize < 0) { - return BATCH_EXPORT_SIZE; - } - - return batchSize; - } - - - @Override - public void close() { - //mqProducer multi-topic public and cannot shutdown(); - } - - - public List fetchMessageQueue(String topicName) { - try { - return mqProducer.fetchPublishMessageQueues(topicName); - } catch (MQClientException e) { - log.error("fetch queue task error : ", e); - } - return new ArrayList<>(); - } - -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/extension/RmqOutput.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/extension/RmqOutput.java deleted file mode 100644 index ed5c871a3..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/extension/RmqOutput.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.extension; - -import com.xiaomi.mone.log.agent.output.Output; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.io.Serializable; - -/** - * @author shanwb - * @date 2021-07-20 - */ -@Data -@EqualsAndHashCode -public class RmqOutput extends Output implements Serializable { - - public static final String OUTPUT_ROCKETMQ = "rocketmq"; - - private String serviceName = "RocketMQService"; - - /** - * mq fill：namesrv_addr - */ - private String clusterInfo; - - private String producerGroup; - - private String orgId; - - private String ak; - - private String sk; - - private String topic; - - private Integer partitionCnt; - - private Integer batchExportSize; - - @Override - public String getEndpoint() { - return clusterInfo; - } - - @Override - public String getServiceName() { - return serviceName; - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/extension/RocketMQService.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/extension/RocketMQService.java deleted file mode 100644 index 62082004c..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/extension/RocketMQService.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.extension; - -import com.google.common.base.Preconditions; -import com.xiaomi.mone.log.agent.exception.AgentException; -import com.xiaomi.mone.log.agent.export.MsgExporter; -import com.xiaomi.mone.log.agent.output.Output; -import com.xiaomi.mone.log.agent.service.OutPutService; -import com.xiaomi.mone.log.api.model.meta.LogPattern; -import com.xiaomi.mone.log.api.model.meta.MQConfig; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.rocketmq.acl.common.AclClientRPCHook; -import org.apache.rocketmq.acl.common.SessionCredentials; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.producer.DefaultMQProducer; -import org.apache.rocketmq.remoting.RPCHook; - -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; - -import static com.xiaomi.mone.log.common.Constant.DEFAULT_CONSUMER_GROUP; - -/** - * @Description - * @Author dingtao - * @Date 2023/4/7 9:44 AM - */ -@Service(name = "RocketMQService") -@Slf4j -public class RocketMQService implements OutPutService { - - private ConcurrentHashMap producerMap; - - public void init() { - producerMap = new ConcurrentHashMap<>(128); - } - - @Override - public boolean compare(Output oldOutput, Output newOutput) { - // Check if output types are equal - if (!Objects.equals(oldOutput.getOutputType(), newOutput.getOutputType())) { - return false; - } - - // Check if both are instances of RmqOutput - if (oldOutput instanceof RmqOutput && newOutput instanceof RmqOutput) { - RmqOutput oldRmqOutput = (RmqOutput) oldOutput; - RmqOutput newRmqOutput = (RmqOutput) newOutput; - - // Use equals method for detailed comparison - return oldRmqOutput.equals(newRmqOutput); - } - - // If not both instances of RmqOutput, consider them not equal - return false; - } - - @Override - public void preCheckOutput(Output output) { - RmqOutput rmqOutput = (RmqOutput) output; - Preconditions.checkArgument(null != rmqOutput.getClusterInfo(), "rmqOutput.getClusterInfo can not be null"); - Preconditions.checkArgument(null != rmqOutput.getTopic(), "rmqOutput.getTopic can not be null"); - Preconditions.checkArgument(null != rmqOutput.getProducerGroup(), "rmqOutput.getProducerGroup can not be null"); - } - - @Override - public MsgExporter exporterTrans(Output output) { - RmqOutput rmqOutput = (RmqOutput) output; - String nameSrvAddr = rmqOutput.getClusterInfo(); - DefaultMQProducer mqProducer = producerMap.get(nameSrvAddr); - if (null == mqProducer) { - mqProducer = initMqProducer(rmqOutput); - producerMap.put(String.valueOf(nameSrvAddr), mqProducer); - } - - RmqExporter rmqExporter = new RmqExporter(mqProducer); - rmqExporter.setRmqTopic(rmqOutput.getTopic()); - rmqExporter.setBatchSize(rmqOutput.getBatchExportSize()); - - return rmqExporter; - } - - @Override - public void removeMQ(Output output) { - } - - @Override - public Output configOutPut(LogPattern logPattern) { - MQConfig mqConfig = logPattern.getMQConfig(); - RmqOutput output = new RmqOutput(); - output.setOutputType(RmqOutput.OUTPUT_ROCKETMQ); - output.setClusterInfo(mqConfig.getClusterInfo()); - output.setProducerGroup(mqConfig.getProducerGroup()); - output.setAk(mqConfig.getAk()); - output.setSk(mqConfig.getSk()); - output.setTopic(mqConfig.getTopic()); - output.setPartitionCnt(mqConfig.getPartitionCnt()); - output.setTag(mqConfig.getTag()); - output.setProducerGroup(DEFAULT_CONSUMER_GROUP + (null == logPattern.getPatternCode() ? "" : logPattern.getPatternCode())); - return output; - } - - private DefaultMQProducer initMqProducer(RmqOutput rmqOutput) { - DefaultMQProducer producer; - if (StringUtils.isNotEmpty(rmqOutput.getAk()) && StringUtils.isNotEmpty(rmqOutput.getSk())) { - RPCHook rpcHook = new AclClientRPCHook(new SessionCredentials(rmqOutput.getAk(), rmqOutput.getSk())); - producer = new DefaultMQProducer(rmqOutput.getProducerGroup() + "x", rpcHook, true, null); - } else { - producer = new DefaultMQProducer(rmqOutput.getProducerGroup() + "x", true); - } - producer.setNamesrvAddr(rmqOutput.getClusterInfo()); - try { - producer.start(); - return producer; - } catch (MQClientException e) { - log.error("ChannelBootstrap.initMqProducer error, RocketmqConfig: {}", rmqOutput, e); - throw new AgentException("initMqProducer exception", e); - } - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/factory/OutPutServiceFactory.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/factory/OutPutServiceFactory.java deleted file mode 100644 index 793480912..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/factory/OutPutServiceFactory.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.factory; - -import com.xiaomi.mone.log.agent.service.ChannelDefineLocatorExtension; -import com.xiaomi.mone.log.agent.service.OutPutService; -import com.xiaomi.mone.log.common.Config; -import com.xiaomi.youpin.docean.Ioc; - -/** - * @Description OutPutServiceFactory Get the factory class of ChannelDefineLocatorExtension - * @Author dingtao - * @Date 2023/4/7 9:59 AM - */ -public class OutPutServiceFactory { - - private static String defaultServiceName; - - private static final String DEFAULT_CHANNEL_DEFINE_LOCATOR_EXTENSION = "OuterChannelDefineLocatorExtensionImpl"; - - static { - defaultServiceName = Config.ins().get("default.output.service", "RocketMQService"); - } - - public static OutPutService getOutPutService(String serviceName) { - OutPutService bean = Ioc.ins().getBean(serviceName); - return bean == null ? Ioc.ins().getBean(defaultServiceName) : bean; - } - - public static ChannelDefineLocatorExtension getChannelDefineLocatorExtension() { - String channelDefineLocatorExtensionServiceName = Config.ins().get("channel.define.locator.extension", DEFAULT_CHANNEL_DEFINE_LOCATOR_EXTENSION); - return Ioc.ins().getBean(channelDefineLocatorExtensionServiceName); - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/filter/FilterChain.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/filter/FilterChain.java deleted file mode 100644 index dc37d9b91..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/filter/FilterChain.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.filter; - -import com.xiaomi.mone.log.agent.filter.ratelimit.RatelimitFilter; -import com.xiaomi.mone.log.api.model.meta.FilterConf; -import com.xiaomi.mone.log.api.model.meta.FilterType; -import lombok.extern.slf4j.Slf4j; - -import java.util.List; -import java.util.Objects; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -/** - * Filter chain implementation class - */ -@Slf4j -public class FilterChain { - private CopyOnWriteArrayList filterList = new CopyOnWriteArrayList<>(); - - private ReadWriteLock lock = new ReentrantReadWriteLock(); - - private volatile Invoker invoker; - - public void reset() { - lock.writeLock().lock(); - try { - Invoker curr = () -> { - }; - int size = filterList.size(); - // The element that comes before is executed later, i.e. the larger the order, the later it is executed - for (int i = 0; i < size; i++) { - MilogFilter filter = filterList.get(i); - Invoker last = curr; - curr = () -> filter.doFilter(last); - } - this.invoker = curr; - } catch (Exception e) { - log.error("Filter Chain reload err", e); - } finally { - lock.writeLock().unlock(); - } - - } - - public void loadFilterList(List confs) { - if (confs == null || confs.size() < 1) { - return; - } - List sortedConf = confs.stream().filter(Objects::nonNull).sorted((a, b) -> { - Integer x = a.getOrder(); - Integer y = b.getOrder(); - // The order value is large first - return y.compareTo(x); - }).sorted((a, b) -> { - // The filter for the global attribute comes last - if (FilterType.GLOBAL.equals(a.getType()) && FilterType.GLOBAL.equals(b.getType())) { - return 0; - } else if (FilterType.GLOBAL.equals(a.getType())) { - return 1; - } else if (FilterType.GLOBAL.equals(b.getType())) { - return -1; - } else { - return 0; - } - }).toList(); - CopyOnWriteArrayList filters = new CopyOnWriteArrayList<>(); - for (FilterConf conf : sortedConf) { - switch (conf.getName()) { - case RATELIMITER: - RatelimitFilter ratelimitFilter = new RatelimitFilter(); - if (ratelimitFilter.init(conf)) { - filters.add(ratelimitFilter); - } - break; - default: - break; - } - } - this.filterList = filters; - } - - public void doFilter() { - try { - lock.readLock().lock(); - this.invoker.doInvoker(); - } finally { - lock.readLock().unlock(); - } - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/filter/FilterTrans.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/filter/FilterTrans.java deleted file mode 100644 index 92d9515c2..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/filter/FilterTrans.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.filter; - -import com.xiaomi.mone.log.api.filter.Common; -import com.xiaomi.mone.log.api.filter.RateLimitStrategy; -import com.xiaomi.mone.log.api.model.meta.FilterConf; -import com.xiaomi.mone.log.api.model.meta.FilterDefine; -import com.xiaomi.mone.log.api.model.meta.FilterName; - -import java.util.Map; - -/** - * Filter packaging conversion class - */ -public class FilterTrans { - public static FilterConf filterConfTrans(FilterDefine define) { - if (define == null) { - return null; - } - if (define.getCode() != null) { - if (define.getCode().startsWith(Common.RATE_LIMIT_CODE)) { - RateLimitStrategy rateLimiterStrategy = RateLimitStrategy.getRateLimiterStrategy(define.getCode()); - if (rateLimiterStrategy != null) { - return consFilterConf(FilterName.RATELIMITER, rateLimiterStrategy, define.getArgs()); - } - } - } - return null; - } - - private static FilterConf consFilterConf(FilterName name, RateLimitStrategy strategy, Map args) { - return new FilterConf(strategy.getCode(), name, strategy.getType(), strategy.getOrder(), strategy.getLifecycle(), args); - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/filter/Invoker.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/filter/Invoker.java deleted file mode 100644 index 227cdc832..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/filter/Invoker.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.filter; - -public interface Invoker { - void doInvoker(); -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/filter/MilogFilter.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/filter/MilogFilter.java deleted file mode 100644 index 3c3f52b5e..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/filter/MilogFilter.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.filter; - - -import com.xiaomi.mone.log.api.model.meta.FilterConf; - -/** - * rate filtering package - */ -public interface MilogFilter { - void doFilter(Invoker next); - - boolean init(FilterConf conf); -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/filter/ratelimit/RatelimitFilter.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/filter/ratelimit/RatelimitFilter.java deleted file mode 100644 index b13f6524c..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/filter/ratelimit/RatelimitFilter.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.filter.ratelimit; - -import com.google.common.util.concurrent.RateLimiter; -import com.xiaomi.mone.log.agent.filter.Invoker; -import com.xiaomi.mone.log.agent.filter.MilogFilter; -import com.xiaomi.mone.log.api.filter.Common; -import com.xiaomi.mone.log.api.model.meta.FilterConf; -import lombok.extern.slf4j.Slf4j; - -import java.util.Map; - -/** - * Limit the log agent sending log rate - */ -@Slf4j -public class RatelimitFilter implements MilogFilter { - private RateLimiter rateLimiter; - - @Override - public void doFilter(Invoker invoker) { - this.rateLimiter.acquire(); - if (invoker != null) { - invoker.doInvoker(); - } else { - log.error("do filter but next invoker is null"); - } - } - - @Override - public boolean init(FilterConf conf) { - Map args = conf.getArgs(); - if (args != null) { - String ppsStr = args.get(Common.PERMITS_PER_SECOND); - try { - int pps = Integer.parseInt(ppsStr); - this.rateLimiter = RateLimiter.create(pps); - return true; - } catch (Exception e) { - log.error("init rateLimit err PERMITSPERSECOND:{},FilterConf:{}", ppsStr, conf, e); - } - } else { - log.warn("wrong rateLimit args FilterConf:{}", conf); - } - return false; - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/input/AppLogInput.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/input/AppLogInput.java deleted file mode 100644 index 0f0d2ea2d..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/input/AppLogInput.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.input; - -import java.io.Serializable; - -/** - * Apply log type input - * - * @author shanwb - * @date 2021-07-19 - */ -public class AppLogInput extends Input implements Serializable { - - public AppLogInput() { - super(); - } - -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/input/FreeLogInput.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/input/FreeLogInput.java deleted file mode 100644 index 43ed4241a..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/input/FreeLogInput.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.input; - -import java.io.Serializable; - -public class FreeLogInput extends Input implements Serializable { - public FreeLogInput() { - super(); - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/input/Input.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/input/Input.java deleted file mode 100644 index 3b069adb6..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/input/Input.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.input; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -/** - * @author shanwb - * @date 2021-07-19 - */ -@Data -@EqualsAndHashCode -public abstract class Input { - /** - * LogTypeEnum.name()， - * You can decide whether to collect single lines or multiple lines based on this type (supports java exception stack) - * - * @see com.xiaomi.mone.log.api.enums.LogTypeEnum - */ - private String type; - - /** - * log path format - * Currently supports single-file, multi-file, and directory-level wildcard modes - *

- * single file: /home/work/log/xxapp/server.log - * multiple files: /home/work/log/neo-logs/(cxx01|cxx022)/server.log - * Directory level wildcarding:/home/work/log/xxapp/ * /server.log - */ - private String logPattern; - - /** - * - */ - private String patternCode; - - private String logSplitExpress; - /** - * User-defined regularity at the beginning of the line, the default is "" - */ - private String linePrefix; - -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/input/MisAppLogInput.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/input/MisAppLogInput.java deleted file mode 100644 index 2513a6a2d..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/input/MisAppLogInput.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.input; - -import java.io.Serializable; - -/** - * mis application log type input - * - * @author shanwb - * @date 2021-11-23 - */ -public class MisAppLogInput extends Input implements Serializable { - - public MisAppLogInput() { - super(); - } - -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/input/NginxInput.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/input/NginxInput.java deleted file mode 100644 index 8ba120de1..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/input/NginxInput.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.input; - -import java.io.Serializable; - -/** - * nginx type input - * - * @author shanwb - * @date 2021-07-19 - */ -public class NginxInput extends Input implements Serializable { - - public NginxInput() { - super(); - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/input/OpentelemetryInput.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/input/OpentelemetryInput.java deleted file mode 100644 index 2be223862..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/input/OpentelemetryInput.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.input; - -import java.io.Serializable; - -/** - * opentelemetry type input - * - * @author shanwb - * @date 2021-09-13 - */ -public class OpentelemetryInput extends Input implements Serializable { - - public OpentelemetryInput() { - super(); - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/input/OriginLogInput.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/input/OriginLogInput.java deleted file mode 100644 index 5aacd0512..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/input/OriginLogInput.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.input; - -import java.io.Serializable; - -/** - * Log type input as is - * This type of log is only responsible for collecting mq as it is, not for consumption. - * - * @author shanwb - * @date 2022-12-23 - */ -public class OriginLogInput extends Input implements Serializable { - - public OriginLogInput() { - super(); - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/output/Output.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/output/Output.java deleted file mode 100644 index f4a8e9c36..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/output/Output.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.output; - -import lombok.EqualsAndHashCode; - -import java.io.Serializable; - -/** - * @author shanwb - * @date 2021-07-20 - */ -@EqualsAndHashCode -public abstract class Output implements Serializable { - - /** - * rocketmq、talos - */ - private String type; - - private String tag; - - public String getOutputType() { - return type; - } - - public void setOutputType(String outputType) { - this.type = outputType; - } - - public String getTag() { - return tag; - } - - public void setTag(String tag) { - this.tag = tag; - } - - /** - * getEndpoint - * @return - */ - public abstract String getEndpoint(); - - public abstract String getServiceName(); -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/rpc/processor/IService.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/rpc/processor/IService.java deleted file mode 100644 index 611c52895..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/rpc/processor/IService.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.rpc.processor; - -/** - * @author goodjava@qq.com - * @date 2020/6/18 - */ -public interface IService { - - /** - * Service initialization - */ - void init(); -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/rpc/processor/LogProcessor.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/rpc/processor/LogProcessor.java deleted file mode 100644 index f0c59211d..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/rpc/processor/LogProcessor.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.rpc.processor; - -import com.xiaomi.data.push.rpc.netty.NettyRequestProcessor; -import com.xiaomi.data.push.rpc.protocol.RemotingCommand; -import com.xiaomi.mone.log.agent.channel.ChannelDefine; -import com.xiaomi.mone.log.agent.channel.ChannelEngine; -import com.xiaomi.mone.log.agent.channel.locator.ChannelDefineRpcLocator; -import com.xiaomi.mone.log.api.model.meta.LogCollectMeta; -import com.xiaomi.mone.log.api.model.vo.LogCmd; -import com.xiaomi.youpin.docean.Ioc; -import com.xiaomi.youpin.docean.anno.Component; -import io.netty.channel.ChannelHandlerContext; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; - -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantLock; - -import static com.xiaomi.mone.log.common.Constant.GSON; - -/** - * @author goodjava@qq.com - */ -@Slf4j -@Component -public class LogProcessor implements NettyRequestProcessor { - - private ReentrantLock lock = new ReentrantLock(); - - @Override - public RemotingCommand processRequest(ChannelHandlerContext channelHandlerContext, RemotingCommand remotingCommand) throws Exception { - LogCollectMeta req = remotingCommand.getReq(LogCollectMeta.class); - - log.info("logCollect config req:{}", GSON.toJson(req)); - - RemotingCommand response = RemotingCommand.createResponseCommand(LogCmd.logRes); - response.setBody("ok".getBytes()); - log.info("【config change】receive data：{}", GSON.toJson(req)); - metaConfigEffect(req); - log.info("config change success"); - return response; - } - - private void metaConfigEffect(LogCollectMeta req) { - try { - lock.lock(); - ChannelEngine channelEngine = Ioc.ins().getBean(ChannelEngine.class); - // Whether the initialization is completed or not, wait for 30 s before executing - int count = 0; - while (true) { - if (!channelEngine.isInitComplete()) { - try { - TimeUnit.SECONDS.sleep(5L); - ++count; - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - if (channelEngine.isInitComplete() || count >= 20) { - break; - } - } - if (CollectionUtils.isNotEmpty(req.getAppLogMetaList())) { - try { - List channelDefines = ChannelDefineRpcLocator.agentTail2ChannelDefine(ChannelDefineRpcLocator.logCollectMeta2ChannelDefines(req)); - channelEngine.refresh(channelDefines); - } catch (Exception e) { - log.error("refresh config error,req:{}", GSON.toJson(req), e); - } - } - } finally { - lock.unlock(); - } - } - - @Override - public boolean rejectRequest() { - return false; - } - - - @Override - public int cmdId() { - return LogCmd.logReq; - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/rpc/processor/ProcessorInit.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/rpc/processor/ProcessorInit.java deleted file mode 100644 index 72b3e1105..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/rpc/processor/ProcessorInit.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.rpc.processor; - -import com.xiaomi.data.push.rpc.RpcClient; -import com.xiaomi.data.push.rpc.common.Pair; -import com.xiaomi.data.push.rpc.netty.NettyRequestProcessor; -import com.xiaomi.youpin.docean.Ioc; -import com.xiaomi.youpin.docean.anno.Component; - -import javax.annotation.Resource; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * @author goodjava@qq.com - * @date 2020/6/20 - */ -@Component(desc = "processor service") -public class ProcessorInit implements IService { - - @Resource - private RpcClient client; - - public ProcessorInit(RpcClient client) { - this.client = client; - } - - public ProcessorInit() { - } - - @Override - public void init() { - Set set = Ioc.ins().getBeans(NettyRequestProcessor.class); - client.setProcessorList(set.stream().map(it-> new Pair<>(it.cmdId(),it)).collect(Collectors.toList())); - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/rpc/task/PingTask.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/rpc/task/PingTask.java deleted file mode 100644 index b5eeb91a1..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/rpc/task/PingTask.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.rpc.task; - -import com.xiaomi.data.push.rpc.RpcClient; -import com.xiaomi.data.push.rpc.RpcCmd; -import com.xiaomi.data.push.rpc.netty.ResponseFuture; -import com.xiaomi.data.push.rpc.protocol.RemotingCommand; -import com.xiaomi.data.push.task.Task; -import com.xiaomi.mone.log.api.model.meta.AppLogMeta; -import com.xiaomi.mone.log.api.model.vo.PingReq; -import com.xiaomi.mone.log.utils.NetUtil; -import lombok.extern.slf4j.Slf4j; - -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; - -import static com.xiaomi.mone.log.common.Constant.GSON; - -/** - * @author goodjava@qq.com - */ -@Slf4j -public class PingTask extends Task { - - private static final AtomicBoolean docker = new AtomicBoolean(false); - - public static final AtomicBoolean stop = new AtomicBoolean(false); - - private static final AtomicBoolean load = new AtomicBoolean(false); - - private static final AtomicReference aip = new AtomicReference<>(""); - - private static final String IP = NetUtil.getLocalIp(); - - - public PingTask(RpcClient client) { - super(() -> { - if (stop.get()) { - return; - } - try { - PingReq ping = new PingReq(); - ping.setIp(IP); - String message = load.get() ? "ping:" + System.currentTimeMillis() : "load"; - ping.setMessage(message); - - RemotingCommand req = RemotingCommand.createRequestCommand(RpcCmd.pingReq); - req.setBody(GSON.toJson(ping).getBytes()); - for (String service : client.getServerList().get()) { - client.sendMessage(service, req, PingTask::handleResponse); - } - RpcClient.startLatch.countDown(); - } catch (Exception ex) { - log.error("ping error:{}", ex.getMessage()); - } - }, 10); - } - - private static void handleResponse(ResponseFuture responseFuture) { - if (responseFuture.getResponseCommand() == null) { - return; - } - String body = new String(responseFuture.getResponseCommand().getBody()); - if (!load.get()) { - AppLogMeta alm = GSON.fromJson(body, AppLogMeta.class); - load.set(true); - log.info("load config finish: {}", alm); - } - log.info("ping res: {}", body); - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/ChannelDefineLocatorExtension.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/ChannelDefineLocatorExtension.java deleted file mode 100644 index 63ee88740..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/ChannelDefineLocatorExtension.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.service; - -import com.xiaomi.mone.log.agent.output.Output; -import com.xiaomi.mone.log.api.model.meta.LogPattern; - -public interface ChannelDefineLocatorExtension { - - Output getOutPutByMQConfigType(LogPattern logPattern); -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/OutPutService.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/OutPutService.java deleted file mode 100644 index f92341069..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/OutPutService.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.service; - -import com.xiaomi.mone.log.agent.export.MsgExporter; -import com.xiaomi.mone.log.agent.output.Output; -import com.xiaomi.mone.log.api.model.meta.LogPattern; - -public interface OutPutService { - /** - * comparison of old and new configurations to determine the life of production - * - * @param oldOutput - * @param newOutput - * @return - */ - boolean compare(Output oldOutput, Output newOutput); - - /** - * configuration check - * - * @param output - */ - void preCheckOutput(Output output); - - /** - * initialize message exposer - * - * @param output - * @return - * @throws Exception - */ - MsgExporter exporterTrans(Output output) throws Exception; - - /** - * remove message producer based on configuration - * - * @param output - */ - void removeMQ(Output output); - - /** - * get Output according to Log Pattern - * - * @param logPattern - * @return - */ - Output configOutPut(LogPattern logPattern); -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/ServiceRegistryService.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/ServiceRegistryService.java deleted file mode 100644 index c72d68a1f..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/ServiceRegistryService.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.xiaomi.mone.log.agent.service; - -import com.alibaba.nacos.api.naming.pojo.Instance; -import com.xiaomi.data.push.nacos.NacosNaming; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/12/19 14:15 - */ - -public interface ServiceRegistryService { - - void init(); - - void registerServiceInstance(); - - void addShutdownHook(NacosNaming nacosNaming, String serviceName); - - Instance buildInstance(String serviceName); - - void initializeEnvironmentParameters(); -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/impl/OuterChannelDefineLocatorExtensionImpl.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/impl/OuterChannelDefineLocatorExtensionImpl.java deleted file mode 100644 index 495030c05..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/impl/OuterChannelDefineLocatorExtensionImpl.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.service.impl; - -import com.xiaomi.mone.log.agent.factory.OutPutServiceFactory; -import com.xiaomi.mone.log.agent.output.Output; -import com.xiaomi.mone.log.agent.service.ChannelDefineLocatorExtension; -import com.xiaomi.mone.log.api.enums.MQSourceEnum; -import com.xiaomi.mone.log.api.model.meta.LogPattern; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; - -/** - * @Description - * @Author dingtao - * @Date 2023/4/27 12:20 PM - */ -@Service(name = "OuterChannelDefineLocatorExtensionImpl") -@Slf4j -public class OuterChannelDefineLocatorExtensionImpl implements ChannelDefineLocatorExtension { - @Override - public Output getOutPutByMQConfigType(LogPattern logPattern) { - if (null != logPattern.getMQConfig()) { - String typeName = logPattern.getMQConfig().getType(); - MQSourceEnum middlewareEnum = MQSourceEnum.queryByName(typeName); - if (null != middlewareEnum) { - return OutPutServiceFactory.getOutPutService(middlewareEnum.getServiceName()).configOutPut(logPattern); - } - } - return null; - } -} diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/impl/ServiceRegistryServiceImpl.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/impl/ServiceRegistryServiceImpl.java deleted file mode 100644 index 681c0a96e..000000000 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/impl/ServiceRegistryServiceImpl.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.xiaomi.mone.log.agent.service.impl; - -import com.alibaba.nacos.api.exception.NacosException; -import com.alibaba.nacos.api.naming.pojo.Instance; -import com.google.common.collect.Maps; -import com.xiaomi.data.push.common.SafeRun; -import com.xiaomi.data.push.nacos.NacosNaming; -import com.xiaomi.data.push.rpc.RpcClient; -import com.xiaomi.mone.log.agent.service.ServiceRegistryService; -import com.xiaomi.mone.log.common.Config; -import com.xiaomi.mone.log.utils.NetUtil; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; -import java.net.InetAddress; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Map; -import java.util.Objects; - -import static com.xiaomi.mone.log.common.Constant.STREAM_CONTAINER_POD_NAME_KEY; -import static com.xiaomi.mone.log.utils.ConfigUtils.getDataHashKey; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/12/19 14:15 - */ -@Service -@Slf4j -public class ServiceRegistryServiceImpl implements ServiceRegistryService { - - @Resource - private RpcClient rpcClient; - - private final static String SERVER_PREFIX = "prometheus_server"; - - private final static String APP_NAME_LABEL = "app_name"; - - private final static String APP_ID_LABEL = "app_id"; - - private final static String ENV_ID_LABEL = "env_id"; - - private final static String ENV_NAME_LABEL = "env_name"; - - private static final String DEFAULT_TIME_DATE_FORMAT = "yyyy-MM-dd hh:mm:ss"; - - public static final String STREAM_VERSION = "hera-log-agent:1.0.0:2023-12-20"; - - private String appName; - - private String appId; - - private String envId; - - private String envName; - - private Integer port; - - private String ip; - - @Override - public void init() { - this.initializeEnvironmentParameters(); - String registrationInitiationFlag = Config.ins().get("registration_initiation_flag", "false"); - if (Objects.equals("true", registrationInitiationFlag)) { - this.registerServiceInstance(); - } - } - - @Override - public void registerServiceInstance() { - NacosNaming nacosNaming = rpcClient.getNacosNaming(); - int appIndex = getDataHashKey(ip, Integer.parseInt(Config.ins().get("app_max_index", "30"))); - String serviceName = String.format("%s_%s_%s_%s", SERVER_PREFIX, appId, appName, appIndex); - - try { - nacosNaming.registerInstance(serviceName, buildInstance(serviceName)); - addShutdownHook(nacosNaming, serviceName); - } catch (NacosException e) { - log.error("registerService error,serviceName:{}", serviceName, e); - } - } - - @Override - public void addShutdownHook(NacosNaming nacosNaming, String serviceName) { - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { - log.info("agent unregisters the instance and the service name:{}", serviceName); - nacosNaming.deregisterInstance(serviceName, ip, port); - } catch (NacosException e) { - log.error("agent unregisters the instance error,service name:{}", serviceName, e); - } - })); - } - - @Override - public Instance buildInstance(String serviceName) { - Instance instance = new Instance(); - instance.setEnabled(true); - instance.setHealthy(true); - instance.setIp(ip); - instance.setPort(port); - instance.setServiceName(serviceName); - - Map metaData = Maps.newHashMap(); - metaData.put("ctime", new SimpleDateFormat(DEFAULT_TIME_DATE_FORMAT).format(new Date())); - metaData.put("version", STREAM_VERSION); - metaData.put(STREAM_CONTAINER_POD_NAME_KEY, System.getenv(STREAM_CONTAINER_POD_NAME_KEY)); - metaData.put(ENV_ID_LABEL, envId); - metaData.put(ENV_NAME_LABEL, envName); - - SafeRun.run(() -> metaData.put("hostname", InetAddress.getLocalHost().getHostName())); - instance.setMetadata(metaData); - - return instance; - } - - @Override - public void initializeEnvironmentParameters() { - appName = StringUtils.isNotBlank(System.getenv(APP_NAME_LABEL)) ? System.getenv(APP_NAME_LABEL) : "log_agent"; - appId = StringUtils.isNotBlank(System.getenv(APP_ID_LABEL)) ? System.getenv(APP_ID_LABEL) : "10010"; - envName = StringUtils.isNotBlank(System.getenv(ENV_NAME_LABEL)) ? System.getenv(ENV_NAME_LABEL) : "default_env"; - envId = StringUtils.isNotBlank(System.getenv(ENV_ID_LABEL)) ? System.getenv(ENV_ID_LABEL) : "1"; - port = Integer.parseInt(Config.ins().get("port", "9799")); - ip = NetUtil.getLocalIp(); - } -} diff --git a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/Comparator.java b/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/Comparator.java deleted file mode 100644 index d6b2a4a5e..000000000 --- a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/Comparator.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent; - -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.RateLimiter; -import com.google.gson.Gson; -import com.xiaomi.mone.log.agent.channel.comparator.OutputSimilarComparator; -import com.xiaomi.mone.log.agent.extension.RmqOutput; -import lombok.extern.slf4j.Slf4j; -import org.junit.Test; - -import java.io.Serializable; -import java.util.concurrent.*; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/8/29 19:52 - */ -@Slf4j -public class Comparator { - - private Gson gson = new Gson(); - - @Test - public void testCompare() { - String msg1 = "{\"clusterInfo\":\"http://127.0.0.1\",\"producerGroup\":\"subGroup_tags_1_4_620\",\"ak\":\"\",\"sk\":\"\",\"topic\":\"mone_hera_staging_trace_etl_server\",\"type\":\"talos\",\"tag\":\"tags_1_4_620\"}"; - RmqOutput outputOld = gson.fromJson(msg1, RmqOutput.class); - OutputSimilarComparator outputSimilarComparator = new OutputSimilarComparator(outputOld); - String msg2 = "{\"clusterInfo\":\"http://127.0.0.1\",\"producerGroup\":\"subGroup_tags_1_4_620\",\"ak\":\"\",\"sk\":\"\",\"topic\":\"mione_staging_jaeger_etl_sidecar_first\",\"type\":\"talos\",\"tag\":\"tags_1_4_620\"}"; - RmqOutput outputNew = gson.fromJson(msg2, RmqOutput.class); - log.info("result:{}", outputSimilarComparator.compare(outputNew)); - } - - @Test - public void test() { - ImmutableList list = ImmutableList.of("1", "2", 3); - log.info("result:{}", list); - } - - @Test - public void testCache() throws ExecutionException { - LoadingCache graphs = CacheBuilder.newBuilder().maximumSize(1000) - .build(new CacheLoader() { - public String load(String key) { - // Here is the method of taking the value of the key according to the actual value, such as going to the database according to the key or through complex and time-consuming calculations - System.out.println("no cache,load from db"); - return "123"; - } - }); - String val1 = graphs.get("key"); - System.out.println("1 value is: " + val1); - String val2 = graphs.get("key"); - System.out.println("2 value is: " + val2); - Cache cache = CacheBuilder.newBuilder() - .maximumSize(1000).build(); - String val = cache.get("key", new Callable() { - public String call() { - // Here is the method of taking the value of the key according to the actual value, such as going to the database according to the key or through complex and time-consuming calculations - System.out.println("val call method is invoked"); - return "123"; - } - }); - System.out.println("1 value is: " + val1); - val = cache.get("key", new Callable() { - public String call() { - System.out.println("val call method is invoked"); - return "123"; - } - }); - System.out.println("1 value is: " + val1); - - val = cache.get("testKey", new Callable() { - public String call() { - System.out.println("testKey val call method is invoked"); - return "456"; - } - }); - System.out.println("2 value is: " + val); - - } - - - @Test - public void testRateLimiter() throws InterruptedException { - //A QPS setting of 5 means that only five concurrent requests are allowed to be processed per second - RateLimiter rateLimiter = RateLimiter.create(5); - ExecutorService executorService = Executors.newFixedThreadPool(10); - int nTasks = 10; - CountDownLatch countDownLatch = new CountDownLatch(nTasks); - long start = System.currentTimeMillis(); - for (int i = 0; i < nTasks; i++) { - final int j = i; - executorService.submit(() -> { - rateLimiter.acquire(); - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - } - log.info(Thread.currentThread().getName() + " gets job " + j + " done"); - countDownLatch.countDown(); - }); - } - executorService.shutdown(); - countDownLatch.await(); - long end = System.currentTimeMillis(); - log.info("10 jobs gets done by 5 threads concurrently in " + (end - start) + " milliseconds"); - } -} diff --git a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/DefaultFileMonitorListenerTest.java b/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/DefaultFileMonitorListenerTest.java deleted file mode 100644 index f8385fd59..000000000 --- a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/DefaultFileMonitorListenerTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent; - -import org.junit.Test; - -import java.io.IOException; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/8/4 19:31 - */ -public class DefaultFileMonitorListenerTest { - - @Test - public void testFileChange() throws IOException { -// DefaultFileMonitorListener defaultFileMonitorListener = new DefaultFileMonitorListener(); -// defaultFileMonitorListener.startFileMonitor("/home/work/log/test"); -// System.in.read(); - } -} diff --git a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/FileInodeTest.java b/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/FileInodeTest.java deleted file mode 100644 index 9307862b7..000000000 --- a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/FileInodeTest.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent; - -import com.google.common.collect.Lists; -import org.junit.Assert; -import org.junit.Test; - -import java.io.File; -import java.nio.file.Files; -import java.nio.file.attribute.BasicFileAttributeView; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/8/16 18:52 - */ -public class FileInodeTest { - - @Test - public void testFileNode() { -// ChannelMemory.UnixFileNode unixFileNode = ChannelUtil.buildUnixFileNode("/home/work/log/log-test/server.log"); -// System.out.println("result:" + unixFileNode); - } - - @Test - public void test() { - File file = new File("/home/work/log-test/server.log"); - try { - BasicFileAttributeView basicview = Files.getFileAttributeView(file.toPath(), BasicFileAttributeView.class); - BasicFileAttributes attr = basicview.readAttributes(); - System.out.println("attr.fileKey():" + attr.fileKey() - + " attr.creationTime:" + attr.creationTime() - + " attr.lastModifiedTime:" + attr.lastModifiedTime()); - Assert.assertNotNull(basicview); - } catch (Exception ex) { - System.out.println(ex.getMessage()); - } - } - - @Test - public void testLength() { - String str = "/home/work/log/nr-xmstore-xmstore-stockms-930394-66b6fd6b9-6tzq9/xmstore-stockms/trace/trace.log, /home/work/log/mifaas-xmstore-dispatcher-stock-180683-30346-644ff756d9-xd6jq/mifaas/trace/trace.log, /home/work/log/nr-xmstore-xmstore-maindata-960202-c7b7bd45-hrpxm/xmstore-maindata/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-fdc669864-4n6q2/xmstore-stockms/trace/trace.log, /home/work/log/mishop-goods-gms-930358-d74f8fc4d-6mdwg/gms/trace/trace.log, /home/work/log/rikaaa0928-nacostestclient-930731-7dc4455bfb-f9dck/nacostestclient/trace/trace.log, /home/work/log/nr-fortune-fortune-calculation-rebate-calculator-990607-6876l9l/rebate-calculator/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-fdc669864-c46t9/xmstore-stockms/trace/trace.log, /home/work/log/nr-xmstore-xmstore-performance-990692-69784475fd-24qq9/xmstore-performance/trace/trace.log, /home/work/log/nr-trade-nr-trade-center-930194-7dfd6c6fc9-7xfw9/nr-trade-center/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-fdc669864-vt8pd/xmstore-stockms/trace/trace.log, /home/work/log/youpin-mkact-award-center-990754-c557dcc68-xdfjp/award-center-server/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-fdc669864-68vnk/xmstore-stockms/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-fdc669864-dtkfd/xmstore-stockms/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-fdc669864-n255p/xmstore-stockms/trace/trace.log, /home/work/log/nr-fortune-fortune-calculation-rebate-calculator-990607-5dwcgb4/rebate-calculator/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-fdc669864-sqhg5/xmstore-stockms/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-5779c9c599-fgvsd/xmstore-stockms/trace/trace.log, /home/work/log/nr-trade-nr-trade-center-930194-5d78b65fd5-jbgvs/nr-trade-center/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-fdc669864-4hb4z/xmstore-stockms/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-fdc669864-hn65p/xmstore-stockms/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-fdc669864-mglhs/xmstore-stockms/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-fdc669864-jjh52/xmstore-stockms/trace/trace.log, /home/work/log/nr-trade-nr-trade-center-930194-7dfd6c6fc9-qklkh/nr-trade-center/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-fdc669864-hwwm2/xmstore-stockms/trace/trace.log, /home/work/log/youpin-mkact-entity-prize-930161-6dc664c58-jjdvw/entity-prize-server/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-fdc669864-f9blv/xmstore-stockms/trace/trace.log, /home/work/log/youpin-mkact-wukong-990796-6fd6f5b4bb-mc2sp/wukong-job/trace/trace.log, /home/work/log/nr-back-xmstore-mishow-tv-930546-5565957ccc-xgxns/xmstore-mishow-tv/trace/trace.log, /home/work/log/mishop-goods-gms-960477-678fbbbb49-sxqfz/gms/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-fdc669864-2dhfg/xmstore-stockms/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-fdc669864-wsgk7/xmstore-stockms/trace/trace.log, /home/work/log/nr-back-xmstore-replenish-990477-695657dd65-x5p4s/xmstore-replenish/trace/trace.log, /home/work/log/mifaas-zzytest102-240720-60310-8547dc8755-bnv99/mifaas/trace/trace.log, /home/work/log/mit-new-retail-cn-finance-cn-fin-price-match-960547-8b8596mwqzg/cn-fin-price-math/trace/trace.log, /home/work/log/rikaaa0928-nacostestclient-930731-57b6b8cb8b-gnsng/nacostestclient/trace/trace.log, /home/work/log/nr-back-xmstore-procurement-930390-885445b85-vlgnm/xmstore-procurement/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-fdc669864-6rqqm/xmstore-stockms/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-fdc669864-kxk5r/xmstore-stockms/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-fdc669864-gfzz9/xmstore-stockms/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-fdc669864-2n2g5/xmstore-stockms/trace/trace.log, /home/work/log/nr-xmstore-xmstore-maindata-nearby-930540-54999d668f-hmmd5/maindata-nearby/trace/trace.log, /home/work/log/proretail-proretail-live-960549-b5fff7c78-bjwv2/proretail-live/trace/trace.log, /home/work/log/mishop-goods-gms-930358-d74f8fc4d-r47xl/gms/trace/trace.log, /home/work/log/b2c-svr-youpin-consumable-990805-78668cc8d4-4q6fp/consumable-server/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-77d7bb7c76-l5cwm/xmstore-stockms/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-fdc669864-lqgf7/xmstore-stockms/trace/trace.log, /home/work/log/nr-fortune-fortune-accounting-accounting-biz-930535-5f49b8rnqg2/accounting-biz/trace/trace.log, /home/work/log/nr-fortune-fortune-calculation-rebate-policy-center-960655hqlcp/rebate-policy-center-server/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-fdc669864-mvpxl/xmstore-stockms/trace/trace.log, /home/work/log/nr-back-storems-990482-8667997b65-w95cw/xmstore-storems/trace/trace.log, /home/work/log/nr-back-storems-990482-8667997b65-4bvl6/xmstore-storems/trace/trace.log, /home/work/log/nr-back-xmstore-procurement-930390-797bf69667-g72kn/xmstore-procurement/trace/trace.log, /home/work/log/rikaaa0928-nacostestclient-930391-d8f76f5c9-rx49s/nacostestclient/trace/trace.log, /home/work/log/rikaaa0928-nacostestclient-1050016-5b6db46f78-7wg8r/nacostestclient/trace/trace.log, /home/work/log/nr-xmstore-xmstore-sample-120683-5697f469c6-8gqhr/xmstore-sample/trace/trace.log, /home/work/log/nr-xmstore-xmstore-store-90678-8856d889b-2dq8q/xmstore-store/trace/trace.log"; - System.out.println("length:" + str.split(",").length); - } - - @Test - public void testStreamForeachReturn() { - List list = Lists.newArrayList("1", "2", "3", "4", "5"); - Map map = new HashMap<>(); - list.stream().forEach(s -> { - if (Objects.equals("3", s)) { - return; - } - map.put(s, s); - }); - System.out.println("result:" + map); - } - -} diff --git a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/FilterMonitorTest.java b/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/FilterMonitorTest.java deleted file mode 100644 index f42d4ff90..000000000 --- a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/FilterMonitorTest.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent; - -import com.google.common.collect.Lists; -import com.google.gson.Gson; -import com.xiaomi.mone.log.agent.channel.file.FileListener; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.monitor.FileAlterationMonitor; -import org.apache.commons.io.monitor.FileAlterationObserver; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.util.List; -import java.util.function.Consumer; - -/** - * @author: wtt - * @date: 2022/6/2 12:12 - * @description: - */ -@Slf4j -public class FilterMonitorTest { - - @Test - public void testScheduleExecutor() throws IOException { -// ScheduledThreadPoolExecutor scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(5); -// scheduledThreadPoolExecutor.scheduleAtFixedRate(() -> { -// log.info("current thread:{}", Thread.currentThread().getName()); -// }, 0, 100, TimeUnit.MILLISECONDS); -// new Thread(() -> { -// ScheduledFuture scheduledFuture = scheduledThreadPoolExecutor.scheduleAtFixedRate(() -> { -// log.info("child current thread:{}", Thread.currentThread().getName()); -// }, 0, 100, TimeUnit.MILLISECONDS); -// try { -// TimeUnit.MINUTES.sleep(1); -// } catch (InterruptedException e) { -// e.printStackTrace(); -// } -// scheduledFuture.cancel(true); -// }).start(); -// System.in.read(); - } - - @Test - public void test() throws IOException { - Consumer consumer = s -> { - System.out.println("file come:" + s); - }; - List watchList = Lists.newArrayList("/home/work/log/test/"); - FileAlterationMonitor monitor = new FileAlterationMonitor(5000); - log.info("agent monitor files:{}", new Gson().toJson(watchList)); - for (String watch : watchList) { - File watchFile = new File(watch); -// if (!watchFile.exists()) { -// log.error("##!!## agent monitor file not exists:{}, filePattern:{}", watch, filePattern); -// continue; -// } - - FileAlterationObserver observer = new FileAlterationObserver(new File(watch)); - observer.addListener(new FileListener(consumer)); - - log.info("## agent monitor file:{}, filePattern:{}", watch); - monitor.addObserver(observer); - } - - try { - monitor.start(); - log.info("## agent monitor filePattern:{} started"); - } catch (Exception e) { - log.error(String.format("agent file monitor start err,monitor filePattern:%s"), e); - } -// System.in.read(); - } -} diff --git a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/LogProcessorTest.java b/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/LogProcessorTest.java deleted file mode 100644 index 56304e2b0..000000000 --- a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/LogProcessorTest.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent; - -import lombok.extern.slf4j.Slf4j; -import org.junit.Test; - -import java.io.IOException; -import java.util.concurrent.CompletableFuture; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/8/29 10:35 - */ -@Slf4j -public class LogProcessorTest { - - @Test - public void test() { - CompletableFuture reFreshFuture = CompletableFuture.runAsync(() -> { - }); - CompletableFuture stopChannelFuture = CompletableFuture.runAsync(() -> { - }); - CompletableFuture.allOf(reFreshFuture, stopChannelFuture).join(); - log.info("config change success"); - } - - @Test - public void testFile() { -// String defaultMonitorPath = "/home/work/log/"; -// long size = FileUtils.listFiles(new File(defaultMonitorPath), null, true).size(); -// log.info("result:{}", size); - } - - /** - * Can't stop - * - * @throws IOException - */ - @Test - public void testComplete() throws IOException { -// String defaultMonitorPath = "/home/work/log/"; -// int result = 0; -// CompletableFuture fileSizeFuture = CompletableFuture -// .supplyAsync(() -> { -// try { -// TimeUnit.MILLISECONDS.sleep(30); -// } catch (InterruptedException e) { -// throw new RuntimeException(e); -// } -// log.info("testes"); -// return FileUtils.listFiles(new File(defaultMonitorPath), null, true).size(); -// }); -// try { -// result = fileSizeFuture.get(1, TimeUnit.SECONDS); -// } catch (Exception e) { -// log.info("getDefaultFileSize error", e); -// } -// log.info("result:{}", result); -// fileSizeFuture.complete(1); -// System.in.read(); - } -} diff --git a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/channel/ChannelServiceTest.java b/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/channel/ChannelServiceTest.java deleted file mode 100644 index 61820b4e8..000000000 --- a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/channel/ChannelServiceTest.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.channel; - -import com.google.common.collect.Lists; -import com.xiaomi.mone.file.LogFile; -import com.xiaomi.mone.log.utils.SimilarUtils; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.junit.Test; - -import java.nio.file.FileSystems; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/2/16 11:10 - */ -@Slf4j -public class ChannelServiceTest { - - @Test - public void monitorTest() { - String logPattern = "/home/work/logs/neo-logs/jump-game-stable-74bbdbcf9d-sf2vx/applogs/i18n-shop-jump-game/log_debug.log|log_error.log|log_info.log|log_warn.log|sys.log"; - String changedFilePath = "/home/work/logs/neo-logs/jump-game-stable-74bbdbcf9d-sf2vx/applogs/i18n-shop-jump-game/info/log-info.2023443"; - String logSplitExpress = "/home/work/logs/neo-logs/jump-game-stable-74bbdbcf9d-sf2vx/applogs/i18n-shop-jump-game/(debug/log-debug.*|error/log-error.*|info/log-info.*|warn/log-warn.*|sys.*)"; - List patterns = Arrays.asList("/home/work/logs/neo-logs/jump-game-stable-74bbdbcf9d-sf2vx/applogs/i18n-shop-jump-game/log_debug.log", - "/home/work/logs/neo-logs/jump-game-stable-74bbdbcf9d-sf2vx/applogs/i18n-shop-jump-game/log_error.log", - "/home/work/logs/neo-logs/jump-game-stable-74bbdbcf9d-sf2vx/applogs/i18n-shop-jump-game/log_info.log", - "/home/work/logs/neo-logs/jump-game-stable-74bbdbcf9d-sf2vx/applogs/i18n-shop-jump-game/log_warn.log", - "/home/work/logs/neo-logs/jump-game-stable-74bbdbcf9d-sf2vx/applogs/i18n-shop-jump-game/sys.log"); - ConcurrentHashMap logFileMap = new ConcurrentHashMap<>(); - String channelId = ""; - String localIp = ""; - String separator = "/"; - String baseFileName = logPattern.substring(logPattern.lastIndexOf(separator) + 1); - log.warn("#### watch consumer accept file:{},logPattern:{}", changedFilePath, logPattern); - String changeFileName = StringUtils.substringAfterLast(changedFilePath, separator); - Pattern pattern = makeLogPattern(logPattern, logSplitExpress); - String originFileName = StringUtils.substringAfterLast(logPattern, separator); - boolean ifTo = true; - if (changeFileName.contains("wf")) { - String changeFilePrefix = StringUtils.substringBeforeLast(changeFileName, "-"); - ifTo = changeFilePrefix.equals(originFileName); - } - if (pattern.matcher(changedFilePath).matches() && ifTo) { - List fileNames = Arrays.stream(baseFileName.split("\\|")).collect(Collectors.toList()); - String fileName = SimilarUtils.findHighestSimilarityStr(changeFileName, fileNames); - String newFilePath = changedFilePath.substring(0, changedFilePath.lastIndexOf(separator)) + separator + fileName; - if (StringUtils.isNotEmpty(logSplitExpress)) { - String finalFileName = fileName; - newFilePath = patterns.stream().filter(logPath -> logPath.contains(finalFileName)).findFirst().get(); - } - log.warn("newFilePath:{}", newFilePath); - LogFile logFile = logFileMap.get(newFilePath); - if (null == logFile) { -// readFile(input.getPatternCode(), localIp, newFilePath, channelId); - log.info("watch new file create for chnnelId:{},ip:{},path:{}", channelId, localIp, newFilePath); - } else { - try { - TimeUnit.SECONDS.sleep(7); - } catch (InterruptedException e) { - e.printStackTrace(); - } - logFile.setReOpen(true); - log.info("file reOpen: chnnelId:{},ip:{},path:{}", channelId, localIp, newFilePath); - } - } else { - log.info("invalid file create event, logPattern:{}, changedFilePath:{}", logPattern, changedFilePath); - } - } - - public Pattern makeLogPattern(String logPattern, String logSplitExpress) { - if (StringUtils.isNotEmpty(logSplitExpress)) { - return Pattern.compile(logSplitExpress); - } - String separator = FileSystems.getDefault().getSeparator(); - List pathList = Lists.newArrayList(); - for (String filePath : logPattern.split(",")) { - String filePrefix = StringUtils.substringBeforeLast(filePath, separator); - String multipleFileNames = StringUtils.substringAfterLast(filePath, separator); - if (filePath.contains("*") && !filePath.contains(".*")) { - logPattern = logPattern.replaceAll("\\*", ".*"); - } else { - logPattern = Arrays.stream(multipleFileNames.split("\\|")) - .map(s -> filePrefix + separator + s + ".*") - .collect(Collectors.joining("|")); - } - if (!logPattern.endsWith(".*")) { - logPattern = logPattern + ".*"; - } - pathList.add(logPattern); - } - log.warn("logPattern -> regex:{}", logPattern); - return Pattern.compile(logPattern); - } -} diff --git a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/common/ExecutorUtilTest.java b/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/common/ExecutorUtilTest.java deleted file mode 100644 index 7b13fd736..000000000 --- a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/common/ExecutorUtilTest.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.common; - -import junit.framework.TestCase; -import lombok.extern.slf4j.Slf4j; - -import java.util.concurrent.TimeUnit; - -/** - * - * @description - * @version 1.0 - * @author wtt - * @date 2024/7/11 16:55 - * - */ -@Slf4j -public class ExecutorUtilTest extends TestCase { - - public void testScheduleAtFixedRate() throws InterruptedException { - - ExecutorUtil.scheduleAtFixedRate(() -> { - log.info("I am a task"); - }, 10, 10, java.util.concurrent.TimeUnit.SECONDS); - - ExecutorUtil.submit(() -> { - while(true){ - log.info("I am a submit task"); - try { - TimeUnit.SECONDS.sleep(10); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - }); - - Thread.sleep(TimeUnit.MINUTES.toMillis(1)); - } - -} \ No newline at end of file diff --git a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/common/trace/TraceUtilTest.java b/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/common/trace/TraceUtilTest.java deleted file mode 100644 index 4e764e0c6..000000000 --- a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/common/trace/TraceUtilTest.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.common.trace; - -import org.junit.Test; - -public class TraceUtilTest { - - private static String spanStr = ""; - - @Test - public void toTSpanDataTest() { -// TSpanData tSpanData = TraceUtil.toTSpanData(spanStr); -// Assert.assertTrue(tSpanData != null && tSpanData.isSetKind() && tSpanData.isSetStatus() -// && tSpanData.isSetAttributes() && tSpanData.isSetEvents() && tSpanData.isSetLinks() -// && tSpanData.isSetParentSpanContext() && tSpanData.isSetExtra()); - } -} diff --git a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/extension/RocketMQServiceTest.java b/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/extension/RocketMQServiceTest.java deleted file mode 100644 index 651ddde3f..000000000 --- a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/extension/RocketMQServiceTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.extension; - -import lombok.extern.slf4j.Slf4j; -import org.junit.Test; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/9/22 15:21 - */ -@Slf4j -public class RocketMQServiceTest { - - @Test - public void exporterTransSendTest() { - RocketMQService rocketMQService = new RocketMQService(); - } -} diff --git a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/filter/ComparatorTest.java b/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/filter/ComparatorTest.java deleted file mode 100644 index 71e3d4bac..000000000 --- a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/filter/ComparatorTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.filter; - -import com.xiaomi.mone.log.agent.input.AppLogInput; -import org.junit.Assert; -import org.junit.Test; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/3/29 15:11 - */ -public class ComparatorTest { - - @Test - public void test1() { - AppLogInput newInput = new AppLogInput(); - newInput.setLogPattern("/fsdfdsf/sfsdfs"); - newInput.setLogSplitExpress("/fsfds/sdfsdf"); - AppLogInput oldInput = new AppLogInput(); - oldInput.setLogSplitExpress("/fsfds/sdfsdf"); - Assert.assertEquals(false, newInput.equals(oldInput)); - } - -} diff --git a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/filter/FilterTransTest.java b/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/filter/FilterTransTest.java deleted file mode 100644 index 3fac3fbdc..000000000 --- a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/filter/FilterTransTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.filter; - -import com.xiaomi.mone.log.api.filter.Common; -import com.xiaomi.mone.log.api.model.meta.FilterConf; -import com.xiaomi.mone.log.api.model.meta.FilterDefine; -import junit.framework.TestCase; - -import java.util.HashMap; - -public class FilterTransTest extends TestCase { - - public void testFilterConfTrans() { - FilterDefine filterDefine = new FilterDefine(); - filterDefine.setCode(Common.RATE_LIMIT_CODE + 0); - filterDefine.setArgs(new HashMap() {{ - put(Common.PERMITS_PER_SECOND, "100"); - }}); - FilterConf filterConf = FilterTrans.filterConfTrans(filterDefine); - System.out.println(filterConf); - } -} \ No newline at end of file diff --git a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/lock/LockTest.java b/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/lock/LockTest.java deleted file mode 100644 index a7c8b3812..000000000 --- a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/lock/LockTest.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.lock; - -import org.junit.Test; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * @author shanwb - * @date 2021-12-21 - */ -public class LockTest { - - private static byte[] lock = new byte[0]; - - @Test - public void lockTest() { - lock1(); - lock2(); - } - - private void lock1() { - long now = System.currentTimeMillis(); - CountDownLatch countDownLatch = new CountDownLatch(9999999); - for (int i=0; i< 9999999; i++) { - new Runnable(){ - - @Override - public void run() { - synchronized (lock) { - countDownLatch.countDown(); - } - } - }.run(); - } - try { - countDownLatch.await(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - System.out.println("cost1:" + (System.currentTimeMillis() - now)); - } - - private void lock2() { - long now = System.currentTimeMillis(); - CountDownLatch countDownLatch = new CountDownLatch(9999999); - MoneSpinLock spinLock = new MoneSpinLock(); - for (int i=0; i< 9999999; i++) { - new Runnable(){ - @Override - public void run() { - - try { - spinLock.lock(); - countDownLatch.countDown(); - } finally { - spinLock.unlock(); - } - } - }.run(); - } - try { - countDownLatch.await(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - System.out.println("cost2:" + (System.currentTimeMillis() - now)); - } - - public static class MoneSpinLock { - //true: Can lock, false : in lock. - private AtomicBoolean spinLock = new AtomicBoolean(true); - - public void lock() { - boolean flag; - do { - flag = this.spinLock.compareAndSet(true, false); - } - while (!flag); - } - - public void unlock() { - this.spinLock.compareAndSet(false, true); - } - } -} diff --git a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/rpc/processor/service/ChannelDefineRpcLocatorTest.java b/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/rpc/processor/service/ChannelDefineRpcLocatorTest.java deleted file mode 100644 index 6cacb990a..000000000 --- a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/rpc/processor/service/ChannelDefineRpcLocatorTest.java +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.rpc.processor.service; - -import com.google.common.collect.Lists; -import com.google.gson.Gson; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Arrays; -import java.util.List; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/9 10:36 - */ -@Slf4j -public class ChannelDefineRpcLocatorTest { - - private Gson gson = new Gson(); - - @Test - public void testDataTrans() { -// Ioc.ins().init("com.xiaomi"); -// ChannelDefineRpcLocator channelDefineRpcLocator = Ioc.ins().getBean(ChannelDefineRpcLocator.class); -// System.out.println(channelDefineRpcLocator); -// String str = ""; -// LogCollectMeta logCollectMeta = JSONUtil.toBean(str, LogCollectMeta.class); -// List channelDefine = ChannelDefineRpcLocator.agentTail2ChannelDefine(channelDefineRpcLocator.logCollectMeta2ChannelDefines(logCollectMeta)); -// log.info("Returns data：{}", channelDefine); -// Assert.assertNotNull(channelDefine); - } - - @Test - public void testPattern() { - String logPattern = "/home/work/log/log/error.log"; - String changedFilePath = "/home/work/log/log/error.log.202323"; - logPattern = makePattern(logPattern); - log.warn("logPattern -> regex:{}", logPattern); - Pattern pattern = Pattern.compile(logPattern); - Assert.assertEquals(true, pattern.matcher(changedFilePath).matches()); - - } - - @Test - public void testPattern1() { - String logPattern = "/home/work/log/log/server.log,/home/work/log/test/server.log"; - String changedFilePath = "/home/work/log/test/server.log.202323"; - logPattern = makePattern(logPattern); - log.warn("logPattern -> regex:{}", logPattern); - Pattern pattern = Pattern.compile(logPattern); - Assert.assertEquals(true, pattern.matcher(changedFilePath).matches()); - - } - - @Test - public void testPattern11() { - String logPattern = "/home/work/logs/applogs/cn_xm_stock_soa/cn_xm_stock_soa.log"; - String changedFilePath = "/home/work/logs/applogs/cn_xm_stock_soa/cn_xm_stock_soa.log.wf-2021122714.gz"; - logPattern = makePattern(logPattern); - log.warn("logPattern -> regex:{}", logPattern); - Pattern pattern = Pattern.compile(logPattern); - Assert.assertEquals(true, pattern.matcher(changedFilePath).matches()); - - } - - @Test - public void test123() { - String logPattern = "/home/work/logs/applogs/cn_xm_stock_soa/cn_xm_stock_soa.log"; - String separator = "/"; - String changedFilePath = "/home/work/logs/applogs/cn_xm_stock_soa/cn_xm_stock_soa.log-2021122715.gz"; - String changeFileName = StringUtils.substringAfterLast(changedFilePath, separator); - String originFileName = StringUtils.substringAfterLast(logPattern, separator); - boolean ifTo = true; - if (changeFileName.startsWith(originFileName + "-")) { - String changeFilePrefix = StringUtils.substringBefore(changeFileName, "-"); - ifTo = changeFilePrefix.equals(originFileName); - } - Assert.assertEquals(true, ifTo); - } - - @Test - public void test1234() { - String logPattern = "/home/work/logs/applogs/cn_xm_stock_soa/cn_xm_stock_soa.log"; - String separator = "/"; - String changedFilePath = "/home/work/logs/applogs/cn_xm_stock_soa/cn_xm_stock_soa.log-2021122715.gz"; - String changeFileName = StringUtils.substringAfterLast(changedFilePath, separator); - String originFileName = StringUtils.substringAfterLast(logPattern, separator); - boolean ifTo = true; - if (changeFileName.contains("wf")) { - String changeFilePrefix = StringUtils.substringBefore(changeFileName, "-"); - ifTo = changeFilePrefix.equals(originFileName); - } - Assert.assertEquals(true, ifTo); - } - - @Test - public void test12345() { - String logPattern = "/home/work/logs/applogs/cn_xm_stock_soa/cn_xm_stock_soa.log.wf"; - String separator = "/"; - String changedFilePath = "/home/work/logs/applogs/cn_xm_stock_soa/cn_xm_stock_soa.log.wf-2021122715.gz"; - String changeFileName = StringUtils.substringAfterLast(changedFilePath, separator); - String originFileName = StringUtils.substringAfterLast(logPattern, separator); - boolean ifTo = true; - if (changeFileName.contains("wf")) { - String changeFilePrefix = StringUtils.substringBefore(changeFileName, "-"); - ifTo = changeFilePrefix.equals(originFileName); - } - Assert.assertEquals(true, ifTo); - } - - @Test - public void test12346() { - String logPattern = "/home/work/logs/applogs/cn_xm_stock_soa/cn_xm_stock_soa.log"; - String separator = "/"; - String changedFilePath = "/home/work/logs/applogs/cn_xm_stock_soa/cn_xm_stock_soa.log.wf-2021121210.gz"; - String changeFileName = StringUtils.substringAfterLast(changedFilePath, separator); - String originFileName = StringUtils.substringAfterLast(logPattern, separator); - boolean ifTo = true; - if (changeFileName.contains("wf")) { - String changeFilePrefix = StringUtils.substringBefore(changeFileName, "-"); - ifTo = changeFilePrefix.equals(originFileName); - } - Assert.assertEquals(false, ifTo); - } - @Test - public void test12347() { - String logPattern = "/home/work/logs/applogs/cn_xm_stock_soa/server.log"; - String separator = "/"; - String changedFilePath = "/home/work/logs/applogs/cn_xm_stock_soa/server.log.2021-12-27-07"; - String changeFileName = StringUtils.substringAfterLast(changedFilePath, separator); - String originFileName = StringUtils.substringAfterLast(logPattern, separator); - boolean ifTo = true; - if (changeFileName.contains("wf")) { - String changeFilePrefix = StringUtils.substringBefore(changeFileName, "-"); - ifTo = changeFilePrefix.equals(originFileName); - } - Assert.assertEquals(true, ifTo); - } - - @Test - public void testPattern2() { - String logPattern = "/home/work/log/*/trace.log"; - String changedFilePath = "/home/work/log/log-agent/trace.log.2021-12-23-16"; - logPattern = makePattern(logPattern); - log.warn("logPattern -> regex:{}", logPattern); - Pattern pattern = Pattern.compile(logPattern); - Assert.assertEquals(true, pattern.matcher(changedFilePath).matches()); - - } - - private String makePattern(String logPattern) { - String separator = "/"; - List pathList = Lists.newArrayList(); - for (String filePath : logPattern.split(",")) { - String filePrefix = StringUtils.substringBeforeLast(filePath, separator); - String multipleFileNames = StringUtils.substringAfterLast(filePath, separator); - if (filePath.contains("*") && !filePath.contains(".*")) { - logPattern = logPattern.replaceAll("\\*", ".*"); - } else { - logPattern = Arrays.stream(multipleFileNames.split("\\|")) - .map(s -> filePrefix + separator + s + ".*") - .collect(Collectors.joining("|")); - } - if (!logPattern.endsWith(".*")) { - logPattern = logPattern + ".*"; - } - pathList.add(logPattern); - } - return pathList.stream().collect(Collectors.joining("|")); - } - - @Test - public void test2() { - String logPattern = "/home/work/log/log/server.log|error.log"; - String changedFilePath = "/home/work/log/log/server.log.2021-12-23-16"; - String suffix = ".log"; - String separator = "/"; - String baseFileName = logPattern.substring(logPattern.lastIndexOf(separator) + 1); - System.out.println(); - System.out.println(baseFileName); - - } -} diff --git a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/rpc/processor/service/LimitTest.java b/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/rpc/processor/service/LimitTest.java deleted file mode 100644 index 440e7bd36..000000000 --- a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/rpc/processor/service/LimitTest.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.rpc.processor.service; - -import com.google.common.util.concurrent.RateLimiter; -import lombok.SneakyThrows; -import org.junit.Test; - -import java.util.Date; - -/** - * @Author goodjava@qq.com - * @Date 2021/8/25 16:59 - */ -public class LimitTest { - - - @SneakyThrows - @Test - public void testLimiter() { - final RateLimiter rateLimiter = RateLimiter.create(3); - new Thread(() -> { - while (true) { - rateLimiter.acquire(); - System.out.println(new Date()); - } - }).start(); -// System.in.read(); - } - - @Test - public void testIf() { - int num = 1; - if (num == 1) { - System.out.println("hello "); - } else if (1 == 1) { - System.out.println("Vatican City"); - } - } - -} diff --git a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/rpc/processor/service/RpcTest.java b/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/rpc/processor/service/RpcTest.java deleted file mode 100644 index 7a425cc05..000000000 --- a/ozhera-log/log-agent/src/test/java/com/xiaomi/mone/log/agent/rpc/processor/service/RpcTest.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.agent.rpc.processor.service; - -import cn.hutool.core.util.ObjectUtil; -import com.xiaomi.mone.log.api.model.msg.LineMessage; -import lombok.extern.slf4j.Slf4j; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/8/5 14:18 - */ -@Slf4j -public class RpcTest { - - @Test - public void testRpc() { -// Ioc.ins().init("com.xiaomi"); -// ChannelDefineRpcLocator channelDefineRpcLocator = Ioc.ins().getBean(ChannelDefineRpcLocator.class); -// System.out.println(channelDefineRpcLocator); -// List channelDefine = channelDefineRpcLocator.getChannelDefine(); -// log.info("Returns data：{}", channelDefine); -// Assert.assertNotNull(channelDefine); - } - - @Test - public void test() { -// Ioc.ins().init("com.xiaomi"); -// ChannelDefineRpcLocator channelDefineRpcLocator = Ioc.ins().getBean(ChannelDefineRpcLocator.class); -// List logCollectMetaFromManager = channelDefineRpcLocator.getChannelDefine("127.0.0.1"); -// System.out.println(new Gson().toJson(logCollectMetaFromManager)); -// Assert.assertNotNull(logCollectMetaFromManager); - - } - - @Test - public void testCopy() { - LineMessage lineMessage = new LineMessage(); - lineMessage.setLineNumber(2L); - List messageList = Arrays.asList(lineMessage); - List cloneList = messageList.stream().map(message -> ObjectUtil.clone(message)).collect(Collectors.toList()); -// List cloneList = messageList.stream().collect(Collectors.toList()); - log.info("return data：{}", cloneList); - Assert.assertNotNull(cloneList); - } - - - @Test - public void test1() throws Exception { - -// ChannelEngine channelEngine = new ChannelEngine(); -// TalosOutput talosOutput = new TalosOutput(); -// talosOutput.setTopic("519_planet-opmsg-postprocess_191_china"); -// TalosProducer talosProducer = channelEngine.initTalosProducer(talosOutput, ""); -// TalosProducer talosProducer2 = channelEngine.initTalosProducer(talosOutput, ""); -// List messages = Lists.newArrayList(); -// Message message = new Message(ByteBuffer.wrap("helllo ！！！！！！".getBytes(StandardCharsets.UTF_8))); -// messages.add(message); -// IntStream.range(0, 10).forEach(value -> { -// try { -// talosProducer.addUserMessage(messages); -// } catch (ProducerNotActiveException e) { -// log.error("", e); -// } -// }); -// List messages2 = Lists.newArrayList(); -// Message message2 = new Message(ByteBuffer.wrap("test test！！！！！！".getBytes(StandardCharsets.UTF_8))); -// messages2.add(message2); -// talosProducer2.addUserMessage(messages2); -// talosProducer.shutdown(); -// talosProducer2.addUserMessage(messages2); -// talosProducer.addUserMessage(messages); -// System.in.read(); - } -} diff --git a/ozhera-log/log-api/pom.xml b/ozhera-log/log-api/pom.xml index 280dba458..78585d485 100644 --- a/ozhera-log/log-api/pom.xml +++ b/ozhera-log/log-api/pom.xml @@ -3,9 +3,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - run.mone + org.apache.ozhera ozhera-log - 1.4.0-jdk21 + 2.0.0-SNAPSHOT 4.0.0 diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/AppTypeEnum.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/AppTypeEnum.java deleted file mode 100644 index 9b5927936..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/AppTypeEnum.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.enums; - -import lombok.Getter; - -import java.util.Arrays; -import java.util.Objects; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/16 11:35 - */ -@Getter -public enum AppTypeEnum { - - LOG_MILOG(0, "milog"), - LOG_AGENT(1, "milog-agent"), - LOG_STREAM(2, "milog_stream_server_open"), - LOG_MANAGER(3, "log-manager"); - - private final Integer type; - private final String name; - - AppTypeEnum(Integer type, String name) { - this.type = type; - this.name = name; - } - - public static AppTypeEnum queryEnumByType(int type) { - return Arrays.stream(AppTypeEnum.values()).filter(machineTypeEnum -> { - if (Objects.equals(machineTypeEnum.type, type)) { - return true; - } - return false; - }).findFirst().orElse(null); - } - -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/DeployWayEnum.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/DeployWayEnum.java deleted file mode 100644 index 3da133372..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/DeployWayEnum.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.enums; - -import lombok.Getter; - -@Getter -public enum DeployWayEnum { - MIONE(1, "mione"); - - private final Integer code; - private final String name; - - DeployWayEnum(Integer code, String name) { - this.code = code; - this.name = name; - } -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/ESClusterEnum.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/ESClusterEnum.java deleted file mode 100644 index 2014097a4..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/ESClusterEnum.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.enums; - -import lombok.Getter; - -@Getter -public enum ESClusterEnum { - CN(); - private String dept; - private String name; - private String esClusterId; - - public static ESClusterEnum name2enum(String name) { - for (ESClusterEnum ecEnum : ESClusterEnum.values()) { - if (ecEnum.name.equals(name)) { - return ecEnum; - } - } - return null; - } - - public static ESClusterEnum dept2enum(String dept) { - for (ESClusterEnum ecEnum : ESClusterEnum.values()) { - if (ecEnum.dept.equals(dept)) { - return ecEnum; - } - } - return null; - } -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/EsOperatorEnum.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/EsOperatorEnum.java deleted file mode 100644 index 9c1897619..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/EsOperatorEnum.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.enums; - -import lombok.Getter; - -@Getter -public enum EsOperatorEnum { - AND_OPERATOR(" and "), - OR_OPERATOR(" or "), - NOT_OPERATOR(" not "), - ; - private final String code; - - EsOperatorEnum(String code) { - this.code = code; - } - - -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/EsOperatorMatchEnum.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/EsOperatorMatchEnum.java deleted file mode 100644 index ac21ab2e2..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/EsOperatorMatchEnum.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.enums; - -import lombok.Getter; - -@Getter -public enum EsOperatorMatchEnum { - /** - * Non-K:v matching, no field is specified - */ - ALL_MATCH_OPERATOR, - /** - * Specifies the field field - */ - KV_MATCH_OPERATOR, - ; -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/FavouriteSearchEnum.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/FavouriteSearchEnum.java deleted file mode 100644 index 0bc17e5c7..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/FavouriteSearchEnum.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.enums; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -import java.util.Arrays; -import java.util.Objects; - -@Getter -@AllArgsConstructor -public enum FavouriteSearchEnum { - TEXT(1, "text"), - TAIL(2, "tail"), - STORE(3, "store"); - - private final Integer code; - - private final String name; - - public static FavouriteSearchEnum queryByCode(Integer code) { - return Arrays.stream(FavouriteSearchEnum.values()).filter(machineTypeEnum -> { - if (Objects.equals(machineTypeEnum.code, code)) { - return true; - } - return false; - }).findFirst().orElse(null); - } - -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/K8sPodTypeEnum.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/K8sPodTypeEnum.java deleted file mode 100644 index f2f7aabce..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/K8sPodTypeEnum.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.enums; - -import lombok.Getter; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/9/6 15:05 - */ -@Getter -public enum K8sPodTypeEnum { - /** - * The pod name does not change, that is, the log path does not change - */ - STATEFUL, - STATELESS -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/LogStorageTypeEnum.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/LogStorageTypeEnum.java deleted file mode 100644 index c184a32a6..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/LogStorageTypeEnum.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.xiaomi.mone.log.api.enums; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/11/10 14:31 - */ -public enum LogStorageTypeEnum { - - ELASTICSEARCH, - DORIS, - CLICKHOUSE; - - public static LogStorageTypeEnum queryByName(String name) { - if (null == name || name.isEmpty()) { - return null; - } - for (LogStorageTypeEnum value : values()) { - if (value.name().equals(name.toUpperCase())) { - return value; - } - } - return null; - } -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/LogStructureEnum.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/LogStructureEnum.java deleted file mode 100644 index 48af11c6e..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/LogStructureEnum.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.enums; - -import lombok.Getter; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/6/13 14:43 - */ -@Getter -public enum LogStructureEnum { - - SPACE("log-space"), - - STORE("log-store"), - - TAIL("log-tail"); - - private String code; - - LogStructureEnum(String code) { - this.code = code; - } -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/LogTypeEnum.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/LogTypeEnum.java deleted file mode 100644 index 882f38d57..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/LogTypeEnum.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.enums; - -import lombok.Getter; - -/** - * @author shanwb - * @date 2021-07-08 - */ -@Getter -public enum LogTypeEnum { - APP_LOG_MULTI(1, "多行应用日志", 1, 1, - "时间格式开始的日志可以多行采集，例如2022或者[2022]等，如果不是以时间开始,则会多行糅合在一起"), - NGINX(2, "nginx日志", 3, 1, "单行采集，按文件的每一行采集成一列"), - OPENTELEMETRY(3, "opentelemetry日志", 4, 0, "多行采集，采集到固定的topic中"), - DOCKER(4, "docker日志", 5, 1, ""), - APP_LOG_SIGNAL(8, "单行应用日志", 2, 1, "单行采集，按文件的每一行采集成一列"), - ORIGIN_LOG(9, "原始格式日志", 6, 1, "向固定的topic中写入原始数据,hera不消费"), - FREE(0, "自定义日志", 7, 1, "单行采集，按文件的每一行采集成一列"); - - - private final Integer type; - private final String typeName; - private final Integer sort; - /** - * Support self-production and self-consumption - */ - private final Integer supportedConsume; - private final String describe; - - - LogTypeEnum(Integer type, String typeName, Integer sort, Integer supportedConsume, String describe) { - this.type = type; - this.typeName = typeName; - this.sort = sort; - this.supportedConsume = supportedConsume; - this.describe = describe; - } - - public Integer getType() { - return type; - } - - public String getTypeName() { - return typeName; - } - - public static LogTypeEnum name2enum(String enumName) { - for (LogTypeEnum ltEnum : LogTypeEnum.values()) { - if (ltEnum.name().equals(enumName)) { - return ltEnum; - } - } - - return null; - } - - public static LogTypeEnum type2enum(Integer typeCode) { - for (LogTypeEnum ltEnum : LogTypeEnum.values()) { - if (ltEnum.getType().equals(typeCode)) { - return ltEnum; - } - } - return null; - } - - public static String queryNameByType(Integer typeCode) { - LogTypeEnum logTypeEnum = type2enum(typeCode); - if (null != logTypeEnum) { - return logTypeEnum.getTypeName(); - } - return ""; - } - - public static String getLogTypeName(Integer typeCode) { - LogTypeEnum logTypeEnum = type2enum(typeCode); - if (null != logTypeEnum) { - return logTypeEnum.name(); - } - return ""; - } -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/MQSourceEnum.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/MQSourceEnum.java deleted file mode 100644 index 94d92a314..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/MQSourceEnum.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.enums; - -import lombok.Getter; - -import java.util.Arrays; -import java.util.Objects; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/11/28 10:03 - */ -@Getter -public enum MQSourceEnum { - ROCKETMQ(1, "rocketmq", "RocketMQService"), - KAFKA(2, "kafka", "KafkaService"); - - private final Integer code; - private final String name; - - private final String serviceName; - - - MQSourceEnum(Integer code, String name, String serviceName) { - this.code = code; - this.name = name; - this.serviceName = serviceName; - } - - public static String queryName(Integer code) { - for (MQSourceEnum value : MQSourceEnum.values()) { - if (Objects.equals(value.getCode(), code)) { - return value.getName(); - } - } - return ROCKETMQ.getName(); - } - - public static MQSourceEnum queryByName(String name) { - if (name == null || "".equals(name)) { - return null; - } - return Arrays.stream(MQSourceEnum.values()).sequential().filter(sourceEnum -> { - if (sourceEnum.getName().equals(name)) { - return true; - } - return false; - }).findFirst().orElse(null); - } -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/MachineRegionEnum.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/MachineRegionEnum.java deleted file mode 100644 index d432dd119..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/MachineRegionEnum.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.enums; - -import lombok.Getter; - -import java.util.Arrays; -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/10/18 15:41 - */ -@Getter -public enum MachineRegionEnum { - - CN_MACHINE("cn", "大陆机房", Arrays.asList("c1", "c2", "c3")); - - private final String en; - - private final String cn; - - private List childZone; - - MachineRegionEnum(String en, String cn, List childZone) { - this.en = en; - this.cn = cn; - this.childZone = childZone; - } - - public static String queryMchineInfoByZone(String zone) { - for (MachineRegionEnum machineRegionEnum : MachineRegionEnum.values()) { - if (machineRegionEnum.getChildZone().contains(zone)) { - return String.format("%s%s", machineRegionEnum.getEn(), machineRegionEnum.getCn()); - } - } - return ""; - } - - public static MachineRegionEnum queryMchineRegionByZone(String zone) { - for (MachineRegionEnum machineRegionEnum : MachineRegionEnum.values()) { - if (machineRegionEnum.getChildZone().contains(zone)) { - return machineRegionEnum; - } - } - return null; - } - - public static String queryCnByEn(String en) { - MachineRegionEnum machineRegionEnum = queryRegionByEn(en); - if (null != machineRegionEnum) { - return machineRegionEnum.getCn(); - } - return ""; - } - - public static MachineRegionEnum queryRegionByEn(String en) { - for (MachineRegionEnum machineRegionEnum : MachineRegionEnum.values()) { - if (machineRegionEnum.getEn().equals(en)) { - return machineRegionEnum; - } - } - return null; - } - -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/MachineTypeEnum.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/MachineTypeEnum.java deleted file mode 100644 index 96507fb23..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/MachineTypeEnum.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.enums; - -import lombok.Getter; - -import java.util.Arrays; -import java.util.Objects; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/11/18 12:42 - */ -@Getter -public enum MachineTypeEnum { - - CONTAINER_MACHINE(0, "容器"), - PHYSICAL_MACHINE(1, "物理机"); - - private final Integer type; - private final String name; - - MachineTypeEnum(Integer type, String name) { - this.type = type; - this.name = name; - } - - public static MachineTypeEnum queryEnumByType(int type) { - return Arrays.stream(MachineTypeEnum.values()).filter(machineTypeEnum -> { - if (Objects.equals(machineTypeEnum.type, type)) { - return true; - } - return false; - }).findFirst().orElse(null); - } - - public static String queryNameByType(int type) { - MachineTypeEnum machineTypeEnum = queryEnumByType(type); - if (null != machineTypeEnum) { - return machineTypeEnum.getName(); - } - return ""; - } -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/MiddlewareEnum.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/MiddlewareEnum.java deleted file mode 100644 index 59de38e18..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/MiddlewareEnum.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.enums; - -import lombok.Getter; - -import java.util.Arrays; - -/** - * @author wtt - * @version 1.0 - * @description this should be associated with MQSourceEnum, - * and values shouldn't be added arbitrarily - * @date 2021/9/17 15:57 - */ -@Getter -public enum MiddlewareEnum { - - ROCKETMQ(1, "rocketmq", "RocketMQService"), - KAFKA(2, "kafka", "KafkaService"), - NCOS(3, "nacos", ""), - ELASTICSEARCH(4, "elasticsearch", ""), - HDFS(5, "hdfs", ""), - LOKI(6, "loki", ""), - ; - - private final Integer code; - private final String name; - - private final String serviceName; - - MiddlewareEnum(Integer code, String name, String serviceName) { - this.code = code; - this.name = name; - this.serviceName = serviceName; - } - - public static MiddlewareEnum queryByCode(Integer code) { - if (null == code) { - return null; - } - return Arrays.stream(MiddlewareEnum.values()).sequential().filter(middlewareEnum -> { - if (middlewareEnum.getCode().intValue() == code.intValue()) { - return true; - } - return false; - }).findFirst().orElse(null); - } - - public static String queryNameByCode(Integer code) { - String name = ""; - for (MiddlewareEnum middlewareEnum : MiddlewareEnum.values()) { - if (middlewareEnum.getCode().equals(code)) { - name = middlewareEnum.getName(); - } - } - return name; - } - - public static MiddlewareEnum queryByName(String name) { - if (name == null || "".equals(name)) { - return null; - } - return Arrays.stream(MiddlewareEnum.values()).sequential().filter(middlewareEnum -> { - if (middlewareEnum.getName().equals(name)) { - return true; - } - return false; - }).findFirst().orElse(null); - } -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/OperateEnum.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/OperateEnum.java deleted file mode 100644 index db26efd32..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/OperateEnum.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.enums; - -import lombok.Getter; - -import java.util.Arrays; -import java.util.Objects; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/16 16:45 - */ -@Getter -public enum OperateEnum { - - ADD_OPERATE(1, "Add"), - UPDATE_OPERATE(2, "Update"), - DELETE_OPERATE(3, "Delete"), - STOP_OPERATE(4, "Stop"); - - private final Integer code; - private final String describe; - - OperateEnum(Integer code, String describe) { - this.code = code; - this.describe = describe; - } - - public static OperateEnum queryByCode(Integer code) { - return Arrays.stream(OperateEnum.values()) - .filter(operateEnum -> Objects.equals(operateEnum.getCode(), code)) - .findFirst() - .orElse(null); - } -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/ProjectSourceEnum.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/ProjectSourceEnum.java deleted file mode 100644 index 73e0bf7a8..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/ProjectSourceEnum.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.enums; - -import lombok.Getter; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/2 14:25 - */ -@Getter -public enum ProjectSourceEnum { - - ONE_SOURCE(0, "test1", "test1"), - TWO_SOURCE(1, "test2", "test2"), - NO_KNOW_SOURCE(-1, "un_know", "Unknown"); - - private final Integer code; - private final String source; - private final String describe; - - ProjectSourceEnum(Integer code, String source, String describe) { - this.code = code; - this.source = source; - this.describe = describe; - } - - private static Map codeLookup = new HashMap(); - private static Map sourceLookup = new HashMap(); - - static { - Arrays.stream(values()).forEach(projectSourceEnum -> { - codeLookup.put(projectSourceEnum.code, projectSourceEnum); - sourceLookup.put(projectSourceEnum.source, projectSourceEnum); - }); - } - - public static ProjectSourceEnum queryForCode(int code) { - ProjectSourceEnum anEnum = codeLookup.get(code); - return anEnum == null ? NO_KNOW_SOURCE : anEnum; - } - - public static ProjectSourceEnum queryForSource(String source) { - ProjectSourceEnum anEnum = sourceLookup.get(source); - return anEnum == null ? NO_KNOW_SOURCE : anEnum; - } - - public static int queryCodeBySource(String source) { - - return Arrays.stream(ProjectSourceEnum.values()).filter(machineTypeEnum -> { - if (Objects.equals(machineTypeEnum.source, source)) { - return true; - } - return false; - }).findFirst().map(ProjectSourceEnum::getCode).orElse(null); - } - - public static String querySourceByDesc(String desc) { - - return Arrays.stream(ProjectSourceEnum.values()).filter(machineTypeEnum -> { - if (Objects.equals(machineTypeEnum.describe, desc)) { - return true; - } - return false; - }).findFirst().map(ProjectSourceEnum::getSource).orElse(ONE_SOURCE.getSource()); - } - -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/ProjectTypeEnum.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/ProjectTypeEnum.java deleted file mode 100644 index 890b3e2ce..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/ProjectTypeEnum.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.enums; - -import lombok.Getter; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/10/14 10:28 - */ -@Getter -public enum ProjectTypeEnum { - - MIONE_TYPE(0, "hera"); - private final Integer code; - private final String type; - - ProjectTypeEnum(Integer code, String type) { - this.code = code; - this.type = type; - } - - public static String queryTypeByCode(Integer code) { - for (ProjectTypeEnum value : ProjectTypeEnum.values()) { - if (value.getCode().equals(code)) { - return value.getType(); - } - } - return ""; - } -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/RateLimitEnum.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/RateLimitEnum.java deleted file mode 100644 index 623128684..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/RateLimitEnum.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.enums; - -import com.xiaomi.mone.log.api.filter.Common; -import com.xiaomi.mone.log.api.filter.RateLimitStrategy; -import com.xiaomi.mone.log.api.model.meta.FilterDefine; -import lombok.Getter; - -import java.util.Arrays; -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/2/22 15:09 - */ -@Getter -public enum RateLimitEnum { - - RATE_LIMIT_FAST("FAST"), - RATE_LIMIT_MEDIUM("MEDIUM"), - RATE_LIMIT_SLOW("SLOW"), - RATE_LIMIT_NONE("NONE"); - - private final String rateLimit; - - RateLimitEnum(String rateLimit) { - this.rateLimit = rateLimit; - } - - public static RateLimitEnum queryByRateLimit(String rateLimit) { - return Arrays.stream(RateLimitEnum.values()).filter(rateLimitEnum -> rateLimit.equals(rateLimitEnum.getRateLimit())).findFirst().orElse(null); - } - - - public static String consTailRate(List defines) { - // DEFAULT MEDIUM SPEED ACQUISITION - if (defines == null) { - return RATE_LIMIT_MEDIUM.getRateLimit(); - } - for (FilterDefine define : defines) { - if (define != null && define.getCode() != null && define.getCode().startsWith(Common.RATE_LIMIT_CODE)) { - if (define.getCode().equals(RateLimitStrategy.REGINAL_FAST.getCode())) { - return RATE_LIMIT_FAST.getRateLimit(); - } else if (define.getCode().equals(RateLimitStrategy.REGINAL_MEDIUM.getCode())) { - return RATE_LIMIT_MEDIUM.getRateLimit(); - } else if (define.getCode().equals(RateLimitStrategy.REGINAL_SLOW.getCode())) { - return RATE_LIMIT_SLOW.getRateLimit(); - } else if (define.getCode().equals(RateLimitStrategy.REGINAL_NONE.getCode())) { - return RATE_LIMIT_NONE.getRateLimit(); - } - } - } - return RATE_LIMIT_MEDIUM.getRateLimit(); - } - -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/ResourceEnum.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/ResourceEnum.java deleted file mode 100644 index e08e6c04d..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/ResourceEnum.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.enums; - -import lombok.Getter; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/17 15:57 - */ -@Getter -public enum ResourceEnum { - - MQ(1, "MQ Info"), STORAGE(4, "Storage Info"); - - private final Integer code; - private final String name; - - - ResourceEnum(Integer code, String name) { - this.code = code; - this.name = name; - } - -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/filter/Common.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/filter/Common.java deleted file mode 100644 index 0bd6012b0..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/filter/Common.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.filter; - -/** - * @author milog - */ -public class Common { - /** - * filter args - */ - - public static final String PERMITS_PER_SECOND = "permitsPerSecond"; - - /** - * filter code prefix - */ - - public static final String RATE_LIMIT_CODE = "RATELIMITCODE"; -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/filter/RateLimitStrategy.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/filter/RateLimitStrategy.java deleted file mode 100644 index ba01070d2..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/filter/RateLimitStrategy.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.filter; - -import com.xiaomi.mone.log.api.model.meta.FilterType; -import lombok.Getter; - -/** - * @author milog - * @version 1.0 - * @description - * @date 2022/2/22 15:41 - */ -@Getter -public enum RateLimitStrategy { - /** - * permitsPerSecond Number of throttling times per second - * For the log collection scenario, limit the number of strips = permitsPerSecond batchSize - */ - REGINAL_FAST(Common.RATE_LIMIT_CODE + 0, FilterType.REGIONAL, 100, "", 1500), - REGINAL_MEDIUM(Common.RATE_LIMIT_CODE + 1, FilterType.REGIONAL, 100, "", 100), - REGINAL_SLOW(Common.RATE_LIMIT_CODE + 2, FilterType.REGIONAL, 100, "", 30), - REGINAL_NONE(Common.RATE_LIMIT_CODE + 3, FilterType.REGIONAL, 100, "", 10), - - GLOBAL_FAST(Common.RATE_LIMIT_CODE + 4, FilterType.GLOBAL, 10, "", 50), - GLOBAL_MEDIUM(Common.RATE_LIMIT_CODE + 5, FilterType.GLOBAL, 10, "", 20), - GLOBAL_SLOW(Common.RATE_LIMIT_CODE + 6, FilterType.GLOBAL, 10, "", 10), - GLOBAL_NONE(Common.RATE_LIMIT_CODE + 7, FilterType.GLOBAL, 10, "", 5); - - private String code; - private Integer order; - private FilterType type; - private String lifecycle; - private Integer permitsPerSecond; - - - RateLimitStrategy(String code, FilterType type, Integer order, String lifecycle, Integer permitsPerSecond) { - this.code = code; - this.order = order; - this.type = type; - this.lifecycle = lifecycle; - this.permitsPerSecond = permitsPerSecond; - } - - public static RateLimitStrategy getRateLimiterStrategy(String code) { - if (code == null) { - return null; - } - switch (code) { - case Common.RATE_LIMIT_CODE + 0: - return REGINAL_FAST; - case Common.RATE_LIMIT_CODE + 1: - return REGINAL_MEDIUM; - case Common.RATE_LIMIT_CODE + 2: - return REGINAL_SLOW; - case Common.RATE_LIMIT_CODE + 3: - return REGINAL_NONE; - case Common.RATE_LIMIT_CODE + 4: - return GLOBAL_FAST; - case Common.RATE_LIMIT_CODE + 5: - return GLOBAL_MEDIUM; - case Common.RATE_LIMIT_CODE + 6: - return GLOBAL_SLOW; - case Common.RATE_LIMIT_CODE + 7: - return GLOBAL_NONE; - default: - return null; - } - } -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/bo/AlertInfo.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/bo/AlertInfo.java deleted file mode 100644 index ff4194771..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/bo/AlertInfo.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.bo; - -import lombok.Data; - -import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; - -@Data -public class AlertInfo implements Serializable { - - private long alertId; - private long ruleId; - private Map info; - - public void addInfo(String key, String value) { - if (info == null) { - info = new HashMap<>(); - } - info.put(key, value); - } - - public String getInfo(String key) { - if (info == null) { - info = new HashMap<>(); - } - return info.get(key); - } - - public String getInfoOrDefault(String key, String defaultValue) { - if (info == null || key == null || "".equals(key)) { - return defaultValue; - } - return info.get(key) != null && !"".equals(info.get(key)) ? info.get(key) : defaultValue; - } - -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/bo/MiLogMoneTransfer.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/bo/MiLogMoneTransfer.java deleted file mode 100644 index 46938ca31..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/bo/MiLogMoneTransfer.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.bo; - -import lombok.Data; - -import java.io.Serializable; -import java.util.List; - -/** - * @author: wtt - * @date: 2022/5/24 18:24 - * @description: - */ -@Data -public class MiLogMoneTransfer implements Serializable { - private Long milogAppId; - private Long appId; - private String appName; - private Long envId; - private String envName; - private List tailNames; -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/bo/MiLogResource.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/bo/MiLogResource.java deleted file mode 100644 index 70f697664..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/bo/MiLogResource.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.bo; - -import com.xiaomi.mone.log.api.model.vo.EsIndexVo; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/5/10 17:52 - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -public class MiLogResource implements Serializable { - private Long id; - private Integer operateCode; - private Integer resourceCode; - private String storageType; - private String alias; - private String clusterName; - private String regionEn; - private String conWay; - private String serviceUrl; - private String ak; - private String sk; - private String brokerName; - private String orgId; - private String teamId; - private Integer isDefault; - private String esToken; - private String catalog; - private String database; - private Integer mqType; - private List labels = new ArrayList<>(0); - private List multipleEsIndex = new ArrayList<>(); -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/bo/ResourcePage.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/bo/ResourcePage.java deleted file mode 100644 index d975e5cab..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/bo/ResourcePage.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.bo; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * @author: wtt - * @Date: 2022/5/12 14:54 - * @Description: - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -public class ResourcePage { - private Integer resourceCode; - private String regionEnCode; - private String aliasName; - private Integer page = 1; - private Integer pageSize = 10; -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/dto/MontorAppDTO.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/dto/MontorAppDTO.java deleted file mode 100644 index 75a2f5e58..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/dto/MontorAppDTO.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.dto; - -import lombok.Data; - -import java.io.Serializable; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/12/8 14:27 - */ -@Data -public class MontorAppDTO implements Serializable { - private Long appId; - private String appName; - private String source; - private Boolean isAccess = false; -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/dto/TraceLogDTO.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/dto/TraceLogDTO.java deleted file mode 100644 index 260d77fa6..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/dto/TraceLogDTO.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.dto; - -import lombok.Data; - -import java.io.Serializable; -import java.util.Set; -import java.util.TreeSet; - -@Data -public class TraceLogDTO implements Serializable { - private Set dataList; - - public TraceLogDTO(Set dataList) { - this.dataList = dataList; - } - - public static TraceLogDTO emptyData() { - return new TraceLogDTO(new TreeSet<>()); - } -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/AppLogMeta.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/AppLogMeta.java deleted file mode 100644 index 574493088..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/AppLogMeta.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.meta; - -import lombok.Data; - -import java.io.Serializable; -import java.util.List; - -/** - * @author shanwb - * @date 2021-07-08 - */ -@Data -public class AppLogMeta implements Serializable { - - private Long appId; - - private String appName; - - private List logPatternList; - -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/FilterConf.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/FilterConf.java deleted file mode 100644 index 1ab746111..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/FilterConf.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.meta; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; - -import java.io.Serializable; -import java.util.Map; - -@Data -@NoArgsConstructor -@AllArgsConstructor -@EqualsAndHashCode -public class FilterConf implements Serializable { - private String code; - private FilterName name; - private FilterType type; - private Integer order; - private String lifecycle; - private Map args; -} \ No newline at end of file diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/FilterDefine.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/FilterDefine.java deleted file mode 100644 index 47bdd48a7..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/FilterDefine.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.meta; - -import com.xiaomi.mone.log.api.enums.RateLimitEnum; -import com.xiaomi.mone.log.api.filter.Common; -import com.xiaomi.mone.log.api.filter.RateLimitStrategy; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; - -/** - * Used to transfer filter parameters between manage and agent - * - * @author milog - */ -@Data -@NoArgsConstructor -@AllArgsConstructor -public class FilterDefine implements Serializable { - /** - * One-to-one correspondence with the code in Filter Conf - */ - private String code; - - private Map args; - - public static FilterDefine Of(String code, Map args) { - return new FilterDefine(code, args); - } - - public static FilterDefine consFilterDefine(RateLimitStrategy strategy) { - return FilterDefine.Of(strategy.getCode(), new HashMap() {{ - put(Common.PERMITS_PER_SECOND, String.valueOf(strategy.getPermitsPerSecond())); - }}); - } - - public static FilterDefine consRateLimitFilterDefine(String rateLimit) { - if (rateLimit == null) { - return null; - } - RateLimitEnum rateLimitEnum = RateLimitEnum.queryByRateLimit(rateLimit); - switch (rateLimitEnum) { - case RATE_LIMIT_FAST: - return FilterDefine.consFilterDefine(RateLimitStrategy.REGINAL_FAST); - case RATE_LIMIT_MEDIUM: - return FilterDefine.consFilterDefine(RateLimitStrategy.REGINAL_MEDIUM); - case RATE_LIMIT_SLOW: - return FilterDefine.consFilterDefine(RateLimitStrategy.REGINAL_SLOW); - case RATE_LIMIT_NONE: - return FilterDefine.consFilterDefine(RateLimitStrategy.REGINAL_NONE); - default: - return null; - } - } - -} \ No newline at end of file diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/FilterName.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/FilterName.java deleted file mode 100644 index 694c1b520..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/FilterName.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.meta; - -import lombok.Getter; - -@Getter -public enum FilterName { - RATELIMITER("RATELIMITER"); - - private String name; - - FilterName(String name) { - this.name = name; - } -} \ No newline at end of file diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/FilterType.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/FilterType.java deleted file mode 100644 index badc686c0..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/FilterType.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.meta; - -public enum FilterType { - GLOBAL("GLOBAL"), - REGIONAL("REGIONAL"); - - private String type; - - FilterType(String type) { - this.type = type; - } -} \ No newline at end of file diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/LogCollectMeta.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/LogCollectMeta.java deleted file mode 100644 index 853c0b4a7..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/LogCollectMeta.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.meta; - -import lombok.Data; - -import java.io.Serializable; -import java.util.List; - -/** - * @author shanwb - * @date 2021-07-08 - */ -@Data -public class LogCollectMeta implements Serializable { - /** - * agent mark - */ - private String agentId; - /** - * agent machine name - */ - private String agentMachine; - /** - * agent physical machine ip - */ - private String agentIp; - /** - * Application metadata collected by the agent - */ - private List appLogMetaList; - - private AgentDefine agentDefine; - - /** - * A single configuration data is configured by default to the full configuration of the machine - */ - private Boolean singleMetaData; - - private String podType; - - /** - * Log collection in this directory that needs to be deleted when a machine is offline has a value only when - * the machine of an application is offline - */ - private String delDirectory; - -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/LogPattern.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/LogPattern.java deleted file mode 100644 index fabf89074..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/LogPattern.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.meta; - -import com.xiaomi.mone.log.api.enums.OperateEnum; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.io.File; -import java.io.Serializable; -import java.util.List; - -/** - * @author shanwb - * @date 2021-07-19 - */ -@Data -public class LogPattern implements Serializable { - /** - * logtail primary key ID - */ - private Long logtailId; - - private String tailName; - - private List ips; - - /** - * IP and directory correspondence - */ - private List ipDirectoryRel; - - /** - * LogTypeEnum.name()， - * Based on this type, it can be determined whether to collect single or multiple lines (supporting JAVA exception stacks) - * - * @see com.xiaomi.mone.log.api.enums.LogTypeEnum - */ - private Integer logType; - - /** - * Log path， - */ - private String logPattern; - /** - * Log slicing expression - */ - private String logSplitExpress; - - /** - * Row regex - */ - private String firstLineReg; - - /** - * Each pathCode corresponds to a different mq tag; - * Generated by the app + logPath combination for message isolation - */ - private String patternCode; - /** - * The default is not delete - */ - private OperateEnum operateEnum; - - private List filters; - - /** - * mq configuration - */ - private MQConfig mQConfig; - - /** - * The IP corresponding relationship, if it is not on K8S, there is only 1, K8S DeamonSet deployment hand, key for each pod name - */ - @Data - @AllArgsConstructor - @NoArgsConstructor - @Builder - public static class IPRel implements Serializable { - private String key = File.separator; - private String ip; - } -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/MQConfig.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/MQConfig.java deleted file mode 100644 index a27fccb6d..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/MQConfig.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.meta; - -import lombok.Data; - -import java.io.Serializable; - -/** - * @author shanwb - * @date 2021-07-19 - */ -@Data -public class MQConfig implements Serializable { - /** - * rocketmq、talos - */ - private String type; - - /** - * mq：namesrv_addr - */ - private String clusterInfo; - - private String producerGroup; - - private String ak; - - private String sk; - - private String topic; - - private String tag; - - private Integer partitionCnt; - - /** - * es consumption group, which can be extended to other groups for other analysis scenarios - */ - private String esConsumerGroup; - - private Integer batchSendSize; - -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/package-info.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/package-info.java deleted file mode 100644 index c198e5483..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/meta/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * @author shanwb - * @date 2021-07-19 - */ -package com.xiaomi.mone.log.api.model.meta; -/** - * Encapsulated under the meta package are structs that interact with the log-manager - **/ \ No newline at end of file diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/msg/LineMessage.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/msg/LineMessage.java deleted file mode 100644 index 02507f75f..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/msg/LineMessage.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.msg; - -import lombok.ToString; - -import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; - -/** - * @author shanwb - * @date 2021-07-19 - */ -@ToString -public class LineMessage implements Serializable { - - public static transient String KEY_IP = "ip"; - public static transient String KEY_COLLECT_TIMESTAMP = "ct"; - public static transient String KEY_MQ_TOPIC = "t"; - public static transient String KEY_MQ_TOPIC_TAG = "tag"; - public static transient String KEY_MESSAGE_TYPE = "type"; - - private Long lineNumber; - - private String fileName; - - private Long pointer; - - private Integer msgLength; - - private String msgBody; - - private Map extMap; - - public LineMessage() { - } - - public long getTimestamp() { - String value = extMap.get(KEY_COLLECT_TIMESTAMP); - if (value == null || "".equals(value)) { - return 0; - } else { - return Long.parseLong(value); - } - } - - public void setTimeStamp(long time) { - extMap.put(KEY_COLLECT_TIMESTAMP, String.valueOf(time)); - } - - public Long getLineNumber() { - return lineNumber; - } - - public void setLineNumber(Long lineNumber) { - this.lineNumber = lineNumber; - } - - public Long getPointer() { - return pointer; - } - - public void setPointer(Long pointer) { - this.pointer = pointer; - } - - public Integer getMsgLength() { - return msgLength; - } - - public void setMsgLength(Integer msgLength) { - this.msgLength = msgLength; - } - - public void setMsgBody(String msgBody) { - this.msgBody = msgBody; - } - - public String getMsgBody() { - return msgBody; - } - - public Map getExtMap() { - return extMap; - } - - public void setExtMap(Map extMap) { - this.extMap = extMap; - } - - public synchronized void setProperties(String key, String value) { - if (null == extMap) { - extMap = new HashMap<>(); - } - extMap.put(key, value); - } - - public String getProperties(String key) { - return this.getProperties(key, null); - } - - public String getProperties(String key, String defaultValue) { - if (null == this.extMap) { - return defaultValue; - } - - if (key == null || "".equals(key)) { - return defaultValue; - } - - String value = this.extMap.get(key); - if (key == null || "".equals(key)) { - return defaultValue; - } - - return value; - } - - public String getFileName() { - return fileName; - } - - public void setFileName(String fileName) { - this.fileName = fileName; - } -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/AgentLogProcessDTO.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/AgentLogProcessDTO.java deleted file mode 100644 index 3e6274b0a..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/AgentLogProcessDTO.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.vo; - -import lombok.Data; - -import java.io.Serializable; - -@Data -public class AgentLogProcessDTO implements Serializable { - private String path; - private Long fileRowNumber; - private Long pointer; - private Long fileMaxPointer; - private String appName; - private String collectPercentage; - private Long collectTime; -// private String envName; -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/AlarmPattern.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/AlarmPattern.java deleted file mode 100644 index 6d7684637..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/AlarmPattern.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.vo; - -import lombok.Builder; -import lombok.Data; - -/** - * @author: wtt - * @date: 2022/5/24 14:59 - * @description: - */ -@Data -@Builder -public class AlarmPattern { - /** - * The number of times it was matched - */ - private Integer count; - - private String matchMessage; -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/CommonVo.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/CommonVo.java deleted file mode 100644 index e319c46ff..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/CommonVo.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.vo; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.experimental.SuperBuilder; - -import java.io.Serializable; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/5/10 16:27 - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -@SuperBuilder -public class CommonVo implements Serializable { - private Long ctime; - private Long utime; - private String creator; - private String updater; -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/EsIndexVo.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/EsIndexVo.java deleted file mode 100644 index 69eba4b0c..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/EsIndexVo.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.vo; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.List; - -/** - * @Auther: wtt - * @Date: 2022/5/12 12:57 - * @Description: - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -public class EsIndexVo { - private Integer logTypeCode; - private String logTypeName; - private List esIndexList; -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/LogCmd.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/LogCmd.java deleted file mode 100644 index 22c362fa9..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/LogCmd.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.vo; - - -/** - * @author goodjava@qq.com - */ -public class LogCmd { - - - public static final int logReq = 3000; - public static final int logRes = 3001; - - - -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/LogRecordConfig.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/LogRecordConfig.java deleted file mode 100644 index f9c0bf3c5..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/LogRecordConfig.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.vo; - -import lombok.Data; - -import java.io.Serializable; - -/** - * @Author goodjava@qq.com - * @Date 2021/6/24 11:07 - */ -@Data -public class LogRecordConfig implements Serializable { - - private String id; - - private String logPath; - - private String app; - - private String group; - - -} \ No newline at end of file diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/MiLogMoneEnv.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/MiLogMoneEnv.java deleted file mode 100644 index 2595f79d4..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/MiLogMoneEnv.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.vo; - -import lombok.Data; - -import java.io.Serializable; - -/** - * @author: wtt - * @date: 2022/5/24 18:29 - * @description: - */ -@Data -public class MiLogMoneEnv implements Serializable { - private Long oldAppId; - private String oldAppName; - private Long oldEnvId; - private String oldEnvName; - private Long newAppId; - private String newAppName; - private Long newEnvId; - private String newEnvName; - /** - * If it rolls, rollback = 1 - */ - private Integer rollback; -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/PingReq.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/PingReq.java deleted file mode 100644 index ea1567729..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/PingReq.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.vo; - -import lombok.Data; - -import java.io.Serializable; - -/** - * @Author goodjava@qq.com - * @Date 2021/6/24 11:56 - */ -@Data -public class PingReq implements Serializable { - - - private String message; - - private String ip; - -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/ResourceInfo.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/ResourceInfo.java deleted file mode 100644 index e4bfb2bf1..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/ResourceInfo.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.vo; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.experimental.SuperBuilder; - -import java.io.Serializable; -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/5/10 11:25 - */ -@Data -@AllArgsConstructor -@SuperBuilder -public class ResourceInfo extends CommonVo implements Serializable { - private Long id; - private String alias; - private String regionEn; - private String regionCn; - private String serviceUrl; - private String ak; - private String sk; - private String orgId; - private String teamId; - private String clusterName; - private String brokerName; - private String esToken; - private String conWay; - private String catalog; - private String database; - private List labels; - private List multipleEsIndex; - private String storageType; - private Integer mqType; -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/ResourceUserSimple.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/ResourceUserSimple.java deleted file mode 100644 index e49462d2c..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/ResourceUserSimple.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.vo; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/5/11 14:26 - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -@Builder -public class ResourceUserSimple { - /** - * Whether the resource list is initialized - */ - private Boolean initializedFlag = false; - /** - * The presentation message content is not initialized - */ - private String notInitializedMsg; - /** - * Whether the list of resources is displayed - */ - private Boolean showFlag = false; - /** - * After displaying the MQ resource list, the list is filtered - */ - private List> mqResourceList; - /** - * After displaying the ES resource list, the list is filtered - */ - private List> storageResourceList; - - -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/TailLogProcessDTO.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/TailLogProcessDTO.java deleted file mode 100644 index 42f2551ed..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/TailLogProcessDTO.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.vo; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.io.Serializable; - -@Data -@NoArgsConstructor -@AllArgsConstructor -@Builder -public class TailLogProcessDTO implements Serializable { - private String tailName; - private String ip; - private String path; - private Long fileRowNumber; - private Long collectTime; - private String collectPercentage; - -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/TraceLogQuery.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/TraceLogQuery.java deleted file mode 100644 index 6a7504789..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/TraceLogQuery.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.vo; - -import lombok.Data; - -import java.io.Serializable; - -@Data -public class TraceLogQuery implements Serializable { - private Long appId; - private String ip; - private String traceId; - private String generationTime; - private String level; - // The maximum number of returned data bars - private Integer total = 1000; - // ES query expiration time (ms) - private Long timeout = 2000L; - - public TraceLogQuery(Long appId, String ip, String traceId) { - this.appId = appId; - this.ip = ip; - this.traceId = traceId; - } - - public TraceLogQuery() { - } -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/UpdateLogProcessCmd.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/UpdateLogProcessCmd.java deleted file mode 100644 index 975e4559e..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/UpdateLogProcessCmd.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.vo; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; - -import java.io.Serializable; -import java.util.List; - -/** - * It doesn't matter if docker corresponds to one IP, K8S has multiple pods under a node, so a tail may have multiple IPs - */ -@Data -public class UpdateLogProcessCmd implements Serializable { - /** - * Machine IP->K8S corresponds to Node IP - */ - private String ip; - - private List collectList; - - @Data - @EqualsAndHashCode - public static class CollectDetail implements Serializable { - - private String tailId; - - private String tailName; - /** - * k8s may have multiple IPs - */ - private List ipList; - /** - * Configured path (original path) - */ - private String path; - - private Long appId; - - private String appName; - - private List fileProgressDetails; - } - - @Data - @Builder - @AllArgsConstructor - @NoArgsConstructor - @EqualsAndHashCode - public static class FileProgressDetail implements Serializable { - // The path to the real acquisition file - private String pattern; - // Design the configured IP - private String configIp; - - // Log file line number - private Long fileRowNumber; - - private Long pointer; - - private Long fileMaxPointer; - - // Collection time - private Long collectTime; - - // Collect progress - private String collectPercentage; - } -} - - diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/ValueKeyObj.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/ValueKeyObj.java deleted file mode 100644 index 707ebdedb..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/model/vo/ValueKeyObj.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.model.vo; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/5/11 15:52 - */ -@Data -@NoArgsConstructor -@AllArgsConstructor -public class ValueKeyObj { - private T key; - private String value; -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/service/AgentConfigService.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/service/AgentConfigService.java deleted file mode 100644 index 109a6eaea..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/service/AgentConfigService.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.service; - -import com.xiaomi.mone.log.api.model.meta.LogCollectMeta; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/8/4 19:38 - */ -public interface AgentConfigService { - /** - * Obtain log configuration information - * @param ip - * @return - */ - LogCollectMeta getLogCollectMetaFromManager(String ip); -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/service/LogCountService.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/service/LogCountService.java deleted file mode 100644 index 6a01c567f..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/service/LogCountService.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.service; - -public interface LogCountService { - void alarmLogNum(); -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/service/LogDataService.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/service/LogDataService.java deleted file mode 100644 index 048338fbf..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/service/LogDataService.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.service; - -import com.xiaomi.mone.log.api.model.dto.TraceLogDTO; -import com.xiaomi.mone.log.api.model.vo.TraceLogQuery; - -import java.io.IOException; - -public interface LogDataService { - /** - * Get trace logs - * @param logQuery - * @return - */ - TraceLogDTO getTraceLog(TraceLogQuery logQuery) throws IOException; -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/service/LogProcessCollector.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/service/LogProcessCollector.java deleted file mode 100644 index eeeac0bf3..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/service/LogProcessCollector.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.service; - -import com.xiaomi.mone.log.api.model.vo.AgentLogProcessDTO; -import com.xiaomi.mone.log.api.model.vo.TailLogProcessDTO; -import com.xiaomi.mone.log.api.model.vo.UpdateLogProcessCmd; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/12/6 14:32 - */ -public interface LogProcessCollector { - - /** - * Update log collection progress - * - * @param cmd - */ - void collectLogProcess(UpdateLogProcessCmd cmd); - - /** - * Query the log collection progress of tail - * - * @param tailId - * @param tailName - * @param targetIp - * @return - */ - List getTailLogProcess(Long tailId, String tailName, String targetIp); - - /** - * query by ip - * - * @param ip - * @return - */ - List getAgentLogProcess(String ip); - - /** - * The acquisition progress is less than the progress ratio - * - * @param progressRation Match the collection progress - * @return - */ - List getColProcessImperfect(Double progressRation); - - /** - * Get the collection details under a tail - * - * @param tailId - * @return - */ - List getFileProcessDetailByTail(Long tailId); - - /** - * get all coll detail - * - * @return - */ - List getAllCollectDetail(String ip); -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/service/LogProcessService.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/service/LogProcessService.java deleted file mode 100644 index 11e09a0a5..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/service/LogProcessService.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.service; - -import com.xiaomi.mone.log.api.model.vo.UpdateLogProcessCmd; - -public interface LogProcessService { - - /** - * Update log collection progress - * @param cmd - */ - void updateLogProcess(UpdateLogProcessCmd cmd); -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/service/MilogOpenService.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/service/MilogOpenService.java deleted file mode 100644 index 7864fce39..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/service/MilogOpenService.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.service; - -import com.xiaomi.mone.log.api.model.bo.MiLogMoneTransfer; -import com.xiaomi.mone.log.api.model.dto.MontorAppDTO; -import com.xiaomi.mone.log.api.model.vo.MiLogMoneEnv; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/12/8 14:53pom - */ -public interface MilogOpenService { - /** - * Query whether the log system is connected - * - * @param iamTreeId - * @return - */ - MontorAppDTO queryHaveAccessMilog(Long iamTreeId, String bingId, Integer platformType); - - /** - * Query the last spaceId accessed - * - * @param iamTreeId - * @return - */ - Long querySpaceIdByIamTreeId(Long iamTreeId); - - /** - * Mione Migration Data Cleansing - */ - MiLogMoneTransfer ypMoneEnvTransfer(MiLogMoneEnv logMoneEnv); -} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/service/PublishConfigService.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/service/PublishConfigService.java deleted file mode 100644 index 559360e9e..000000000 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/service/PublishConfigService.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api.service; - -import com.xiaomi.mone.log.api.model.meta.LogCollectMeta; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/12/6 17:42 - */ -public interface PublishConfigService { - - void sengConfigToAgent(final String agentIp, LogCollectMeta logCollectMeta); - - List getAllAgentList(); -} diff --git a/ozhera-log/log-api/src/test/java/com/xiaomi/mone/log/api/LogStorageTypeEnumTest.java b/ozhera-log/log-api/src/test/java/com/xiaomi/mone/log/api/LogStorageTypeEnumTest.java deleted file mode 100644 index ae36bdfbb..000000000 --- a/ozhera-log/log-api/src/test/java/com/xiaomi/mone/log/api/LogStorageTypeEnumTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.api; - -import com.xiaomi.mone.log.api.enums.LogStorageTypeEnum; -import lombok.extern.slf4j.Slf4j; -import org.junit.Assert; -import org.junit.Test; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/11/10 14:35 - */ -@Slf4j -public class LogStorageTypeEnumTest { - - @Test - public void queryByNameTest() { - String name = "doris"; - LogStorageTypeEnum storageTypeEnum = LogStorageTypeEnum.queryByName(name); - Assert.assertNotNull(storageTypeEnum); - log.info("result:{}", storageTypeEnum); - } -} diff --git a/ozhera-log/log-common/pom.xml b/ozhera-log/log-common/pom.xml index ad54f7ebb..1299283d9 100644 --- a/ozhera-log/log-common/pom.xml +++ b/ozhera-log/log-common/pom.xml @@ -3,9 +3,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - run.mone + org.apache.ozhera ozhera-log - 1.4.0-jdk21 + 2.0.0-SNAPSHOT log-common diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/annotation/UnifiedResponse.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/annotation/UnifiedResponse.java deleted file mode 100644 index 92d173b9c..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/annotation/UnifiedResponse.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.annotation; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** - * @author wtt - * @version 1.0 - * @description Uniform packaging response - * @date 2022/5/10 11:33 - */ -@Target({ElementType.TYPE, ElementType.METHOD}) -@Retention(RetentionPolicy.RUNTIME) -public @interface UnifiedResponse { -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/Config.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/Config.java deleted file mode 100644 index 3bbe8c771..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/Config.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.common; - -import lombok.extern.slf4j.Slf4j; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Properties; - -/** - * @author goodjava@qq.com - */ -@Slf4j -public class Config { - - private Properties properties; - - - private Config() { - InputStream is = Config.class.getClassLoader().getResourceAsStream("config.properties"); - properties = new Properties(); - try { - properties.load(is); - } catch (IOException e) { - log.warn("load config error:{}", e.getMessage()); - } - } - - private final static class LazyHolder { - private static Config ins = new Config(); - } - - public final static Config ins() { - return LazyHolder.ins; - } - - - public String get(String key, String defaultValue) { - return properties.getOrDefault(key, defaultValue).toString().trim(); - } - - - public void set(String key, String value) { - properties.setProperty(key, value); - } - -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/Constant.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/Constant.java deleted file mode 100644 index 3ea644ed6..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/Constant.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.common; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/9 10:45 - */ -public class Constant { - /** - * mq http The request succeeds with the return code - */ - public static final int SUCCESS_CODE = 100; - public static final Double SUCCESS_CODE_DOUBLE = 100.0; - - public static final int RPCCMD_AGENT_CODE = 13579; - public static final int RPCCMD_AGENT_CONFIG_CODE = 24680; - /** - * mq http The request returned a message successfully - */ - public static final String SUCCESS_MESSAGE = "success"; - - public static final String COMMON_MESSAGE = "common"; - - public static final String STREAM_CONTAINER_POD_NAME_KEY = "MONE_CONTAINER_S_POD_NAME"; - - public static final String DEFAULT_STREAM_SERVER_NAME = "hera_log_stream"; - - public static final String HAS_CREATED_MESSAGE = "This name has been used!"; - /** - * Operator = - */ - public static final String EQUAL_OPERATE = "="; - - public static final String LIKE_OPERATE = "like"; - - public static final String DEFAULT_TAIL_SEPARATOR = "|"; - /** - * MQ message type - */ - public static final String ROCKET_MQ_TYPE = "rocketmq"; - - public static final Integer DEFAULT_IMPORT_LEVEL = 3; - - public static final Integer MAX_IMPORT_LEVEL = 1; - - public static final String DEFAULT_OPERATOR = "system"; - - public static final String DEFAULT_JOB_OPERATOR = "job"; - - public static final String LOG_MANAGE_PREFIX = "log_manage_"; - public static final String LOG_MANAGE_GWDASH_TOKEN = "gwdash_token"; - - public static final String NAMESPACE_CONFIG_DATA_ID = "create_namespace_config"; - - public static final String TAIL_CONFIG_DATA_ID = "create_tail_config:"; - - public static final String DEFAULT_GROUP_ID = "DEFAULT_GROUP"; - - public static final String DEFAULT_APP_NAME = ""; - - public static final Long DEFAULT_TIME_OUT_MS = 3000L; - - public static final String DEFAULT_TAGS = "tags_"; - - public static final String DEFAULT_CONSUMER_GROUP = "subGroup_"; - - public static final String UNDERLINE_SYMBOL = "_"; - - public static final String STRIKETHROUGH_SYMBOL = "-"; - - public static final String ES_INDEX_PREFIX = "milog_"; - - public static final String LOG_STORE = "log-store"; - - public static final String SYMBOL_COLON = ":"; - - public static final String SYMBOL_COMMA = ","; - - public static final String SYMBOL_MULTI = ".*"; - - public static final String SYMBOL_COMMA_SPACE = " "; - - public static final Integer YES = 1; - - public static final Integer COUNT_NUM = 10000; - - public static final String TAILID_KEY = "tailId"; - - public static final String LOG_KEY = "log"; - - public static final String TRACE_ID_KEY = "traceId"; - - public static final String COMMON_MQ_PREFIX = "common_mq_miLog"; - - public static final List COMMON_MQ_SUFFIX = new ArrayList<>(Arrays.asList("first", "second", "third", "fourth", "fifth")); - - public static final Integer COMMON_MQ_PARTITION_NUM = 2; - - public static final String DEFAULT_SPACE_SUFFIX = "namespace"; - - public static final String DEFAULT_STORE_SUFFIX = "store"; - - public static final String DEFAULT_SPACE_DESC = "Our namespaces, all in one basket, belong to: %s project"; - - public static final Integer MIFAAS_STORE_NUM = 10; - - public static final String DEFAULT_COLUMN_TYPE_LIST = "date,keyword,keyword,text,text,keyword,keyword,text,keyword,text,keyword,keyword,keyword,keyword,keyword,keyword,long"; - - public static final String DEFAULT_KEY_LIST = "timestamp:1,level:1,traceId:1,threadName:1,className:1,line:1,methodName:1,message:1,podName:1,faas:2,logstore:3,logsource:3,mqtopic:3,mqtag:3,logip:3,tail:3,linenumber:3"; - - public static final String DEFAULT_VALUE_LIST = "0,1,2,3,4,-1,-1,7,6,5"; - - public static final Integer MIFAAS_APP_TYPE_CODE = 4; - - public static final String MIFAAS_APP_TYPE_TEXT = "serverLess"; - - /** - * RocketMQ data monitoring - */ - - public static final String ROCKETMQ_GROUP_DIFF = "rocketmq_group_diff"; - public static final String ROCKETMQ_CLIENT_CONSUME_FAILED_MSG_COUNT = "rocketmq_client_consume_fail_msg_count"; - public static final String ROCKETMQ_CONSUMER_TPS = "rocketmq_consumer_tps"; - public static final String ROCKETMQ_PRODUCER_TPS = "rocketmq_producer_tps"; - public static final String ROCKETMQ_PRODUCER_OFFSET = "rocketmq_producer_offset"; - - /** - * ES client bean name prefix - */ - public static String LOG_STORAGE_SERV_BEAN_PRE = "logStorageServiceBean_"; - - /** - * ES computer room - */ - public static String ES_REGION_EROUP = "Amstega"; - - public static final String NULLVALUE = "null"; - - public static final String PRODENV = "prod"; - - public static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create(); - - // milog -alarm use - public static final String ACCESS_KEY = "accessKey"; - public static final String SECRET_KEY = "secretKey"; - public static final String PRODUCER_ACCESS_KEY = "producerAccessKey"; - public static final String PRODUCER_SECRET_KEY = "producerSecretKey"; - public static final String ALERT_ID = "alertId"; - public static final String CONSUMER_SERVER = "consumerServer"; - public static final String PRODUCER_SERVER = "producerServer"; - public static final String CONSUMER_TOPIC = "consumerTopic"; - public static final String PRODUCER_TOPIC = "producerTopic"; - public static final String MQ_TYPE = "mqType"; - public static final String CONSUMER_GROUP = "consumerGroup"; - public static final String CONSUMER_TAG = "consumerTag"; - public static final String ALERT_RULES_INPUT = "alertRulesInput"; - public static final String ALERT_PARALLELISM = "parallelism"; - public static final String USER_SET = "userSet"; - - public static final String ES_CONWAY_PWD = "pwd"; - public static final String ES_CONWAY_TOKEN = "token"; -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/Es.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/Es.java deleted file mode 100644 index cc8ada447..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/Es.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.common; - -import java.text.SimpleDateFormat; -import java.util.Date; - -public class Es { - - /** - * Gets the index suffix - * @return - */ - public static String indexPostfix() { - return "-" + new SimpleDateFormat("yyyy.MM.dd").format(new Date()); - } - -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/HeraLocalCache.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/HeraLocalCache.java deleted file mode 100644 index b74026099..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/HeraLocalCache.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.common; - - -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import lombok.extern.slf4j.Slf4j; - -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -/** - * @author shanwb - * @date 2022-12-29 - */ -@Slf4j -public class HeraLocalCache { - private Cache localCache = null; - - public static HeraLocalCache instance() { - return Inner.heraLocalCache; - } - private static class Inner { - private static final HeraLocalCache heraLocalCache = new HeraLocalCache(); - } - - private HeraLocalCache() { - localCache = CacheBuilder.newBuilder() - .recordStats() - .expireAfterWrite(15, TimeUnit.MINUTES) - .maximumSize(5000) - .build(); - - Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> { - try { - log.debug("HeraLocalCache status:{}", localCache.stats().toString()); - } catch (Exception ex) { - //ignore - } - }, 0, 30, TimeUnit.SECONDS); - } - - public void put(String key, Object value) { - this.localCache.put(key, value); - } - - public Object get(String key) { - return this.localCache.getIfPresent(key); - } - - public T getObj(String key, Class c) { - return (T) this.localCache.getIfPresent(key); - } - - public Cache getCache() { - return this.localCache; - } - -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/MiLogCmd.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/MiLogCmd.java deleted file mode 100644 index 1ba3ece17..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/MiLogCmd.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.common; - -/** - * @Author goodjava@qq.com - * @Date 2021/6/24 11:37 - */ -public class MiLogCmd { - - public static final int pingReq = 3000; - public static final int pingRes = 3001; - - -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/NetUtils.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/NetUtils.java deleted file mode 100644 index cb3c24123..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/NetUtils.java +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.common; - -import com.google.common.collect.Lists; -import lombok.extern.slf4j.Slf4j; - -import java.io.IOException; -import java.net.*; -import java.util.Enumeration; -import java.util.List; -import java.util.Random; -import java.util.regex.Pattern; - -/** - * @author goodjava@qq.com - */ -@Slf4j -public class NetUtils { - - - private static volatile InetAddress LOCAL_ADDRESS = null; - - public static final String LOCALHOST = "127.0.0.1"; - public static final String ANYHOST = "0.0.0.0"; - public static final String DOCKERHOST = "127.0.0.1"; - private static final Pattern IP_PATTERN = Pattern.compile("\\d{1,3}(\\.\\d{1,3}){3,5}$"); - - private static final int RND_PORT_START = 30000; - - private static final int MAX_PORT = 65535; - - private static final Random RANDOM = new Random(System.currentTimeMillis()); - - private static final int RND_PORT_RANGE = 10000; - - - public static String getHostName() { - try { - return InetAddress.getLocalHost().getHostName(); - } catch (UnknownHostException e) { - log.error("get host name error:{}", e.getMessage()); - return ""; - } - } - - - public static String getLocalHost() { - InetAddress address = getLocalAddress(); - return address == null ? LOCALHOST : address.getHostAddress(); - } - - public static InetAddress getLocalAddress() { - if (LOCAL_ADDRESS != null) { - return LOCAL_ADDRESS; - } - InetAddress localAddress = getLocalAddress0(); - LOCAL_ADDRESS = localAddress; - return localAddress; - } - - private static InetAddress getLocalAddress0() { - InetAddress localAddress = null; - try { - localAddress = InetAddress.getLocalHost(); - if (localAddress instanceof Inet6Address) { - Inet6Address address = (Inet6Address) localAddress; - if (isValidV6Address(address)) { - return normalizeV6Address(address); - } - } else if (isValidAddress(localAddress)) { - return localAddress; - } - } catch (Throwable e) { - log.warn(e.getMessage()); - } - try { - Enumeration interfaces = NetworkInterface.getNetworkInterfaces(); - if (null == interfaces) { - return localAddress; - } - while (interfaces.hasMoreElements()) { - try { - NetworkInterface network = interfaces.nextElement(); - Enumeration addresses = network.getInetAddresses(); - while (addresses.hasMoreElements()) { - try { - InetAddress address = addresses.nextElement(); - if (address instanceof Inet6Address) { - Inet6Address v6Address = (Inet6Address) address; - if (isValidV6Address(v6Address)) { - return normalizeV6Address(v6Address); - } - } else if (isValidAddress(address)) { - return address; - } - } catch (Throwable e) { - log.warn(e.getMessage()); - } - } - } catch (Throwable e) { - log.warn(e.getMessage()); - } - } - } catch (Throwable e) { - log.warn(e.getMessage()); - } - return localAddress; - } - - static boolean isValidV6Address(Inet6Address address) { - boolean preferIpv6 = Boolean.getBoolean("java.net.preferIPv6Addresses"); - if (!preferIpv6) { - return false; - } - try { - return address.isReachable(100); - } catch (IOException e) { - // ignore - } - return false; - } - - static InetAddress normalizeV6Address(Inet6Address address) { - String addr = address.getHostAddress(); - int i = addr.lastIndexOf('%'); - if (i > 0) { - try { - return InetAddress.getByName(addr.substring(0, i) + '%' + address.getScopeId()); - } catch (UnknownHostException e) { - // ignore - log.debug("Unknown IPV6 address: ", e); - } - } - return address; - } - - - static boolean isValidAddress(InetAddress address) { - if (address == null || address.isLoopbackAddress()) { - return false; - } - String name = address.getHostAddress(); - return (name != null - && !ANYHOST.equals(name) - && !LOCALHOST.equals(name) - && !DOCKERHOST.equals(name) - && IP_PATTERN.matcher(name).matches()); - } - - - public static int getAvailablePort(int port) { - if (port <= 0) { - return getAvailablePort(); - } - for (int i = port; i < MAX_PORT; i++) { - ServerSocket ss = null; - try { - ss = new ServerSocket(i); - return i; - } catch (IOException e) { - // continue - } finally { - if (ss != null) { - try { - ss.close(); - } catch (IOException e) { - } - } - } - } - return port; - } - - public static int getAvailableUdpPort(int port) { - if (port <= 0) { - return getUdpAvailablePort(); - } - for (int i = port; i < MAX_PORT; i++) { - DatagramSocket ss = null; - try { - ss = new DatagramSocket(i); - return i; - } catch (IOException e) { - // continue - } finally { - if (ss != null) { - ss.close(); - } - } - } - return port; - } - - - public static int getAvailablePort() { - ServerSocket ss = null; - try { - ss = new ServerSocket(); - ss.bind(null); - return ss.getLocalPort(); - } catch (IOException e) { - return getRandomPort(); - } finally { - if (ss != null) { - try { - ss.close(); - } catch (IOException e) { - } - } - } - } - - public static int getUdpAvailablePort() { - DatagramSocket ss = null; - try { - ss = new DatagramSocket(); - ss.bind(null); - return ss.getLocalPort(); - } catch (IOException e) { - return getRandomPort(); - } finally { - if (ss != null) { - ss.close(); - } - } - } - - public static List queryIpWithDomain(String domain) { - List ips = Lists.newArrayList(); - try { - InetAddress[] addresses = InetAddress.getAllByName(domain); - for (int i = 0; i < addresses.length; i++) { - ips.add(addresses[i].getHostAddress()); - } - } catch (Exception e) { - log.error("queryIpWithDomain with error:{}", domain, e); - } - return ips; - } - - public static int getRandomPort() { - return RND_PORT_START + RANDOM.nextInt(RND_PORT_RANGE); - } - - -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/PathUtils.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/PathUtils.java deleted file mode 100644 index 88367dad1..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/PathUtils.java +++ /dev/null @@ -1,353 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.common; - -import com.google.common.collect.Lists; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.SYMBOL_MULTI; - -/** - * @author milog - */ -@Slf4j -public class PathUtils { - - /** - * Minimum number of directory hierarchies - */ - private static final int MINIMUM_LEVELS = 3; - /** - * Directory wildcards - */ - public static final String PATH_WILDCARD = "*"; - - public static final String MULTI_FILE_PREFIX = "("; - public static final String MULTI_FILE_SUFFIX = ")"; - private static final String NEO_FILE_PREFIX = "/home/work/logs/neo-logs/("; - public static final String SPLIT_VERTICAL_LINE = "\\|"; - -// public static final String SEPARATOR = FileSystems.getDefault().getSeparator(); - - public static final String SEPARATOR = "/"; - - /** - * Used to parse the penultimate directory wildcard character example: /home/work/log/xapp/ * /server.log - * - * @param origPath - * @return - */ - public static List parseLevel5Directory(String origPath) { - ArrayList pathList = Lists.newArrayList(); - if (StringUtils.isEmpty(origPath)) { - return pathList; - } - String[] pathArray = origPath.split(","); - for (String path : pathArray) { - path = pathTrim(path); - String basePath = path.substring(0, path.lastIndexOf(SEPARATOR)); - String fileName = path.substring(path.lastIndexOf(SEPARATOR) + 1); - String[] fileArray = path.split(SEPARATOR); - if (fileArray.length < MINIMUM_LEVELS) { - pathList.add(path); - } else { - String fixedBasePath = basePath.substring(0, basePath.lastIndexOf(SEPARATOR)); - String regexPattern = basePath.substring(basePath.lastIndexOf(SEPARATOR) + 1); - if (StringUtils.startsWith(fixedBasePath, NEO_FILE_PREFIX)) { - regexPattern = MULTI_FILE_PREFIX + StringUtils.substringBetween(fixedBasePath, MULTI_FILE_PREFIX, MULTI_FILE_SUFFIX) + MULTI_FILE_SUFFIX; - fixedBasePath = NEO_FILE_PREFIX.substring(0, NEO_FILE_PREFIX.length() - 2); - } - // * Wildcard reads multi-level directories - if (regexPattern.trim().equals(PATH_WILDCARD)) { - try { - readFile(fixedBasePath, fileName, pathList); - } catch (FileNotFoundException e) { - log.error("[PathUtils.ParseLevel5Directory] file[{}] not found err:", basePath + fileName, e); - } catch (Exception e) { - log.error("[PathUtils.ParseLevel5Directory] path:[{}],err:", basePath + fileName, e); - } - } else if (basePath.contains(MULTI_FILE_PREFIX) && basePath.contains(MULTI_FILE_SUFFIX) && !fileName.contains(PATH_WILDCARD)) { - String multiDirectories = StringUtils.substringBetween(basePath, MULTI_FILE_PREFIX, MULTI_FILE_SUFFIX); - String directoryPrefix = StringUtils.substringBefore(basePath, MULTI_FILE_PREFIX); - String directorySuffix = StringUtils.substringAfter(basePath, MULTI_FILE_SUFFIX); - handleMultiDirectories(multiDirectories, fileName, directoryPrefix, directorySuffix, SEPARATOR, pathList); - } else if (fileName.contains(PATH_WILDCARD)) { - //Matches many files - handleMultipleDirectoryFile(basePath, fileName, pathList); - } else { - if (origPath.contains(PATH_WILDCARD)) { - basePath = StringUtils.substringBefore(origPath, PATH_WILDCARD); - String fileSuffix = StringUtils.substringAfter(origPath, PATH_WILDCARD); - try { - readFile(basePath, fileSuffix, pathList); - } catch (IOException e) { - e.printStackTrace(); - } - } else { - - } - for (String serverName : fileName.split(SPLIT_VERTICAL_LINE)) { - pathList.add(basePath + SEPARATOR + serverName); - } - } - } - } - return pathList; - } - - public static List buildMultipleDirectories(String multipleDire) { - List pathList = Lists.newArrayList(); - String[] multipleDts = multipleDire.split(","); - for (String multipleDt : multipleDts) { - if (multipleDt.contains(MULTI_FILE_PREFIX) && multipleDt.contains(MULTI_FILE_SUFFIX)) { - String[] directoryArray = multipleDt.split(SEPARATOR); - String directory = ""; - List directories = Lists.newArrayList(); - for (String perDire : directoryArray) { - if (perDire.startsWith(MULTI_FILE_PREFIX) && perDire.endsWith(MULTI_FILE_SUFFIX)) { - directory = perDire; - directories = Arrays.asList(StringUtils.substringBetween(perDire, MULTI_FILE_PREFIX, MULTI_FILE_SUFFIX).split(SPLIT_VERTICAL_LINE)); - } - } - if (StringUtils.isNotBlank(directory)) { - String prefix = StringUtils.substringBeforeLast(multipleDt, directory); - String suffix = StringUtils.substringAfter(multipleDt, directory); - for (String directoryTemp : directories) { - pathList.add(String.format("%s%s%s", prefix, directoryTemp, suffix)); - } - } - } else { - pathList.add(multipleDt); - } - } - return pathList; - } - - private static void handleMultipleDirectoryFile(String basePath, String fileNamePattern, List pathList) { - if (basePath.contains(MULTI_FILE_PREFIX) && basePath.contains(MULTI_FILE_SUFFIX)) { - String multiDirectories = StringUtils.substringBetween(basePath, MULTI_FILE_PREFIX, MULTI_FILE_SUFFIX); - String directoryPrefix = StringUtils.substringBefore(basePath, MULTI_FILE_PREFIX); - String directorySuffix = StringUtils.substringAfter(basePath, MULTI_FILE_SUFFIX); - for (String directory : multiDirectories.split(SPLIT_VERTICAL_LINE)) { - String basePathSingle = directoryPrefix + directory + directorySuffix; - pathList.addAll(findRulePatternFiles(basePathSingle, fileNamePattern)); - } - } else { - pathList.addAll(findRulePatternFiles(basePath, fileNamePattern)); - } - - } - - private static void handleMultiDirectories(String multiDirectories, String fileName, String directoryPrefix, - String directorySuffix, String separator, List pathList) { - for (String directory : multiDirectories.split(SPLIT_VERTICAL_LINE)) { - if (fileName.contains(MULTI_FILE_PREFIX) || fileName.contains(MULTI_FILE_SUFFIX)) { - for (String singleFileName : StringUtils.substringBetween(fileName, MULTI_FILE_PREFIX, MULTI_FILE_SUFFIX).split(SPLIT_VERTICAL_LINE)) { - pathList.add(directoryPrefix + directory + directorySuffix + separator + singleFileName); - } - } else { - for (String singleFileName : fileName.split(SPLIT_VERTICAL_LINE)) { - pathList.add(directoryPrefix + directory + directorySuffix + separator + singleFileName); - } - } - } - } - - /** - * Return to the list of monitoring folders - * - * @param origPath - * @return` - */ - public static List parseWatchDirectory(String origPath) { - List result = new ArrayList<>(); - if (StringUtils.isEmpty(origPath)) { - return result; - } - String[] pathArray = origPath.split(","); - for (String path : pathArray) { - path = pathTrim(path); - String basePath = path.substring(0, path.lastIndexOf(SEPARATOR)); - - String fixedBasePath = basePath.substring(0, basePath.lastIndexOf(SEPARATOR)); - String regexPattern = basePath.substring(basePath.lastIndexOf(SEPARATOR) + 1); - - if (StringUtils.startsWith(fixedBasePath, NEO_FILE_PREFIX)) { - regexPattern = MULTI_FILE_PREFIX + StringUtils.substringBetween(fixedBasePath, MULTI_FILE_PREFIX, MULTI_FILE_SUFFIX) + MULTI_FILE_SUFFIX; - fixedBasePath = NEO_FILE_PREFIX.substring(0, NEO_FILE_PREFIX.length() - 2); - } - - if (regexPattern.trim().equals(PATH_WILDCARD)) { - result.add(fixedBasePath); - } else if (regexPattern.startsWith(MULTI_FILE_PREFIX) && regexPattern.endsWith(MULTI_FILE_SUFFIX)) { - String patterns = regexPattern.substring(1, regexPattern.length() - 1); - String[] patternArr = patterns.split(SPLIT_VERTICAL_LINE); - String originFilePrefix = fixedBasePath + SEPARATOR + regexPattern; - //Adapt the remaining directories on the right side of the regular rule - // /home/work/logs/neo-logs/(xxx|yy)/zz/server.log => add /zz - String regexRightPath = path.substring(originFilePrefix.length() + 1); - String rightDir = ""; - if (regexRightPath.split(SEPARATOR).length > 1) { - rightDir = regexRightPath.substring(0, regexRightPath.lastIndexOf(SEPARATOR)); - rightDir = SEPARATOR + Arrays.stream(rightDir.split(SEPARATOR)).findFirst().get(); - } - for (String p : patternArr) { - String watchPath = fixedBasePath + SEPARATOR + p + rightDir; - result.add(watchPath); - } - } else if (basePath.contains(MULTI_FILE_PREFIX) && basePath.contains(MULTI_FILE_SUFFIX)) { - String multiDirectories = StringUtils.substringBetween(basePath, MULTI_FILE_PREFIX, MULTI_FILE_SUFFIX); - String directoryPrefix = StringUtils.substringBefore(basePath, MULTI_FILE_PREFIX); - String directorySuffix = StringUtils.substringAfter(basePath, MULTI_FILE_SUFFIX); - for (String directory : multiDirectories.split(SPLIT_VERTICAL_LINE)) { - result.add(directoryPrefix + directory + directorySuffix); - } - } else { - result.add(basePath); - } - } - return result.stream().distinct().collect(Collectors.toList()); - } - - private static String pathTrim(String path) { - path = path.replaceAll("//", "/"); - return path; - } - - private static void readFile(String filepath, String fileName, List list) throws FileNotFoundException, IOException { - try { - File file = new File(filepath); - if (!file.isDirectory()) { - return; - } else if (file.isDirectory()) { - String[] fileList = file.list(); - for (int i = 0; i < fileList.length; i++) { - String subPath; - if (filepath.endsWith("/")) { - subPath = filepath + fileList[i]; - } else { - subPath = filepath + "/" + fileList[i]; - } - - File subFile = new File(subPath); - if (!subFile.isDirectory() && StringUtils.equals(subFile.getName(), fileName)) { - list.add(subFile.getPath()); - } else if (subFile.isDirectory()) { - readFile(subPath, fileName, list); - } - } - - } - } catch (FileNotFoundException e) { - throw e; - } catch (IOException e) { - throw e; - } - return; - } - - private static void readFile(String filepath, String fileName, String dictionaries, List list) throws FileNotFoundException, IOException { - try { - File file = new File(filepath); - if (!file.isDirectory()) { - return; - } else if (file.isDirectory()) { - String[] fileList = file.list(); - for (int i = 0; i < fileList.length; i++) { - String subPath; - if (filepath.endsWith("/")) { - subPath = filepath + fileList[i]; - } else { - subPath = filepath + "/" + fileList[i]; - } - - File subFile = new File(subPath); - if (!subFile.isDirectory() && StringUtils.equals(subFile.getName(), fileName)) { - list.add(subFile.getPath()); - } else if (subFile.isDirectory()) { - readFile(subPath, fileName, list); - } - } - - } - } catch (FileNotFoundException e) { - throw e; - } catch (IOException e) { - throw e; - } - return; - } - - - public static List findRulePatternFiles(String directory, String patternStr) { - File file = new File(directory); - if (!patternStr.contains(SYMBOL_MULTI)) { - patternStr = patternStr.replaceAll("\\*", SYMBOL_MULTI); - } - if (patternStr.startsWith(PATH_WILDCARD)) { - patternStr = patternStr.replaceFirst("\\*", SYMBOL_MULTI); - } - Pattern compile = Pattern.compile(patternStr); - if (file.isDirectory()) { - return Arrays.stream(file.list()) - .filter(name -> compile.matcher(name).matches()) - .map(s -> String.format("%s%s%s", directory, SEPARATOR, s)) - .collect(Collectors.toList()); - } - return Collections.EMPTY_LIST; - } - - /** - * /home/work/log/log-agent/server.log.* - * /logSplitExpress:/home/work/log/log-agent/(server.log.*|error.log.*) - * /logSplitExpress:/home/work/log/(log-agent|log-stream)/(a|b)/server.log.* - * Clean the marked path - * - * @param originStr - * @return - */ - public static void dismantlingStrWithSymbol(String originStr, List cleanedPathList) { - if (StringUtils.isBlank(originStr)) { - return; - } - String pathPrefix = StringUtils.substringBefore(originStr, MULTI_FILE_PREFIX); - String betweenStr = StringUtils.substringBetween(originStr, MULTI_FILE_PREFIX, MULTI_FILE_SUFFIX); - if (StringUtils.isBlank(betweenStr)) { - cleanedPathList.add(originStr); - return; - } - String pathSuffix = StringUtils.substringAfter(originStr, MULTI_FILE_SUFFIX); - - if (StringUtils.isNotBlank(betweenStr)) { - String[] directories = StringUtils.split(betweenStr, SPLIT_VERTICAL_LINE); - for (String perDirectory : directories) { - String realPath = String.format("%s%s%s", pathPrefix, perDirectory, pathSuffix); - dismantlingStrWithSymbol(realPath, cleanedPathList); - } - } - } -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/Result.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/Result.java deleted file mode 100644 index 3fdf9ad5f..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/common/Result.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.common; - -import com.xiaomi.mone.log.exception.CommonError; -import com.xiaomi.mone.log.exception.CommonException; - -/** - * Created by zhangzhiyong on 29/05/2018. - * http json returns the result - */ -public class Result { - - private int code; - private String message; - private T data; - - public Result() { - } - - public Result(int code, String message) { - this.code = code; - this.message = message; - } - - public Result(int code, String message, T data) { - this.code = code; - this.message = message; - this.data = data; - } - - public static Result fail(CommonError error) { - return new Result<>(error.getCode(), error.getMessage()); - } - - public static Result failParam(String errorMsg) { - return new Result<>(CommonError.ParamsError.getCode(), errorMsg); - } - - public static Result fail(CommonException ex) { - return new Result<>(ex.getCode(), ex.getMessage()); - } - - public static Result success(T t) { - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), t); - } - - public static Result success() { - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage()); - } - - public static Result fail(Integer code, String msg) { - return new Result<>(code, msg); - } - - - public int getCode() { - return code; - } - - public void setCode(int code) { - this.code = code; - } - - public String getMessage() { - return message; - } - - public void setMessage(String message) { - this.message = message; - } - - public T getData() { - return data; - } - - public void setData(T data) { - this.data = data; - } - - @Override - public String toString() { - return "Result{" + - "code=" + code + - ", message='" + message + '\'' + - ", data=" + data + - '}'; - } -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/exception/CommonError.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/exception/CommonError.java deleted file mode 100644 index cbc3b80a2..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/exception/CommonError.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.exception; - -/** - * @author milog - */ - -public enum CommonError { - Success(0, "success"), - - UnknownError(1, "unknown error"), - - ParamsError(2, "parameter error"), - - NOT_EXISTS_DATA(3, "The data does not exist"), - - SERVER_ERROR(4, "Server exception"), - - UNAUTHORIZED(5, "No operation permissions"); - - private int code; - private String message; - - CommonError(int code, String message) { - this.code = code; - this.message = message; - } - - public int getCode() { - return code; - } - - public String getMessage() { - return message; - } -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/exception/CommonException.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/exception/CommonException.java deleted file mode 100644 index 1e8ce0147..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/exception/CommonException.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.exception; - -/** - * @author milog - */ -public class CommonException extends RuntimeException { - private int code; - private String message; - - public CommonException(int code, String message) { - this.code = code; - this.message = message; - } - - - public CommonException(CommonError error) { - this.code = error.getCode(); - this.message = error.getMessage(); - } - - public int getCode() { - return code; - } - - public void setCode(int code) { - this.code = code; - } - - @Override - public String getMessage() { - return message; - } - - public void setMessage(String message) { - this.message = message; - } -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/model/LogtailConfig.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/model/LogtailConfig.java deleted file mode 100644 index 7035f3a1f..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/model/LogtailConfig.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.model; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -/** - * Corresponds to the logtail configuration, and one logtail corresponds to an RMQ topic - */ -@Data -@EqualsAndHashCode -public class LogtailConfig { - private Long logtailId; - - private String ak; - private String sk; - private String clusterInfo; - private String consumerGroup; - private String topic; - private String tag; - private String type; - private Integer appType; - - private Integer parseType; - private String tail; - /** - * Log delimiter - */ - private String parseScript; - private String valueList; - -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/model/MiLogStreamConfig.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/model/MiLogStreamConfig.java deleted file mode 100644 index b0a6421a5..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/model/MiLogStreamConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.model; - -import lombok.Data; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -@Data -public class MiLogStreamConfig { - /** - * logSpace used to maintain logStream instance management - * key: logstream Instance IP - */ - private Map> config = new ConcurrentHashMap<>(); -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/model/MilogSpaceData.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/model/MilogSpaceData.java deleted file mode 100644 index a22b9e744..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/model/MilogSpaceData.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.model; - -import lombok.Data; - -import java.util.List; - -/** - * Encapsulation syncs to NACOS' milog space configuration - * key:spaceID - */ -@Data -public class MilogSpaceData { - private Long milogSpaceId; - private List spaceConfig; -} \ No newline at end of file diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/model/MilogStreamData.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/model/MilogStreamData.java deleted file mode 100644 index 496ace7c1..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/model/MilogStreamData.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.model; - -import lombok.Data; - -@Data -public class MilogStreamData { - private Long spaceId; - private String dataId; -} \ No newline at end of file diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/model/SinkConfig.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/model/SinkConfig.java deleted file mode 100644 index a0a31f9e4..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/model/SinkConfig.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.model; - -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.util.List; - -/** - * LogStore configuration, corresponding to an ES index - */ -@Data -@EqualsAndHashCode -public class SinkConfig { - private Long logstoreId; - private String logstoreName; - /** - * timestamp is required - */ - private String keyList; - /** - * key:logtailId - */ - @EqualsAndHashCode.Exclude - private List logtailConfigs; - - private String esIndex; - - private StorageInfo esInfo; - - private String storageType; - - /** - * When the type is doris, the order of schema - */ - private List columnList; - - public void updateStoreParam(SinkConfig sinkConfig) { - this.logstoreId = sinkConfig.getLogstoreId(); - this.logstoreName = sinkConfig.getLogstoreName(); - this.keyList = sinkConfig.getKeyList(); - this.esIndex = sinkConfig.getEsIndex(); - this.esInfo = sinkConfig.getEsInfo(); - } - -} \ No newline at end of file diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/model/StorageInfo.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/model/StorageInfo.java deleted file mode 100644 index 4cf4e9052..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/model/StorageInfo.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.model; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@AllArgsConstructor -@EqualsAndHashCode -public class StorageInfo { - private Long id; - /** - * ES address - */ - private String addr; - - /** - * ES user name - */ - private String user; - - /** - * ES password - */ - private String pwd; - - private String token; - - private String catalog; - - private String database; - - private Integer port; - - public StorageInfo(Long id, String addr, String user, String pwd) { - this.id = id; - this.addr = addr; - this.user = user; - this.pwd = pwd; - } - - public StorageInfo(Long id, String addr, String user, String pwd, Integer port) { - this.id = id; - this.addr = addr; - this.user = user; - this.pwd = pwd; - this.port = port; - } - - public StorageInfo(Long id, String addr, String token, String catalog, String database) { - this.id = id; - this.addr = addr; - this.token = token; - this.catalog = catalog; - this.database = database; - } -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/AbstractLogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/AbstractLogParser.java deleted file mode 100644 index b644638b7..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/AbstractLogParser.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.parse; - -import cn.hutool.core.util.ClassUtil; -import cn.hutool.core.util.ReflectUtil; -import com.google.common.collect.Lists; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/10/31 16:32 - */ -@Slf4j -public abstract class AbstractLogParser implements LogParser { - - protected LogParserData parserData; - - private List fieldInterceptors = Lists.newArrayList(); - - public AbstractLogParser(LogParserData parserData) { - this.parserData = parserData; - createFieldInterceptors(); - } - - private void createFieldInterceptors() { - try { - Set> implementationClasses = ClassUtil.scanPackageBySuper(PACKAGE_NAME, FieldInterceptor.class); - - fieldInterceptors = implementationClasses.stream() - .map(ReflectUtil::newInstance) - .filter(obj -> obj instanceof FieldInterceptor) - .map(obj -> (FieldInterceptor) obj) - .collect(Collectors.toList()); - } catch (Exception e) { - log.error("createFieldInterceptors", e); - } - } - - @Override - public Map parse(String logData, String ip, Long lineNum, Long collectStamp, String fileName) { - Map parseData = doParse(logData, ip, lineNum, collectStamp, fileName); - extractTimeStamp(parseData, logData, collectStamp); - wrapMap(parseData, parserData, ip, lineNum, fileName, collectStamp); - checkMessageExist(parseData, logData); - validRet(parseData, logData); - - fieldInterceptors.forEach(fieldInterceptor -> fieldInterceptor.postProcess(parseData)); - return parseData; - } - - @Override - public Map parseSimple(String logData, Long collectStamp) { - Map parseData = doParseSimple(logData, collectStamp); - fieldInterceptors.stream().forEach(fieldInterceptor -> fieldInterceptor.postProcess(parseData)); - return parseData; - } - - protected void validTimestamp(Map ret, Long collectStamp) { - /** - * If the user configures the parse timestamp field, the time format is checked to be correct, and the incorrect time is set to the current time - */ - if (ret.containsKey(esKeyMap_timestamp)) { - Long time = getTimestampFromString(ret.get(esKeyMap_timestamp).toString(), collectStamp); - ret.put(esKeyMap_timestamp, time); - } - } - - - /** - * Time extraction - */ - void extractTimeStamp(Map ret, String logData, Long collectStamp) { - /** - * The first [2022 XXXX] in extracted text, the first default is time processing - */ - if (!ret.containsKey(esKeyMap_timestamp) && logData.startsWith(LOG_PREFIX)) { - String timeStamp = StringUtils.substringBetween(logData, LOG_PREFIX, LOG_SUFFFIX); - Long time = getTimestampFromString(timeStamp, collectStamp); - ret.put(esKeyMap_timestamp, time); - } - /** - * Special handling, only dates starting with a date in the file such as yyyy-mm-dd HH:mm:ss will be extracted - */ - if (!ret.containsKey(esKeyMap_timestamp) && logData.startsWith(specialTimePrefix)) { - String timeStamp = StringUtils.substring(logData, 0, specialTimeLength); - Long time = getTimestampFromString(timeStamp, collectStamp); - ret.put(esKeyMap_timestamp, time); - } - } - - void wrapMap(Map ret, LogParserData parserData, String ip, - Long lineNum, String fileName, Long collectStamp) { - ret.putIfAbsent(esKeyMap_timestamp, null == collectStamp ? getTimestampFromString("", collectStamp) : collectStamp); - ret.put(esKeyMap_topic, parserData.getTopicName()); - ret.put(esKeyMap_tag, parserData.getMqTag()); - ret.put(esKeyMap_logstoreName, parserData.getLogStoreName()); - ret.put(esKeyMap_tail, parserData.getTailName()); - ret.put(esKeyMap_logip, ip); - ret.put(esKeyMap_lineNumber, lineNum); - ret.put(esKyeMap_fileName, fileName); - } - - void checkMessageExist(Map ret, String originData) { - if (!ret.containsKey(ES_KEY_MAP_MESSAGE)) { - ret.put(ES_KEY_MAP_MESSAGE, originData); - ret.remove(ES_KEY_MAP_LOG_SOURCE); - } - } - - /** - * Field configuration error check If the value corresponding to the key is empty in the result, - * indicating that the corresponding key has not been extracted, the complete log is retained - */ - void validRet(Map ret, String logData) { - if (ret.values().stream().filter(Objects::nonNull).map(String::valueOf).anyMatch(StringUtils::isEmpty)) { - ret.put(ES_KEY_MAP_LOG_SOURCE, logData); - } - } - - public abstract Map doParse(String logData, String ip, Long lineNum, Long collectStamp, String fileName); - - public abstract Map doParseSimple(String logData, Long collectStamp); -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/CustomLogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/CustomLogParser.java deleted file mode 100644 index af85743ef..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/CustomLogParser.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.parse; - -import cn.hutool.core.collection.CollectionUtil; -import cn.hutool.core.util.StrUtil; -import com.google.gson.Gson; -import com.xiaomi.mone.log.utils.IndexUtils; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.util.*; -import java.util.stream.Collectors; - -/** - * @Author: wtt - * @Date: 2021/12/28 21:59 - * @Description: Custom expression parsing - */ -@Data -@Slf4j -public class CustomLogParser extends AbstractLogParser { - - private boolean isParsePattern; - - private Map> mapPattern; - - private Gson gson = new Gson(); - - private String keyValueList; - - private List logPerComments; - - public CustomLogParser(LogParserData parserData) { - super(parserData); - } - - - @Override - public Map doParse(String logData, String ip, Long lineNum, Long collectStamp, String fileName) { - return doParseSimple(logData, collectStamp); - } - - @Override - public Map doParseSimple(String logData, Long collectStamp) { - Map ret = new HashMap<>(); - String originData = logData; - if (logData == null) { - return null; - } - try { - if (!isParsePattern) { - parsePatter(parserData.getParseScript()); - } - if (logData.length() == 0) { - return ret; - } - String originLog = logData; - if (StringUtils.isBlank(keyValueList) && CollectionUtil.isEmpty(logPerComments)) { - ret.put(ES_KEY_MAP_MESSAGE, logData); - return ret; - } - List logDataArray = parseLogData(logData); - for (int i = 0; i < logPerComments.size(); i++) { - if (i >= logDataArray.size()) { - ret.put(logPerComments.get(i), ""); - continue; - } - String value = logDataArray.get(i); - ret.put(logPerComments.get(i), StringUtils.isNotEmpty(value) ? value.trim() : value); - } - if (ret.values().stream().map(String::valueOf).anyMatch(StringUtils::isEmpty)) { - ret.put(ES_KEY_MAP_LOG_SOURCE, originLog); - } - /** - * Pocket does not include esKeyMap_timestamp, esKeyMap_topic, esKeyMap_tag, esKeyMap_logstoreName - */ - if (ret.containsKey(esKeyMap_timestamp)) { - Long time = getTimestampFromString(ret.get(esKeyMap_timestamp).toString(), collectStamp); - ret.put(esKeyMap_timestamp, time); - } - } catch (Exception e) { - ret.put(ES_KEY_MAP_LOG_SOURCE, originData); - } - return ret; - } - - /** - * An array of log contents parsed according to the parsing script - * - * @param logData - * @return - */ - @Override - public List parseLogData(String logData) throws Exception { - parsePatter(parserData.getParseScript()); - List parsedLogs = new ArrayList<>(); - for (int i = 0; i < mapPattern.size(); i++) { - String parsedData = ""; - List list = mapPattern.get(i); - if (StringUtils.isNotEmpty(list.get(0)) && StringUtils.isNotEmpty(list.get(1))) { - parsedData = StringUtils.substringBetween(logData, list.get(0), list.get(1)); - } else { - //Depend on the next index if the first part is not empty - if (i + 1 < mapPattern.size() && StringUtils.isNotEmpty(mapPattern.get(i + 1).get(0))) { - parsedData = StringUtils.substringBetween(logData, "", mapPattern.get(i + 1).get(0)); - } else { - parsedData = logData; - } - } - if (null == parsedData) { - break; - } - parsedLogs.add(parsedData.trim()); - logData = StrUtil.removePrefix(logData.trim(), String.format("%s%s%s", list.get(0), parsedData, list.get(1)).trim()); - } - return parsedLogs; - } - - public void parsePatter(String pattern) { - mapPattern = new HashMap<>(); - String[] split = StringUtils.split(pattern, "-"); - for (int i = 0; i < split.length; i++) { - String[] split1 = split[i].split("%s"); - if (split1.length == 2) { - mapPattern.put(i, Arrays.asList(split1[0], split1[1])); - } else { - mapPattern.put(i, Arrays.asList("", "")); - } - } - - keyValueList = IndexUtils.getKeyValueList(parserData.getKeyList(), parserData.getValueList()); - logPerComments = Arrays.stream(StringUtils.split(keyValueList, ",")).collect(Collectors.toList()); - - isParsePattern = true; - } -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/FieldInterceptor.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/FieldInterceptor.java deleted file mode 100644 index d64046349..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/FieldInterceptor.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.parse; - -import java.util.Map; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/11/17 11:17 - */ -public interface FieldInterceptor { - - void postProcess(Map data); -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/JsonLogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/JsonLogParser.java deleted file mode 100644 index b0bc53f14..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/JsonLogParser.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.parse; - -import cn.hutool.json.JSONUtil; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.ToNumberPolicy; -import com.google.gson.reflect.TypeToken; -import com.xiaomi.mone.log.utils.IndexUtils; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.util.*; - -/** - * @author liyandi - */ -@Slf4j -public class JsonLogParser extends AbstractLogParser { - // can solve the problem of converting long integers to scientific notation - private static final Gson GSON = new GsonBuilder() - .setObjectToNumberStrategy(ToNumberPolicy.LONG_OR_DOUBLE) - .setLenient() - .create(); - - private static final TypeToken> token = new TypeToken>() { - }; - - public JsonLogParser(LogParserData parserData) { - super(parserData); - } - - @Override - public Map doParse(String logData, String ip, Long lineNum, Long collectStamp, String fileName) { - return doParseSimple(logData, collectStamp); - } - - @Override - public Map doParseSimple(String logData, Long collectStamp) { - Map ret = new HashMap<>(); - if (logData == null || logData.length() == 0) { - return ret; - } - try { -// Map rawLogMap = GSON.fromJson(logData, token.getType()); - Map rawLogMap = flattenJson(logData); - // The complete set of index column names - List keyNameList = IndexUtils.getKeyListSlice(parserData.getKeyList()); - // An index subset that marks whether the index column name at the corresponding location is referenced in the current tail - int[] valueIndexList = Arrays.stream(parserData.getValueList().split(",")).mapToInt(Integer::parseInt).toArray(); - for (int i = 0; i < keyNameList.size(); i++) { - // Skip unreferenced keys - if (i >= valueIndexList.length || valueIndexList[i] == -1) { - continue; - } - String currentKey = keyNameList.get(i); - String value = rawLogMap.getOrDefault(currentKey, "").toString(); - ret.put(currentKey, StringUtils.isNotEmpty(value) ? value.trim() : value); - } - //timestamp - validTimestamp(ret, collectStamp); - } catch (Exception e) { - // If an exception occurs, the original log is kept to the logsource field - ret.put(ES_KEY_MAP_LOG_SOURCE, logData); - } - return ret; - } - - @Override - public List parseLogData(String logData) throws Exception { - Map rawLogMap = flattenJson(logData); - List parsedLogs = new ArrayList<>(); - for (String key : rawLogMap.keySet()) { - parsedLogs.add(rawLogMap.getOrDefault(key, "").toString()); - } - return parsedLogs; - } - - public Map flattenJson(String logData) { - Map ret = new HashMap<>(); - if (logData == null || logData.isEmpty()) { - return ret; - } - try { - TypeToken> token = new TypeToken<>() { - }; - Map rawLogMap = GSON.fromJson(logData, token); - flattenMap("", rawLogMap, ret); - } catch (Exception e) { - ret.put(ES_KEY_MAP_LOG_SOURCE, logData); - } - return ret; - } - - private void flattenMap(String prefix, Map source, Map target) { - for (Map.Entry entry : source.entrySet()) { - String key = prefix.isEmpty() ? entry.getKey() : prefix + "." + entry.getKey(); - Object value = entry.getValue(); - if (JSONUtil.isTypeJSON(entry.getValue().toString())) { - if (!(value instanceof Map)) { - value = GSON.fromJson(entry.getValue().toString(), token.getType()); - } - flattenMap(key, (Map) value, target); - } else { - target.put(key, value); - } - } - } -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/LogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/LogParser.java deleted file mode 100644 index 12696faa9..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/LogParser.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.parse; - -import cn.hutool.core.date.DateUtil; -import org.apache.commons.lang3.time.DateParser; -import org.apache.commons.lang3.time.FastDateFormat; - -import java.time.Instant; -import java.util.List; -import java.util.Map; - -/** - * @Author: wtt - * @Date: 2021/12/28 21:57 - * @Description: - */ -public interface LogParser { - - String LOG_PREFIX = "["; - String LOG_SUFFFIX = "]"; - Integer TIME_STAMP_MILLI_LENGTH = 13; - -// Integer MESSAGE_MAX_SIZE = 25000; - - DateParser dateFormat1 = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss"); - DateParser dateFormat2 = FastDateFormat.getInstance("yy-MM-dd HH:mm:ss"); - DateParser dateFormat3 = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss.SSS"); - DateParser dateFormat4 = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss,SSS"); - - Integer specialTimeLength = dateFormat1.getPattern().length(); - - String specialTimePrefix = "20"; - - String esKeyMap_timestamp = "timestamp"; - String esKeyMap_Date = "Date"; - String esKeyMap_topic = "mqtopic"; - String esKeyMap_tag = "mqtag"; - String esKeyMap_logstoreName = "logstore"; - String ES_KEY_MAP_LOG_SOURCE = "logsource"; - String ES_KEY_MAP_MESSAGE = "message"; - String esKeyMap_tail = "tail"; - String ES_KEY_MAP_TAIL_ID = "tailId"; - String esKeyMap_logip = "logip"; - String esKeyMap_lineNumber = "linenumber"; - String esKyeMap_fileName = "filename"; - String TRACE_ID_KEY = "traceId"; - String PACKAGE_NAME = "com.xiaomi.mone.log"; - - Map parse(String logData, String ip, Long lineNum, Long collectStamp, String fileName); - - Map parseSimple(String logData, Long collectStamp); - - List parseLogData(String logData) throws Exception; - - /** - * Compatible with 22-10-19 11:14:29 of this kind - * - * @param logTime - * @param collectStamp - * @return - */ - default Long getTimestampFromString(String logTime, Long collectStamp) { - Long timeStamp; - try { - timeStamp = DateUtil.parse(logTime).getTime(); - } catch (Exception e) { - try { - logTime = String.format("%s%s", String.valueOf(DateUtil.thisYear()).substring(0, 2), logTime); - timeStamp = DateUtil.parse(logTime).getTime(); - } catch (Exception ex) { - timeStamp = collectStamp; - } - } - return (null != timeStamp && timeStamp.toString().length() == TIME_STAMP_MILLI_LENGTH) ? timeStamp : Instant.now().toEpochMilli(); - } -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/LogParserData.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/LogParserData.java deleted file mode 100644 index 55952550a..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/LogParserData.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.parse; - -import lombok.*; - -/** - * @version 1.0 - * @Author wtt - * @description - * @date 2022/5/6 14:31 - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -@EqualsAndHashCode -@Builder -public class LogParserData { - private String keyList; - private String valueList; - private String parseScript; - private String topicName; - private String tailName; - private String mqTag; - private String logStoreName; -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/LogParserFactory.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/LogParserFactory.java deleted file mode 100644 index 5b28f60f4..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/LogParserFactory.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.parse; - -import lombok.Getter; - -import java.util.Arrays; -import java.util.Objects; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/1/5 10:27 - */ -public class LogParserFactory { - - private LogParserFactory() { - } - - public static LogParser getLogParser(Integer parseType, String keyList, String valueList, String parseScript) { - return LogParserFactory.getLogParser(parseType, keyList, valueList, parseScript, "", "", "", ""); - } - - public static LogParser getLogParser(Integer parseType, String keyList, String valueList, String parseScript, - String topicName, String tailName, String mqTag, String logStoreName) { - LogParserData logParserData = LogParserData.builder().keyList(keyList) - .valueList(valueList) - .parseScript(parseScript) - .topicName(topicName) - .tailName(tailName) - .mqTag(mqTag) - .logStoreName(logStoreName).build(); - LogParserEnum parserEnum = LogParserEnum.getByCode(parseType); - if (null == parserEnum) { - return new RawLogParser(logParserData); - } - switch (parserEnum) { - case SEPARATOR_PARSE: - return new SeparatorLogParser(logParserData); - case CUSTOM_PARSE: - return new CustomLogParser(logParserData); - case REGEX_PARSE: - return new RegexLogParser(logParserData); - case JSON_PARSE: - return new JsonLogParser(logParserData); - case NGINX_PARSE: - return new NginxLogParser(logParserData); - default: - return new RawLogParser(logParserData); - } - } - - @Getter - public enum LogParserEnum { - - RAW_LOG_PARSE(1, "原始格式"), - SEPARATOR_PARSE(2, "分割符解析"), - CUSTOM_PARSE(5, "自定义脚本解析"), - REGEX_PARSE(6, "正则表达式"), - JSON_PARSE(7, "JSON解析"), - NGINX_PARSE(8, "Nginx解析"); - - private Integer code; - private String name; - - LogParserEnum(Integer code, String name) { - this.code = code; - this.name = name; - } - - public static LogParserEnum getByCode(Integer code) { - return Arrays.stream(LogParserEnum.values()).filter(logParserEnum -> { - if (Objects.equals(logParserEnum.code, code)) { - return true; - } - return false; - }).findFirst().orElse(null); - } - } -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/NginxLogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/NginxLogParser.java deleted file mode 100644 index dbdfba055..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/NginxLogParser.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.parse; - -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * @author zhangjuan - * @version 1.0 - * @description - */ -@Slf4j -public class NginxLogParser extends AbstractLogParser { - - private RegexLogParser regexLogParser; - - public NginxLogParser(LogParserData parserData) { - super(parserData); - String regexParseScript = generateRegexFromNginxScript(parserData.getParseScript()); - parserData.setParseScript(regexParseScript); - this.regexLogParser = new RegexLogParser(parserData); - } - - @Override - public Map doParse(String logData, String ip, Long lineNum, Long collectStamp, String fileName) { - return this.regexLogParser.parse(logData, ip, lineNum, collectStamp, fileName); - } - - @Override - public Map doParseSimple(String logData, Long collectStamp) { - return this.regexLogParser.doParseSimple(logData, collectStamp); - } - - public String generateRegexFromNginxScript(String nginxFormatStr) { - Pattern regexSpace = Pattern.compile("\\s"); - Pattern regexSpaceChar = Pattern.compile("\\\\[tnvfr]"); - Pattern regexSpacePlus = Pattern.compile("\\s+"); - Pattern regexVar = Pattern.compile("\\$\\{?[a-zA-Z0-9_]+\\}?"); - // Matches columns that may contain spaces (that is, columns that need to be enclosed in quotation marks in the nginx log format) - Pattern regexSVar = Pattern.compile("([\\[\\{\\(\"])(\\$\\{?[a-zA-Z0-9_]+\\}?)([\\]\\}\\)\"])"); - Pattern regexNginxConfBodyGroup = Pattern.compile("'(.*?)'"); - - // Placeholder for columns that do not contain spaces - String varPlaceHolder = "__VAR_PLACE_HOLDER__"; - // Placeholders for columns that may contain spaces - String svarPlaceholder = "__SVAR_PLACE_HOLDER__"; - - String bodyStr = ""; - Matcher bodyMatcher = regexNginxConfBodyGroup.matcher(nginxFormatStr); - while (bodyMatcher.find()) { - for (int i = 1; i <= bodyMatcher.groupCount(); i++) { - bodyStr += bodyMatcher.group(i); - } - } - - String valueRegex = ""; - // 1. Replaces all spaces - valueRegex = regexSpace.matcher(bodyStr).replaceAll(" "); - // 2. Replaces all tabs that represent spaces - valueRegex = regexSpaceChar.matcher(valueRegex).replaceAll(" "); - // 3. Replace columns that contain spaces with placeholders - valueRegex = regexSVar.matcher(valueRegex).replaceAll("$1" + svarPlaceholder + "$3"); - // 4. Replace all variables with placeholders - valueRegex = regexVar.matcher(valueRegex).replaceAll(varPlaceHolder); - valueRegex = escapeExprSpecialWord(valueRegex); - // 5. Replace all spaces with regular expression syntax - valueRegex = regexSpacePlus.matcher(valueRegex).replaceAll("\\\\s+"); - // 6. Replace the placeholders with the corresponding regular expression groupings - valueRegex = valueRegex.replaceAll(svarPlaceholder, "(.*?)"); - valueRegex = valueRegex.replaceAll(varPlaceHolder, "(\\\\S*)"); - // 7. Matches the extra line breaks at the end - valueRegex += "\\s*$"; - return valueRegex; - } - - /** - * Escape special characters - * - * @param keyword - * @return - */ - public static String escapeExprSpecialWord(String keyword) { - if (StringUtils.isNotBlank(keyword)) { - String[] fbsArr = {"\\", "$", "(", ")", "*", "+", ".", "[", "]", "?", "^", "{", "}", "|"}; - for (String key : fbsArr) { - if (keyword.contains(key)) { - keyword = keyword.replace(key, "\\" + key); - } - } - } - return keyword; - } - - @Override - public List parseLogData(String logData) throws Exception { - return this.regexLogParser.parseLogData(logData); - } -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/RawLogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/RawLogParser.java deleted file mode 100644 index 863fab8fa..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/RawLogParser.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.parse; - -import com.google.common.collect.Lists; -import lombok.extern.slf4j.Slf4j; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @author wtt - * @version 1.0 - * @description original log format - * @date 2023/9/15 16:28 - */ -@Slf4j -public class RawLogParser extends AbstractLogParser { - - public RawLogParser(LogParserData parserData) { - super(parserData); - } - - @Override - public Map doParse(String logData, String ip, Long lineNum, Long collectStamp, String fileName) { - Map ret = doParseSimple(logData, collectStamp); - checkMessageExist(ret, logData); - return ret; - } - - @Override - public Map doParseSimple(String logData, Long collectStamp) { - Map ret = new HashMap<>(); - ret.put(ES_KEY_MAP_MESSAGE, logData); - if (null != collectStamp) { - ret.put(esKeyMap_timestamp, collectStamp); - } - return ret; - } - - @Override - public List parseLogData(String logData) { - return Lists.newArrayList(logData); - } -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/RegexLogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/RegexLogParser.java deleted file mode 100644 index ffb707775..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/RegexLogParser.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.parse; - -import com.xiaomi.mone.log.utils.IndexUtils; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * @author zhangjuan - * @version 1.0 - * @description - */ -@Slf4j -public class RegexLogParser extends AbstractLogParser { - - private Pattern pattern; - - public RegexLogParser(LogParserData parserData) { - super(parserData); - pattern = Pattern.compile(parserData.getParseScript(), Pattern.MULTILINE); - } - - @Override - public Map doParse(String logData, String ip, Long lineNum, Long collectStamp, String fileName) { - return doParseSimple(logData, collectStamp); - } - - @Override - public Map doParseSimple(String logData, Long collectStamp) { - Map ret = new HashMap<>(); - if (logData == null || logData.length() == 0) { - return ret; - } - try { - // A list of extracted contents by regular regex - List logArray = parseLogData(logData); - // Index the list of column names - List keyNameList = IndexUtils.getKeyListSlice(parserData.getKeyList()); - // Each index column name corresponds to an array of index values for the content in the regular extracted content list - int[] valueIndexList = Arrays.stream(parserData.getValueList().split(",")).mapToInt(Integer::parseInt).toArray(); - for (int i = 0; i < keyNameList.size(); i++) { - // If the index of the key is outside the range of value, or the index corresponding to value is -1, the current key is skipped - if (i >= valueIndexList.length || valueIndexList[i] == -1) { - continue; - } - // If the index of value does not exceed the regular parsed content array, the key has a corresponding resolution value, otherwise it is "" - String value = ""; - if (valueIndexList[i] < logArray.size()) { - value = logArray.get(valueIndexList[i]); - } - ret.put(keyNameList.get(i), StringUtils.isNotEmpty(value) ? value.trim() : value); - } - validTimestamp(ret, collectStamp); - } catch (Exception e) { - // If an exception occurs, the original log is kept to the logsource field - ret.put(ES_KEY_MAP_LOG_SOURCE, logData); - } - return ret; - } - - @Override - public List parseLogData(String logData) throws Exception { - List ret = new ArrayList<>(); - if (pattern == null) { - throw new Exception("compile failed, empty pattern"); - } -// List matchers = filter.filter(logData); - Matcher matcher = pattern.matcher(logData); - if (matcher.find()) { - // matcher.groupCount() gets how many capture groups the matcher object currently has, excluding group(0), so groupCount does not correspond to the group(i) index - // group(0) does not support user acquisition, and the valueList order can still start from 0 when the user uses it - for (int i = 1; i <= matcher.groupCount(); i++) { - ret.add(matcher.group(i)); - } - } - return ret; - } -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/SeparatorLogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/SeparatorLogParser.java deleted file mode 100644 index a4c25cae3..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/SeparatorLogParser.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.parse; - -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - -/** - * @author wtt - * @version 1.0 - * @description - */ -@Slf4j -public class SeparatorLogParser extends AbstractLogParser { - private String[] keysAndTypes; - private String[] values; - - public SeparatorLogParser(LogParserData parserData) { - super(parserData); - keysAndTypes = StringUtils.split(parserData.getKeyList(), ","); - values = StringUtils.split(parserData.getValueList(), ","); - } - - @Override - public Map doParse(String logData, String ip, Long lineNum, Long collectStamp, String fileName) { - return doParseSimple(logData, collectStamp); - } - - @Override - public Map doParseSimple(String logData, Long collectStamp) { - Map ret = new HashMap<>(); - if (logData == null) { - return ret; - } - if (logData.isEmpty()) { - return ret; - } - try { - - int maxLength = (int) Arrays.stream(values).filter(s -> !s.equals("-1")).count(); - - List logArray = parseLogData(logData, maxLength); - if (0 == maxLength) { - ret.put(ES_KEY_MAP_MESSAGE, logData); - return ret; - } - if (values.length == 1 && logArray.size() == 1 && maxLength == 1) { - String[] ktSplit = keysAndTypes[0].split(":"); - String keysAndType = ktSplit[0]; - ret.put(keysAndType, logArray.get(0)); - return ret; - } - - int count = 0; - int valueCount = 0; - /** - * Normal parsing - */ - for (int i = 0; i < keysAndTypes.length; i++) { - String[] kTsplit = keysAndTypes[i].split(":"); - if (kTsplit.length != 2 || i >= values.length) { - continue; - } - if (kTsplit[0].equals(esKeyMap_topic)) { - count++; - ret.put(esKeyMap_topic, parserData.getTopicName()); - continue; - } else if (kTsplit[0].equals(esKeyMap_tag)) { - count++; - ret.put(esKeyMap_tag, parserData.getMqTag()); - continue; - } else if (kTsplit[0].equals(esKeyMap_logstoreName)) { - count++; - ret.put(esKeyMap_logstoreName, parserData.getLogStoreName()); - continue; - } else if (kTsplit[0].equals(esKeyMap_tail)) { - count++; - ret.put(esKeyMap_tail, parserData.getTailName()); - continue; - } else if (kTsplit[0].equals(ES_KEY_MAP_LOG_SOURCE)) { - count++; - continue; - } - String value = null; - int num = -1; - try { - num = Integer.parseInt(values[i]); - if (num == -1) { - valueCount++; - continue; - } - } catch (Exception e) { - continue; - } - if (num < logArray.size() && num > -1) { - value = logArray.get(num); - } else { - value = ""; - } - if (kTsplit[0].equals(esKeyMap_timestamp) || kTsplit[1].equalsIgnoreCase(esKeyMap_Date)) { - Long time = getTimestampFromString(value, collectStamp); - ret.put(esKeyMap_timestamp, time); - } else { - ret.put(kTsplit[0], StringUtils.isNotEmpty(value) ? value.trim() : value); - } - } - - /** - * The field is misconfigured - * esKeyMap_topic,esKeyMap_tag,esKeyMap_logstoreName,esKeyMap_logSource are not visible to the user, i.e. do not exist in values, logArray - */ - if (ret.values().stream().filter(Objects::nonNull).map(String::valueOf).anyMatch(StringUtils::isEmpty)) { - ret.put(ES_KEY_MAP_LOG_SOURCE, logData); - } - } catch (Exception e) { - ret.put(ES_KEY_MAP_LOG_SOURCE, logData); - } - return ret; - } - - @Override - public List parseLogData(String logData) { - return parseLogData(logData, -1); - } - - private List parseLogData(String logData, Integer maxLength) { - String[] logArray = StringUtils.splitByWholeSeparatorPreserveAllTokens(logData, parserData.getParseScript(), maxLength); - return Arrays.stream(logArray).collect(Collectors.toList()); - } - -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/utils/ConfigUtils.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/utils/ConfigUtils.java deleted file mode 100644 index e69dd0949..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/utils/ConfigUtils.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.utils; - -import cn.hutool.core.util.HashUtil; -import com.xiaomi.mone.log.common.Config; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -/** - * @author: wtt - * @date: 2022/5/19 12:25 - * @description: - */ -@Slf4j -public class ConfigUtils { - - private ConfigUtils() { - - } - - public static String getConfigValue(String propertyKey) { - String propertyValue = ""; - propertyValue = System.getenv(propertyKey); - try { - if(StringUtils.isBlank(propertyValue)) { - propertyValue = System.getProperty(propertyKey); - } - } catch (Exception e) { - log.error("get system param error,propertyKey:{}", propertyKey, e); - } - if (StringUtils.isBlank(propertyValue)) { - propertyValue = Config.ins().get(propertyKey, ""); - } - return propertyValue; - } - - /** - * The data data maps to a value between 0 and max - * - * @param data - * @param max - * @return - */ - public static int getDataHashKey(String data, int max) { - return Math.abs(HashUtil.apHash(data)) % max + 1; - } -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/utils/DateUtils.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/utils/DateUtils.java deleted file mode 100644 index 496fb50fd..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/utils/DateUtils.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.utils; - -import org.apache.commons.lang3.StringUtils; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.Calendar; -import java.util.Date; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/29 10:30 - */ -public class DateUtils { - - public static long dayms = 86400000L; - - public static String getTime() { - DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy.MM.dd"); - return dateTimeFormatter.format(LocalDateTime.now()); - } - - public static String getTime(int plusDays) { - LocalDate today = LocalDate.now(); - - LocalDate tomorrow = today.plusDays(1); - - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy.MM.dd"); - - return tomorrow.format(formatter); - } - - /** - * Get the first millisecond today - * - * @return - */ - public static long getTodayFirstMillisecond() { - Date date = new Date(); - Calendar calendar = Calendar.getInstance(); - calendar.setTime(date); - date = calendar.getTime(); - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); - try { - date = sdf.parse(sdf.format(date)); - } catch (ParseException e) { - e.printStackTrace(); - } - return date.getTime(); - } - - /** - * Get yesterday's first millisecond - * - * @return] - */ - public static long getYesterdayFirstMillisecond() { - return getTodayFirstMillisecond() - dayms; - } - - /** - * Get the first millisecond of the day before yesterday - * - * @return] - */ - public static long getBeforeYesterdayFirstMillisecond() { - return getTodayFirstMillisecond() - dayms * 2; - } - - /** - * Gets the first millisecond of the day - * - * @param thisDay - * @return - */ - public static Long getThisDayFirstMillisecond(String thisDay) { - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); - Date date = null; - try { - date = format.parse(thisDay); - } catch (ParseException e) { - e.printStackTrace(); - } - return date.getTime(); - } - - /** - * Get n days ago date (yyyy-MM-dd) - * - * @param n - * @return - */ - public static String getDaysAgo(int n) { - return new SimpleDateFormat("yyyy-MM-dd").format(System.currentTimeMillis() - n * dayms); - } - - public static String timeStamp2Date(String millSeconds, String format) { - if (StringUtils.isBlank(millSeconds)) { - return ""; - } - if (format == null || format.isEmpty()) format = "yyyy-MM-dd HH:mm:ss"; - SimpleDateFormat sdf = new SimpleDateFormat(format); - return sdf.format(new Date(Long.valueOf(millSeconds))); - } - -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/utils/IndexUtils.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/utils/IndexUtils.java deleted file mode 100644 index 62dc6a2a8..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/utils/IndexUtils.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.utils; - -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/12/29 12:40 - */ -public class IndexUtils { - - private IndexUtils() { - } - - public static String getKeyValueList(String keyList, String valueList) { - List keyListSlice = getKeyListSlice(keyList); - String[] valueS = StringUtils.split(valueList, ","); - Map map = new HashMap<>(); - for (int i = 0; i < valueS.length; i++) { - int orderValue = Integer.parseInt(valueS[i]); - if (orderValue >= 0) { - map.put(orderValue, keyListSlice.get(i)); - } - } - return map.entrySet().stream() - .sorted(Map.Entry.comparingByKey()) - .map(Map.Entry::getValue).collect(Collectors.joining(",")); - } - - public static List getKeyListSlice(String keyList) { - String[] KLSplit = keyList.split(","); - List ret = new ArrayList(); - for (int i = 0; i < KLSplit.length; i++) { - String[] split = KLSplit[i].split(":"); - if (split.length > 1 && !split[1].equals("3")) { - ret.add(split[0]); - } - } - return ret; - } - - public static String getNumberValueList(String keyList, String valueList) { - List keyListSlice = getKeyListSlice(keyList); - String[] values = valueList.split(","); - HashMap valueListMap = new HashMap<>(); - for (int i = 0; i < values.length; i++) { - valueListMap.put(values[i], i); - } - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < keyListSlice.size(); i++) { - if (valueListMap.get(keyListSlice.get(i)) != null) { - Integer val = valueListMap.get(keyListSlice.get(i)); - sb.append(val).append(","); - } else { - sb.append(-1).append(","); - } - } - if (!StringUtils.isEmpty(sb.toString())) { - sb.deleteCharAt(sb.lastIndexOf(",")); - } - return sb.toString(); - } -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/utils/KafkaUtils.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/utils/KafkaUtils.java deleted file mode 100644 index f433875ea..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/utils/KafkaUtils.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.utils; - -import org.apache.commons.lang3.StringUtils; -import org.apache.kafka.clients.CommonClientConfigs; -import org.apache.kafka.clients.admin.AdminClientConfig; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.common.config.SaslConfigs; -import org.apache.kafka.common.config.SslConfigs; - -import java.util.Properties; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/11/30 14:07 - */ -public class KafkaUtils { - - private KafkaUtils() { - } - - public static Properties getVpc9094KafkaProperties(String nameServer, String userName, String password) { - String saslMechanism = "PLAIN"; - // Set Kafka server address - Properties props = new Properties(); - props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, nameServer); - //access protocol， - props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT"); - // Set up SASL account - if (StringUtils.isNotEmpty(userName) && StringUtils.isNotEmpty(password)) { - String prefix = "org.apache.kafka.common.security.scram.ScramLoginModule"; - if ("PLAIN".equalsIgnoreCase(saslMechanism)) { - prefix = "org.apache.kafka.common.security.plain.PlainLoginModule"; - } - String jaasConfig = String.format("%s required username=\"%s\" password=\"%s\";", prefix, userName, password); - props.put(SaslConfigs.SASL_JAAS_CONFIG, jaasConfig); - } - // The difference between scram mode and plain mode - props.put(SaslConfigs.SASL_MECHANISM, saslMechanism); - return props; - } - - public static Properties getDefaultKafkaProperties(String nameServer) { - // Set Kafka server address - Properties props = new Properties(); - props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, nameServer); - //Maximum wait time for requests - props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 30 * 1000); - //Set the number of client internal retries - props.put(ProducerConfig.RETRIES_CONFIG, 5); - //Set client internal retry interval - props.put(ProducerConfig.RECONNECT_BACKOFF_MS_CONFIG, 3000); - return props; - } - - public static Properties getSslKafkaProperties(String nameServer, String userName, String password, String sslLocation) { - String saslMechanism = "PLAIN"; - // Set Kafka server address - Properties props = new Properties(); - props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, nameServer); - - props.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, sslLocation); - //The password of the root certificate store, remains unchanged - props.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, "KafkaOnsClient"); - //Access protocol, currently supports access using SASL SSL protocol - props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_SSL"); - // Set up SASL account - if (!StringUtils.isEmpty(userName) && !StringUtils.isEmpty(password)) { - String prefix = "org.apache.kafka.common.security.scram.ScramLoginModule"; - if ("PLAIN".equalsIgnoreCase(saslMechanism)) { - prefix = "org.apache.kafka.common.security.plain.PlainLoginModule"; - } - String jaasConfig = String.format("%s required username=\"%s\" password=\"%s\";", prefix, userName, password); - props.put(SaslConfigs.SASL_JAAS_CONFIG, jaasConfig); - } - props.put(SaslConfigs.SASL_MECHANISM, saslMechanism); - props.put(SslConfigs.SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG, ""); - return props; - } -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/utils/NetUtil.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/utils/NetUtil.java deleted file mode 100644 index c7aa0d6b6..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/utils/NetUtil.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.utils; - -import com.xiaomi.mone.log.common.NetUtils; -import org.apache.commons.lang3.StringUtils; - -/** - * @author shanwb - * @date 2021-08-16 - */ -public class NetUtil { - - private static final String HERA_K8S_ENV = "hera_buildin_k8s"; - private static final String HERA_IP_ENV = "host_ip"; - - public static String getLocalIp() { - if (StringUtils.isNotEmpty(System.getenv(HERA_IP_ENV))) { - return System.getenv(HERA_IP_ENV); - } - String localIp = System.getenv("host.ip") == null ? NetUtils.getLocalHost() : System.getenv("host.ip"); - return localIp; - } - - public static String getHeraK8sEnv() { - String envStatus = null == System.getenv(HERA_K8S_ENV) ? System.getProperty(HERA_K8S_ENV) : System.getenv(HERA_K8S_ENV); - if (StringUtils.isEmpty(envStatus)) { - return System.getenv(HERA_K8S_ENV.toUpperCase()); - } - return envStatus; - } - -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/utils/PinYin4jUtils.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/utils/PinYin4jUtils.java deleted file mode 100644 index 9081e3e3e..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/utils/PinYin4jUtils.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.utils; - -import net.sourceforge.pinyin4j.PinyinHelper; -import net.sourceforge.pinyin4j.format.HanyuPinyinCaseType; -import net.sourceforge.pinyin4j.format.HanyuPinyinOutputFormat; -import net.sourceforge.pinyin4j.format.HanyuPinyinToneType; -import net.sourceforge.pinyin4j.format.HanyuPinyinVCharType; -import net.sourceforge.pinyin4j.format.exception.BadHanyuPinyinOutputFormatCombination; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/11/26 15:18 - */ -public class PinYin4jUtils { - - public static String getAllPinyin(String hanzi) { - //Output formatting - HanyuPinyinOutputFormat format = new HanyuPinyinOutputFormat(); - /** - * Output case settings - * - * LOWERCASE:Output lowercase - * UPPERCASE:Output uppercase - */ - format.setCaseType(HanyuPinyinCaseType.LOWERCASE); - - /** - * Output phonetic transcription settings - * - * WITH_TONE_MARK:Directly with phonetic symbols (WITH U_UNICODE must be set, otherwise an exception will be thrown) - * WITH_TONE_NUMBER：Numbers 1-4 indicate phonetic transcription - * WITHOUT_TONE：There is no phonetic transcription - */ - format.setToneType(HanyuPinyinToneType.WITHOUT_TONE); - - /** - * Special phonetic transcription U setting - * - * WITH_V：Use v to indicate ü - * WITH_U_AND_COLON：Use "u:" for ü - * WITH_U_UNICODE：Use ü directly - */ - format.setVCharType(HanyuPinyinVCharType.WITH_U_UNICODE); - - char[] hanYuArr = hanzi.trim().toCharArray(); - StringBuilder pinYin = new StringBuilder(); - - try { - for (int i = 0, len = hanYuArr.length; i < len; i++) { - //Whether the match is a Chinese character or not - if (Character.toString(hanYuArr[i]).matches("[\\u4E00-\\u9FA5]+")) { - //If it is a polyphonetic word, return multiple pinyin, here only the first one is taken - String[] pys = PinyinHelper.toHanyuPinyinStringArray(hanYuArr[i], format); - pinYin.append(pys[0]).append(""); - } else { - pinYin.append(hanYuArr[i]).append(""); - } - } - } catch (BadHanyuPinyinOutputFormatCombination badHanyuPinyinOutputFormatCombination) { - badHanyuPinyinOutputFormatCombination.printStackTrace(); - } - return pinYin.toString(); - } -} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/utils/SimilarUtils.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/utils/SimilarUtils.java deleted file mode 100644 index 936fcf4bf..000000000 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/utils/SimilarUtils.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.utils; - -import org.apache.commons.lang3.StringUtils; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/2/16 13:03 - */ -public class SimilarUtils { - - public SimilarUtils() { - } - - public static String findHighestSimilarityStr(String baseStr, List strList) { - String strR = ""; - if (1 == strList.size()) { - strR = strList.get(0); - } else { - for (String s : strList) { - if (baseStr.contains(s)) { - strR = s; - } - } - } - if (StringUtils.isEmpty(strR)) { - // find similar fileName - strR = strList.stream().sorted((o1, o2) -> - Double.compare(computeSimilarity(baseStr, o2), computeSimilarity(baseStr, o1))) - .findFirst() - .get(); - } - return strR; - } - - public static double computeSimilarity(String str1, String str2) { - int levenshteinDistance = computeLevenshteinDistance(str1, str2); - int maxLen = Math.max(str1.length(), str2.length()); - return (1 - ((double) levenshteinDistance / maxLen)) * 100; - } - - private static int computeLevenshteinDistance(String str1, String str2) { - int[][] distance = new int[str1.length() + 1][str2.length() + 1]; - - for (int i = 0; i <= str1.length(); i++) - distance[i][0] = i; - for (int j = 1; j <= str2.length(); j++) - distance[0][j] = j; - - for (int i = 1; i <= str1.length(); i++) - for (int j = 1; j <= str2.length(); j++) - distance[i][j] = minimum( - distance[i - 1][j] + 1, - distance[i][j - 1] + 1, - distance[i - 1][j - 1] + ((str1.charAt(i - 1) == str2.charAt(j - 1)) ? 0 : 1)); - - return distance[str1.length()][str2.length()]; - } - - private static int minimum(int a, int b, int c) { - return Math.min(Math.min(a, b), c); - } - -} diff --git a/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/DateUtilsTest.java b/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/DateUtilsTest.java deleted file mode 100644 index 7835a43db..000000000 --- a/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/DateUtilsTest.java +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.common; \ No newline at end of file diff --git a/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/IndexUtilsTest.java b/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/IndexUtilsTest.java deleted file mode 100644 index 1a5abf9f6..000000000 --- a/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/IndexUtilsTest.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.common; - -import com.xiaomi.mone.log.utils.IndexUtils; -import lombok.extern.slf4j.Slf4j; -import org.junit.Test; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/12/29 12:42 - */ -@Slf4j -public class IndexUtilsTest { - - @Test - public void testGetKeyValueList(){ - String keyList = "timestamp:1,level:1,traceId:1,threadName:1,className:1,message:1,line:1,methodName:1,logstore:3,logsource:3,mqtopic:3,mqtag:3,logip:3,tail:3"; - String valueList = "0,2,1,6,3,4,5,-1"; - String keyValueList = IndexUtils.getKeyValueList(keyList, valueList); - log.info(keyValueList); - } -} diff --git a/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/LineMessage.java b/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/LineMessage.java deleted file mode 100644 index c50587ad3..000000000 --- a/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/LineMessage.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.common; - -import lombok.ToString; -import org.apache.commons.lang3.StringUtils; - -import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; - -/** - * @author shanwb - * @date 2021-07-19 - */ -@ToString -public class LineMessage implements Serializable { - - public static transient String KEY_IP = "ip"; - public static transient String KEY_COLLECT_TIMESTAMP = "ct"; - public static transient String KEY_MQ_TOPIC = "t"; - public static transient String KEY_MQ_TOPIC_TAG = "tag"; - - private Long lineNumber; - - private String fileName; - - private Long pointer; - - private Integer msgLength; - - private String msgBody; - - private Map extMap; - - public LineMessage() { - } - - public long getTimestamp() { - String value = extMap.get(KEY_COLLECT_TIMESTAMP); - if (StringUtils.isEmpty(value)) { - return 0; - } else { - return Long.parseLong(value); - } - } - - public void setTimeStamp(long time) { - extMap.put(KEY_COLLECT_TIMESTAMP, String.valueOf(time)); - } - - public Long getLineNumber() { - return lineNumber; - } - - public void setLineNumber(Long lineNumber) { - this.lineNumber = lineNumber; - } - - public Long getPointer() { - return pointer; - } - - public void setPointer(Long pointer) { - this.pointer = pointer; - } - - public Integer getMsgLength() { - return msgLength; - } - - public void setMsgLength(Integer msgLength) { - this.msgLength = msgLength; - } - - public void setMsgBody(String msgBody) { - this.msgBody = msgBody; - } - - public String getMsgBody() { - return msgBody; - } - - public Map getExtMap() { - return extMap; - } - - public void setExtMap(Map extMap) { - this.extMap = extMap; - } - - public synchronized void setProperties(String key, String value) { - if (null == extMap) { - extMap = new HashMap<>(); - } - extMap.put(key, value); - } - - public String getProperties(String key) { - return this.getProperties(key, null); - } - - public String getProperties(String key, String defaultValue) { - if (null == this.extMap) { - return defaultValue; - } - - if (StringUtils.isBlank(key)) { - return defaultValue; - } - - String value = this.extMap.get(key); - if (StringUtils.isBlank(value)) { - return defaultValue; - } - - return value; - } - - public String getFileName() { - return fileName; - } - - public void setFileName(String fileName) { - this.fileName = fileName; - } -} diff --git a/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/LogParserFactoryTest.java b/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/LogParserFactoryTest.java deleted file mode 100644 index 0b1d1a6ad..000000000 --- a/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/LogParserFactoryTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.common; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.ToNumberPolicy; -import com.google.gson.reflect.TypeToken; -import com.xiaomi.mone.log.parse.LogParser; -import com.xiaomi.mone.log.parse.LogParserFactory; -import lombok.extern.slf4j.Slf4j; -import org.junit.Test; - -import java.time.Instant; -import java.util.Map; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/1/5 10:33 - */ -@Slf4j -public class LogParserFactoryTest { - - private Gson gson = new Gson(); - - @Test - public void test() { - Integer parseType = 5; - String keyList = "timestamp:date,level:keyword,traceId:keyword,threadName:text,className:text,line:keyword,methodName:keyword,message:text,podName:keyword,logstore:keyword,logsource:keyword,mqtopic:keyword,mqtag:keyword,logip:keyword,tail:keyword,linenumber:long\","; - String valueList = "0,1,5,4,2,-1,3,6,-1"; - String parseScript = "%s-[%s]-%s-(%s)-[%s]-[%s]-%s"; - String topicName = "3424"; - String tailName = "test name"; - String mqTag = "fsfsd"; - String logStoreName = "testet"; - String message = ""; - LogParser logParser = LogParserFactory.getLogParser(parseType, keyList, valueList, parseScript, topicName, tailName, mqTag, logStoreName); - - LineMessage lineMessage = Constant.GSON.fromJson(message, LineMessage.class); - if(lineMessage != null) { - String ip = lineMessage.getProperties(LineMessage.KEY_IP); - Long lineNumber = lineMessage.getLineNumber(); - Map parseSimple = logParser.parse(lineMessage.getMsgBody(), ip, lineNumber, Instant.now().toEpochMilli(), lineMessage.getFileName()); - log.info("simple data:{}", gson.toJson(parseSimple)); - } - } - - @Test - public void testGson() { - - Gson gson = new GsonBuilder().setObjectToNumberStrategy(ToNumberPolicy.LONG_OR_DOUBLE).create(); - - String logData = "{\"key2\":\"4564646456645656565464564564545\"}"; - - TypeToken> token = new TypeToken>() { - }; - Map rawLogMap = gson.fromJson(logData, token.getType()); - - log.info("result:{}", rawLogMap); - - Long data = 45646464566456545L; - log.info("data:{}", Double.valueOf(data)); - } - -} diff --git a/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/LogParserTest.java b/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/LogParserTest.java deleted file mode 100644 index a790c5a9e..000000000 --- a/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/LogParserTest.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.common; - -import com.google.common.base.Stopwatch; -import com.xiaomi.mone.log.parse.LogParser; -import com.xiaomi.mone.log.parse.LogParserFactory; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.time.DateParser; -import org.apache.commons.lang3.time.FastDateFormat; -import org.junit.Test; - -import java.time.Instant; -import java.util.Map; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/12/29 14:25 - */ -@Slf4j -public class LogParserTest { - - String topicName = "test"; - String tailName = "test"; - String tag = "test"; - String logStoreName = "test"; - - @Test - public void test1() { - Stopwatch stopwatch = Stopwatch.createStarted(); - String keyList = "timestamp:date,podName:keyword,level:keyword,threadName:text,className:text,line:keyword,methodName:keyword,traceId:keyword,message:text,ip:ip,logstore:keyword,logsource:keyword,mqtopic:keyword,mqtag:keyword,logip:keyword,tail:keyword,linenumber:long"; - String valueList = "0,-1,16,-1,-1,-1,-1,-1,-1,1,2,3,4,5,6,7,8,9,10,11,13,12,17,14,15"; - String parseScript = "|"; - String logData = ""; - String ip = "127.0.0.1"; - Long currentStamp = Instant.now().toEpochMilli(); - Integer parserType = LogParserFactory.LogParserEnum.SEPARATOR_PARSE.getCode(); - LogParser customParse = LogParserFactory.getLogParser(parserType, keyList, valueList, parseScript, topicName, tailName, tag, logStoreName); - Map parse = customParse.parse(logData, ip, 1l, currentStamp, ""); - System.out.println(parse); - - System.out.println(customParse.getTimestampFromString("2023-08-25 10:46:09.239", currentStamp)); - stopwatch.stop(); - log.info("cost time:{}", stopwatch.elapsed().toMillis()); - } - - @Test - public void test2() { - Stopwatch stopwatch = Stopwatch.createStarted(); - String keyList = "message:text,logstore:keyword,logsource:keyword,mqtopic:keyword,mqtag:keyword,logip:keyword,tail:keyword,linenumber:long"; - String valueList = "0"; - String parseScript = "|"; - String logData = ""; - String ip = "127.0.0.1"; - Long currentStamp = Instant.now().toEpochMilli(); - Integer parserType = LogParserFactory.LogParserEnum.CUSTOM_PARSE.getCode(); - LogParser customParse = LogParserFactory.getLogParser(parserType, keyList, valueList, parseScript, topicName, tailName, tag, logStoreName); - Map parse = customParse.parse(logData, ip, 1l, currentStamp, ""); - System.out.println(parse); - stopwatch.stop(); - log.info("cost time:{}", stopwatch.elapsed().toMillis()); - } - - @Test - public void test() { - System.out.println("1.647590227174E12".length()); - System.out.println(String.valueOf(Instant.now().toEpochMilli()).length()); - DateParser dateFormat1 = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss"); - System.out.println(dateFormat1.getPattern().length()); - } - - @Test - public void testGetTime() { - System.out.println("1.647590227174E12".length()); - System.out.println(String.valueOf(Instant.now().toEpochMilli()).length()); - DateParser dateFormat1 = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss"); - System.out.println(dateFormat1.getPattern().length()); - } -} diff --git a/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/NetUtilsTest.java b/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/NetUtilsTest.java deleted file mode 100644 index cbfbb11d0..000000000 --- a/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/NetUtilsTest.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.common; - -import lombok.extern.slf4j.Slf4j; -import org.junit.Test; - -import java.util.List; - -/** - * @author: wtt - * @date: 2022/5/31 11:14 - * @description: - */ -@Slf4j -public class NetUtilsTest { - - @Test - public void testQueryIpWithDomain(){ -// String domain = "blog.csdn.net"; - String domain = "open"; - List ips = NetUtils.queryIpWithDomain(domain); - log.info("result:{}",ips); - } -} diff --git a/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/PathUtilsTest.java b/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/PathUtilsTest.java deleted file mode 100644 index 99e146304..000000000 --- a/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/PathUtilsTest.java +++ /dev/null @@ -1,349 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.common; - -import com.google.common.collect.Lists; -import com.google.gson.Gson; -import org.junit.Assert; -import org.junit.Test; - -import java.math.BigDecimal; -import java.time.Instant; -import java.util.List; -import java.util.regex.Pattern; - -/** - * @author shanwb - * @date 2021-10-20 - */ -public class PathUtilsTest { - - @Test - public void test0() { - String logPattern = "/home/work/log/(aa|bb)/server.log"; - List watches = PathUtils.parseWatchDirectory(logPattern); - System.out.println(new Gson().toJson(watches)); - - Assert.assertEquals(2, watches.size()); - Assert.assertEquals("/home/work/log/aa", watches.get(0)); - Assert.assertEquals("/home/work/log/bb", watches.get(1)); - } - - @Test - public void test00() { - String logPattern = "/home/work/logs/neo-logs/(a|b)/applogs/mi_com_event/mi_com_event.log"; - List watches = PathUtils.parseWatchDirectory(logPattern); - System.out.println(new Gson().toJson(watches)); - - Assert.assertEquals(2, watches.size()); - Assert.assertEquals("/home/work/logs/neo-logs/a/applogs", watches.get(0)); - Assert.assertEquals("/home/work/logs/neo-logs/b/applogs", watches.get(1)); - } - - @Test - public void test000() { - String logPattern = "/home/work/logs/neo-logs/(a|b)/mi_com_event/mi_com_event.log"; - List watches = PathUtils.parseWatchDirectory(logPattern); - System.out.println(new Gson().toJson(watches)); - - Assert.assertEquals(2, watches.size()); - Assert.assertEquals("/home/work/logs/neo-logs/a/mi_com_event", watches.get(0)); - Assert.assertEquals("/home/work/logs/neo-logs/b/mi_com_event", watches.get(1)); - } - - @Test - public void test001() { - String logPattern = "/home/work/logs/neo-logs/(a|b)/mi_com_event.log"; - List watches = PathUtils.parseWatchDirectory(logPattern); - System.out.println(new Gson().toJson(watches)); - - Assert.assertEquals(2, watches.size()); - Assert.assertEquals("/home/work/logs/neo-logs/a", watches.get(0)); - Assert.assertEquals("/home/work/logs/neo-logs/b", watches.get(1)); - } - - @Test - public void test01() { - String logPattern = "/home/work/log/test1/server.log,/home/work/log/test/error.log"; - List watches = PathUtils.parseWatchDirectory(logPattern); - System.out.println(new Gson().toJson(watches)); - - Assert.assertEquals(2, watches.size()); - Assert.assertEquals("/home/work/log/test1", watches.get(0)); - Assert.assertEquals("/home/work/log/test", watches.get(1)); - } - - @Test - public void test011() { - String logPattern = "/home/work/log/test/server.log,/home/work/log/test/error.log"; - List watches = PathUtils.parseWatchDirectory(logPattern); - System.out.println(new Gson().toJson(watches)); - - Assert.assertEquals(1, watches.size()); - Assert.assertEquals("/home/work/log/test", watches.get(0)); - } - - @Test - public void test012() { - String logPattern = "/home/work/log/test/server.log|debug.log"; - List watches = PathUtils.parseWatchDirectory(logPattern); - System.out.println(new Gson().toJson(watches)); - - Assert.assertEquals(1, watches.size()); - Assert.assertEquals("/home/work/log/test", watches.get(0)); - } - - @Test - public void test1() { - String logPattern = "/home/work/log/trace/*/server.log"; - List watches = PathUtils.parseWatchDirectory(logPattern); - System.out.println(new Gson().toJson(watches)); - - Assert.assertEquals(1, watches.size()); - Assert.assertEquals("/home/work/log/trace", watches.get(0)); - } - - @Test - public void test2() { - String logPattern = "/home/work/log//trace//*//server.log"; - List watches = PathUtils.parseWatchDirectory(logPattern); - System.out.println(new Gson().toJson(watches)); - - Assert.assertEquals(1, watches.size()); - Assert.assertEquals("/home/work/log/trace", watches.get(0)); - } - - @Test - public void test71() { - String logPattern = "/home/work/logs/applogs/mars/server.log"; - List pathes = PathUtils.parseLevel5Directory(logPattern); - System.out.println(new Gson().toJson(pathes)); - } - - @Test - public void test72() { - String logPattern = "/home/work/log/(aa|bb)/server.log"; - List pathes = PathUtils.parseLevel5Directory(logPattern); - System.out.println(new Gson().toJson(pathes)); - - Assert.assertEquals(2, pathes.size()); - Assert.assertEquals("/home/work/log/aa/server.log", pathes.get(0)); - Assert.assertEquals("/home/work/log/bb/server.log", pathes.get(1)); - } - - @Test - public void test721() { - String logPattern = "/home/work/log/(aa|bb)/logs/server.log"; - List pathes = PathUtils.parseLevel5Directory(logPattern); - System.out.println(new Gson().toJson(pathes)); - - Assert.assertEquals(2, pathes.size()); - Assert.assertEquals("/home/work/log/aa/logs/server.log", pathes.get(0)); - Assert.assertEquals("/home/work/log/bb/logs/server.log", pathes.get(1)); - } - - @Test - public void test723() { - String logPattern = "/home/work/log/(aa|bb)/logs/log1/server.log"; - List pathes = PathUtils.parseLevel5Directory(logPattern); - System.out.println(new Gson().toJson(pathes)); - - Assert.assertEquals(2, pathes.size()); - Assert.assertEquals("/home/work/log/aa/logs/log1/server.log", pathes.get(0)); - Assert.assertEquals("/home/work/log/bb/logs/log1/server.log", pathes.get(1)); - } - - @Test - public void test73() { - String logPattern = "/home/work/log/aa/server.log"; - List pathes = PathUtils.parseLevel5Directory(logPattern); - System.out.println(new Gson().toJson(pathes)); - - Assert.assertEquals(1, pathes.size()); - Assert.assertEquals("/home/work/log/aa/server.log", pathes.get(0)); - } - - @Test - public void test722() { - String logPattern = "/home/work/log/(aa|bb)/logs/log1/(server.log|error.log)"; - List pathes = PathUtils.parseLevel5Directory(logPattern); - System.out.println(new Gson().toJson(pathes)); - - Assert.assertEquals(4, pathes.size()); - Assert.assertEquals("/home/work/log/aa/logs/log1/server.log", pathes.get(0)); - Assert.assertEquals("/home/work/log/aa/logs/log1/error.log", pathes.get(1)); - Assert.assertEquals("/home/work/log/bb/logs/log1/server.log", pathes.get(2)); - Assert.assertEquals("/home/work/log/bb/logs/log1/error.log", pathes.get(3)); - } - - @Test - public void test74() { - String logPattern = "/home/work/log/test/server.log|error.log"; - List pathes = PathUtils.parseLevel5Directory(logPattern); - System.out.println(new Gson().toJson(pathes)); - - Assert.assertEquals(2, pathes.size()); - Assert.assertEquals("/home/work/log/test/server.log", pathes.get(0)); - Assert.assertEquals("/home/work/log/test/error.log", pathes.get(1)); - } - - @Test - public void test741() { - String logPattern = "/home/work/log/(test1|test2)/server.log|error.log"; - List pathes = PathUtils.parseLevel5Directory(logPattern); - System.out.println(new Gson().toJson(pathes)); - - Assert.assertEquals(4, pathes.size()); - Assert.assertEquals("/home/work/log/test1/server.log", pathes.get(0)); - Assert.assertEquals("/home/work/log/test1/error.log", pathes.get(1)); - Assert.assertEquals("/home/work/log/test2/server.log", pathes.get(2)); - Assert.assertEquals("/home/work/log/test2/error.log", pathes.get(3)); - } - - - @Test - public void test81() { - String logPattern = "/home/work/log/error-2022-08-04_05_1.log"; - Pattern pattern = Pattern.compile(logPattern); - - Assert.assertEquals(false, pattern.matcher("/home/work/log/aa/server.log").matches()); - Assert.assertEquals(false, pattern.matcher("/home/work/log/bb/server.log").matches()); - - Assert.assertEquals(false, pattern.matcher("/home/work/log/aa/bb/server.log").matches()); - Assert.assertEquals(false, pattern.matcher("/home/work/log/bb/cc/server.log").matches()); - - Assert.assertEquals(false, pattern.matcher("/home/work/log/aaa/server.log").matches()); - Assert.assertEquals(false, pattern.matcher("/home/work/log/bbb/server.log").matches()); - - Assert.assertEquals(false, pattern.matcher("/home/work/log/aa/server1.log").matches()); - Assert.assertEquals(false, pattern.matcher("/home/work/log/bb/app.log").matches()); - } - - @Test - public void test82() { - String logPattern = "/home/work/log/.*/server.log"; - Pattern pattern = Pattern.compile(logPattern); - - Assert.assertEquals(true, pattern.matcher("/home/work/log/aa/server.log").matches()); - Assert.assertEquals(false, pattern.matcher("/home/work/log/aa/server.log1").matches()); - Assert.assertEquals(true, pattern.matcher("/x/home/work/log/aa/server.log1").find()); - - Assert.assertEquals(true, pattern.matcher("/home/work/log/aa/bb/server.log").matches()); - Assert.assertEquals(false, pattern.matcher("/home/work/log/server.log").matches()); - } - - @Test - public void test91() { - String logPattern = "/home/work/log/aa/server.log"; - System.out.println(logPattern.contains("*")); - System.out.println(logPattern.replaceAll("\\*", ".*")); - - String logPattern2 = "/home/work/log/*/server.log"; - System.out.println(logPattern2.contains("*")); - System.out.println(logPattern2.replaceAll("\\*", ".*")); - - String logPattern3 = "/home/work/log/*/server.log"; - System.out.println(logPattern3.contains(".*")); - } - - @Test - public void test92() { - String logPattern = "/home/work/log/aa/server.log"; - System.out.println(logPattern.substring(0, logPattern.lastIndexOf("/"))); - - } - - - @Test - public void test93() { - String logPattern = "/home/work/log/test/server.log,/home/work/log/test/error.log"; - List pathes = PathUtils.parseLevel5Directory(logPattern); - System.out.println(new Gson().toJson(pathes)); - - Assert.assertEquals(2, pathes.size()); - Assert.assertEquals("/home/work/log/test/server.log", pathes.get(0)); - Assert.assertEquals("/home/work/log/test/error.log", pathes.get(1)); - } - - @Test - public void test94() { - String logPattern = "/home/work/log/test/server.log|error.log"; - List pathes = PathUtils.parseLevel5Directory(logPattern); - System.out.println(new Gson().toJson(pathes)); - - Assert.assertEquals(2, pathes.size()); - Assert.assertEquals("/home/work/log/test/server.log", pathes.get(0)); - Assert.assertEquals("/home/work/log/test/error.log", pathes.get(1)); - } - - @Test - public void testLogPattern() { - String logSplitExpress = "/home/work/log/promotion-admin/info-.*.log"; - Pattern pattern = Pattern.compile(logSplitExpress); - Assert.assertEquals(true, pattern.matcher("/home/work/log/promotion-admin/info-2022-06-21_09_1.log").matches()); - Assert.assertEquals(true, pattern.matcher("/home/work/log/promotion-admin/info-2022-06-21_20_1.log").matches()); - } - - @Test - public void testMultipleFile() { - String logPattern = "/home/work/logs/applogs/graces.log|applog.log.wf"; - List pathes = PathUtils.parseLevel5Directory(logPattern); - System.out.println(new Gson().toJson(pathes)); - - Assert.assertEquals(2, pathes.size()); - Assert.assertEquals("/home/work/logs/applogs/graces.log", pathes.get(0)); - Assert.assertEquals("/home/work/logs/applogs/applog.log.wf", pathes.get(1)); - } - - @Test - public void testLogPattern1() { - String logSplitExpress = "/home/work/logs/neo-logs/(eventapi-stable-66fd975598-z8fd9|eventapi-stable-66fd975598-zfhwq|eventapi-ams-runscript-stable-5c76f54c68-lrmcl|eventapi-stable-66fd975598-l7c4s|eventapi-stable-66fd975598-ckwwd)/applogs/mi_com_event.log-.*"; - Pattern pattern = Pattern.compile(logSplitExpress); - Assert.assertEquals(true, pattern.matcher("/home/work/logs/neo-logs/eventapi-stable-66fd975598-z8fd9/applogs/mi_com_event.log-2022-01-11-16").matches()); - Assert.assertEquals(true, pattern.matcher("/home/work/logs/neo-logs/eventapi-ams-runscript-stable-5c76f54c68-lrmcl/applogs/mi_com_event.log-2022-01-11-16").matches()); - } - - @Test - public void test() { - Long str = 1648116658120L; - String s1 = String.valueOf(1.648099046556E12); - String s = String.valueOf(str); - BigDecimal bd = new BigDecimal(str); - String callBackScore = bd.toPlainString(); - System.out.println(new BigDecimal(Instant.now().toEpochMilli()).toPlainString()); - System.out.println(callBackScore); - System.out.println(s); - System.out.println(str.toString()); - } - - @Test - public void testQueryRuleFiles() { - String directory = "/home/work/log/log-manager/"; - String serverNamePattern = "server.log.*"; - System.out.println(PathUtils.findRulePatternFiles(directory, serverNamePattern)); - } - - @Test - public void testDismantlingStrWithSymbol() { -// String str = "/home/work/log/(a|b)/server.log.*"; -// String str = "/home/work/log/log-agent/server.log.*"; -// String str = "/home/work/log/log-agent/(server.log.*|error.log.*)"; - String str = "logSplitExpress:/home/work/log/(log-agent|log-stream)/(a|b)/server.log.*"; - List cleanedPathList = Lists.newArrayList(); - PathUtils.dismantlingStrWithSymbol(str, cleanedPathList); - cleanedPathList.stream().forEach(System.out::println); - } -} diff --git a/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/PinYin4jUtilsTest.java b/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/PinYin4jUtilsTest.java deleted file mode 100644 index 81b0a06d8..000000000 --- a/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/PinYin4jUtilsTest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.common; - -import com.xiaomi.mone.log.utils.PinYin4jUtils; -import lombok.extern.slf4j.Slf4j; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/11/10 16:07 - */ -@Slf4j -public class PinYin4jUtilsTest { - - @Test - public void testPinYin() { - String chineseStr = "测试logStore"; - String enStr = PinYin4jUtils.getAllPinyin(chineseStr); - Assert.assertEquals("ceshilogStore", enStr); - } - - @Test - public void testPinYinEmpty() { - String chineseStr = ""; - String enStr = PinYin4jUtils.getAllPinyin(chineseStr); - Assert.assertEquals("", enStr); - } - - @Test - public void testFlatMap() { - String keyList = "timestamp:1,level:1,traceId:1,threadName:1,className:1,line:1,methodName:1,message:1,logstore:3,logsource:3,mqtopic:3,mqtag:3,logip:3,tail:3,linenumber:3"; - String columnTypeList = "date,keyword,keyword,text,text,keyword,keyword,keyword,keyword,keyword,keyword,keyword,keyword,keyword,long"; - - List collect = Arrays.stream(keyList.split(",")) - .flatMap(s -> Stream.of(s.split(":")[0])) - .collect(Collectors.toList()); - Assert.assertNotNull(collect); - } - - @Test - public void testLongMax(){ - AtomicLong sendMsgNumber = new AtomicLong(0); - sendMsgNumber.addAndGet(Long.MAX_VALUE); - long l = sendMsgNumber.addAndGet(10032323); - long l1 = 1000 % l; - System.out.println(l1); - } -} diff --git a/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/SimilarityTest.java b/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/SimilarityTest.java deleted file mode 100644 index 44081531d..000000000 --- a/ozhera-log/log-common/src/test/java/com/xiaomi/mone/log/common/SimilarityTest.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.common; - -import cn.hutool.core.date.DateUtil; -import com.xiaomi.mone.log.utils.SimilarUtils; -import org.junit.Test; - -import java.util.Arrays; -import java.util.List; - -import static org.junit.Assert.assertEquals; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/2/16 12:48 - */ -public class SimilarityTest { - - @Test - public void test2() { - System.out.println(DateUtil.parse("2022/01/23 23:23:34").getTime()); - } - - @Test - public void testFindHighestSimilarityStr() { - String baseStr = "appserver.20230912.log"; - List strList = Arrays.asList("appserver.log", "appserver_warn.log", "error.log", "appserver_error.log"); - String expected = "appserver.log"; - String actual = SimilarUtils.findHighestSimilarityStr(baseStr, strList); - assertEquals(expected, actual); - } - - -} diff --git a/ozhera-log/log-manager/pom.xml b/ozhera-log/log-manager/pom.xml index 70e3da9d7..b6ff5a926 100644 --- a/ozhera-log/log-manager/pom.xml +++ b/ozhera-log/log-manager/pom.xml @@ -3,9 +3,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - run.mone + org.apache.ozhera ozhera-log - 1.4.0-jdk21 + 2.0.0-SNAPSHOT 4.0.0 @@ -163,11 +163,6 @@ docean-plugin-mybatis-plus - - run.mone - log-common - - com.google.code.gson gson @@ -182,6 +177,14 @@ run.mone log-api + + run.mone + log-common + + + org.apache.ozhera + ozhera-app + org.apache.rocketmq @@ -313,7 +316,7 @@ false - com.xiaomi.mone.log.manager.bootstrap.MiLogManagerBootstrap + org.apache.ozhera.log.manager.bootstrap.MiLogManagerBootstrap diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/bootstrap/Cache.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/bootstrap/Cache.java deleted file mode 100644 index bf8b9e8d1..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/bootstrap/Cache.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.bootstrap; - -import com.xiaomi.mone.log.manager.service.impl.LogCountServiceImpl; -import com.xiaomi.youpin.docean.anno.DOceanPlugin; -import com.xiaomi.youpin.docean.plugin.IPlugin; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Resource; - -@DOceanPlugin -@Slf4j -public class Cache implements IPlugin { - @Resource - private LogCountServiceImpl logCountService; - - @Override - public void init() { - // Build a log statistics cache - buildLogCountCache(); - } - - public void buildLogCountCache() { - logCountService.collectTopCount(); - log.info("The log statistics leaderboard cache has been added"); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/bootstrap/LogStoragePlugin.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/bootstrap/LogStoragePlugin.java deleted file mode 100644 index 7c3358419..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/bootstrap/LogStoragePlugin.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.bootstrap; - -import com.xiaomi.mone.log.api.enums.LogStorageTypeEnum; -import com.xiaomi.mone.log.common.Constant; -import com.xiaomi.mone.log.manager.mapper.MilogEsClusterMapper; -import com.xiaomi.mone.log.manager.model.pojo.MilogEsClusterDO; -import com.xiaomi.youpin.docean.Ioc; -import com.xiaomi.youpin.docean.anno.DOceanPlugin; -import com.xiaomi.youpin.docean.plugin.IPlugin; -import com.xiaomi.youpin.docean.plugin.config.anno.Value; -import com.xiaomi.youpin.docean.plugin.es.EsService; -import lombok.extern.slf4j.Slf4j; -import org.apache.ibatis.datasource.pooled.PooledDataSource; - -import javax.annotation.Resource; -import javax.sql.DataSource; -import java.util.List; - -import static com.xiaomi.mone.log.common.Constant.GSON; - -@DOceanPlugin -@Slf4j -public class LogStoragePlugin implements IPlugin { - - @Resource - private MilogEsClusterMapper milogEsClusterMapper; - - @Value("$driver.class") - private String driverClass; - - private static final String ADDR_PREFIX = "http://"; - - @Override - public void init() { - log.info("es init start"); - List esClusterList = milogEsClusterMapper.selectAll(); - if (esClusterList == null || esClusterList.isEmpty()) { - log.warn("no Log storage type"); - return; - } - for (MilogEsClusterDO cluster : esClusterList) { - initializeLogStorage(cluster); - } - } - - public void initializeLogStorage(MilogEsClusterDO cluster) { - LogStorageTypeEnum storageTypeEnum = LogStorageTypeEnum.queryByName(cluster.getLogStorageType()); - try { - if (null == storageTypeEnum || LogStorageTypeEnum.ELASTICSEARCH == storageTypeEnum) { - checkAddrUpdate(cluster); - EsService esService = createEsService(cluster); - registerEsService(cluster, esService); - log.info("ES client[{}]Generated successfully[{}]", cluster.getName(), Constant.LOG_STORAGE_SERV_BEAN_PRE + cluster.getId()); - } else if (LogStorageTypeEnum.DORIS == storageTypeEnum) { - DataSource dataSource = createDorisDataSource(cluster); - registerDorisDataSource(cluster, dataSource); - log.info("doris dataSource[{}]Generated successfully[{}]", cluster.getName(), Constant.LOG_STORAGE_SERV_BEAN_PRE + cluster.getId()); - } - } catch (Exception e) { - log.error("init storage client error,cluster{}", GSON.toJson(cluster), e); - } - } - - private EsService createEsService(MilogEsClusterDO cluster) { - switch (cluster.getConWay()) { - case Constant.ES_CONWAY_PWD: - return new EsService(cluster.getAddr(), cluster.getUser(), cluster.getPwd()); - case Constant.ES_CONWAY_TOKEN: - return new EsService(cluster.getAddr(), cluster.getToken(), cluster.getDtCatalog(), cluster.getDtDatabase()); - default: - log.warn("The ES cluster entered an exception: [{}]", cluster); - throw new IllegalArgumentException("Invalid ES connection way"); - } - } - - private void registerEsService(MilogEsClusterDO cluster, EsService esService) { - Ioc.ins().putBean(Constant.LOG_STORAGE_SERV_BEAN_PRE + cluster.getId(), esService); - } - - private DataSource createDorisDataSource(MilogEsClusterDO cluster) { - String addr = cluster.getAddr(); - PooledDataSource pooledDataSource = new PooledDataSource(driverClass, addr, cluster.getUser(), cluster.getPwd()); - pooledDataSource.setPoolPingEnabled(true); - pooledDataSource.setPoolPingQuery("SELECT 1"); - pooledDataSource.setPoolMaximumActiveConnections(20); - return pooledDataSource; - } - - private void registerDorisDataSource(MilogEsClusterDO cluster, DataSource dataSource) { - Ioc.ins().putBean(Constant.LOG_STORAGE_SERV_BEAN_PRE + cluster.getId(), dataSource); - } - - private void checkAddrUpdate(MilogEsClusterDO cluster) { - String addr = cluster.getAddr(); - if (addr.startsWith(ADDR_PREFIX)) { - cluster.setAddr(addr.substring(ADDR_PREFIX.length() + 1)); - } - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/bootstrap/MiLogManagerBootstrap.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/bootstrap/MiLogManagerBootstrap.java deleted file mode 100644 index dca9b1e27..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/bootstrap/MiLogManagerBootstrap.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.bootstrap; - -import com.xiaomi.mone.log.common.Config; -import com.xiaomi.mone.log.manager.controller.interceptor.HttpRequestInterceptor; -import com.xiaomi.youpin.docean.Aop; -import com.xiaomi.youpin.docean.Ioc; -import com.xiaomi.youpin.docean.anno.RequestMapping; -import com.xiaomi.youpin.docean.aop.EnhanceInterceptor; -import com.xiaomi.youpin.docean.common.Cons; -import com.xiaomi.youpin.docean.config.HttpServerConfig; -import com.xiaomi.youpin.docean.mvc.DoceanHttpServer; -import lombok.extern.slf4j.Slf4j; - -import java.util.LinkedHashMap; - -import static com.xiaomi.mone.log.manager.common.utils.ManagerUtil.getConfigFromNanos; - -/** - * @Author goodjava@qq.com - * @Date 2021/6/24 11:29 - */ -@Slf4j -public class MiLogManagerBootstrap { - - public static void main(String[] args) throws InterruptedException { - getConfigFromNanos(); - - LinkedHashMap m = new LinkedHashMap<>(); - m.put(RequestMapping.class, new HttpRequestInterceptor()); - Aop.ins().init(m); - - Ioc.ins().putBean(Cons.AUTO_FIND_IMPL, "true") - .init("com.xiaomi.mone", "com.xiaomi.youpin"); - Config ins = Config.ins(); - - int port = Integer.parseInt(ins.get("serverPort", "")); - DoceanHttpServer server = new DoceanHttpServer(HttpServerConfig.builder().websocket(false).port(port).build()); - server.start(); - log.info("milog manager start finish"); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/ErrorCode.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/ErrorCode.java deleted file mode 100644 index d687fa1b5..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/ErrorCode.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.common; - -public enum ErrorCode { - success(0, "success"), - unknownError(1, "unknown error"), - CREATE_ALERT_FAILURE(2, "failed to create alert"), - SUBMIT_FLINK_JOB(3, "failed to submit flink job"), - ALERT_NOT_FOUND(4, "Alert not found"), - ALERT_REMOVE_FAILED(5, "failed to remove alert"), - FAIL_PARAM(6, "Parameter exceptions"); - - private int code; - private String message; - - ErrorCode(int code, String message) { - this.code = code; - this.message = message; - } - - public int getCode() { - return code; - } - - public String getMessage() { - return message; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/ExceptionCode.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/ExceptionCode.java deleted file mode 100644 index 71f3e5a8e..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/ExceptionCode.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.common; - -/** - * @author gaoxihui - * @date 2021/7/10 7:36 pm - */ -public class ExceptionCode extends RuntimeException { - - private int code; - private String message; - - public ExceptionCode(int code, String message) { - this.code = code; - this.message = message; - } - - - public ExceptionCode(ErrorCode error) { - this.code = error.getCode(); - this.message = error.getMessage(); - } - - public int getCode() { - return code; - } - - public void setCode(int code) { - this.code = code; - } - - @Override - public String getMessage() { - return message; - } - - public void setMessage(String message) { - this.message = message; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/ManagerConstant.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/ManagerConstant.java deleted file mode 100644 index 624bef930..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/ManagerConstant.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.common; - -import com.google.common.collect.Lists; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/5/11 10:42 - */ -public class ManagerConstant { - - public static final String TPC_HOME_URL_HEAD = "tpc_home_url_head"; - - public static final String SPACE_PAGE_URL = "/milog/space/getbypage"; - - public static final String DEPT_LEVEL_PREFIX = "Department level"; - - public static final String DEPT_NAME_PREFIX = "Department name"; - - public static final List RESOURCE_DEFAULT_INITIALIZED_DEPT = Lists.newArrayList("open source"); - - public static final String RESOURCE_NOT_INITIALIZED_MESSAGE = ",Go to the Resource Management page to initialize resources"; - - public static final Integer USER_DEPT_MAX_DEFAULT_LEVEL = 1; - - public static final String ES_LABEL = "open source"; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/MilogConfig.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/MilogConfig.java deleted file mode 100644 index a61052d57..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/MilogConfig.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.common; - -import com.alibaba.nacos.api.config.ConfigFactory; -import com.alibaba.nacos.api.config.ConfigService; -import com.google.gson.Gson; -import com.xiaomi.data.push.nacos.NacosNaming; -import com.xiaomi.mone.log.common.Config; -import com.xiaomi.youpin.docean.anno.Bean; -import com.xiaomi.youpin.docean.anno.Configuration; -import okhttp3.ConnectionPool; -import okhttp3.OkHttpClient; - -import java.util.concurrent.TimeUnit; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/8 17:26 - */ -@Configuration -public class MilogConfig { - - private String defaultNacosAddres = Config.ins().get("defaultNacosAddres", ""); - - @Bean - public ConfigService nacosConfigService() throws Exception { - return ConfigFactory.createConfigService(defaultNacosAddres); - } - - @Bean - public NacosNaming nacosNaming() { - return buildNacosNaming(defaultNacosAddres); - } - - public static NacosNaming buildNacosNaming(String nacosAddress) { - NacosNaming nacosNaming = new NacosNaming(); - nacosNaming.setServerAddr(nacosAddress); - nacosNaming.init(); - return nacosNaming; - } - - - @Bean - public Gson gson() { - return new Gson(); - } - - @Bean - public OkHttpClient okHttpClient() { - return new OkHttpClient().newBuilder() - .connectTimeout(60 * 1000, TimeUnit.MILLISECONDS) - .readTimeout(5 * 60 * 1000, TimeUnit.MILLISECONDS) - .writeTimeout(5 * 60 * 1000, TimeUnit.MILLISECONDS) - .connectionPool(new ConnectionPool(10, 5, TimeUnit.MINUTES)) - .build(); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/Result.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/Result.java deleted file mode 100644 index 68e16839a..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/Result.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.common; - - -/** - * @author gaoxihui - * @date 2021/7/10 7:36 pm - */ -public class Result { - - private int code; - private String message; - private T data; - - public Result() { - } - - public Result(int code, String message) { - this.code = code; - this.message = message; - } - - public Result(int code, String message, T data) { - this.code = code; - this.message = message; - this.data = data; - } - - public static Result fail(ErrorCode error) { - return new Result<>(error.getCode(), error.getMessage()); - } - - public static Result fail(ExceptionCode ex) { - return new Result<>(ex.getCode(), ex.getMessage()); - } - - public static Result success(T t) { - return new Result<>(ErrorCode.success.getCode(), ErrorCode.success.getMessage(), t); - } - - public int getCode() { - return code; - } - - public void setCode(int code) { - this.code = code; - } - - public String getMessage() { - return message; - } - - public void setMessage(String message) { - this.message = message; - } - - public T getData() { - return data; - } - - public void setData(T data) { - this.data = data; - } - - @Override - public String toString() { - return "Result{" + - "code=" + code + - ", message='" + message + '\'' + - ", data=" + data + - '}'; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/Utils.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/Utils.java deleted file mode 100644 index 41133b324..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/Utils.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.common; - -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ThreadLocalRandom; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import static com.xiaomi.mone.log.common.Constant.COMMON_MQ_PREFIX; -import static com.xiaomi.mone.log.common.Constant.COMMON_MQ_SUFFIX; -import static com.xiaomi.mone.log.common.Constant.DEFAULT_TAGS; -import static com.xiaomi.mone.log.common.Constant.UNDERLINE_SYMBOL; - -@Slf4j -public class Utils { - - public static List parseMilogKeyListFromKeyAndType(String keyAndType) { - String[] keyAndTypelist = keyAndType.split(","); - List keylist = new ArrayList<>(); - for (int i = 0; i < keyAndTypelist.length; i++) { - String[] split = keyAndTypelist[i].split(":"); - if (split.length > 0) { - keylist.add(split[0]); - } - } - return keylist; - } - - public static String parseMilogKeyListStrFromKeyAndType(String keyAndType) { - String[] keyAndTypelist = keyAndType.split(","); - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < keyAndTypelist.length; i++) { - String[] split = keyAndTypelist[i].split(":"); - if (i != keyAndTypelist.length - 1 && split.length > 0) { - sb.append(split[0]).append(","); - } else if (i == keyAndTypelist.length - 1 && split.length > 0) { - sb.append(split[0]); - } - } - return sb.toString(); - } - - public static String parse2KeyAndTypeList(String KeyList, String TypeList) { - String[] KLSplit = KeyList.split(","); - String[] TLSplit = TypeList.split(","); - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < KLSplit.length; i++) { - String[] split = KLSplit[i].split(":"); - if (split.length > 0) { - if (i != KLSplit.length - 1) { - sb.append(split[0]).append(":").append(TLSplit[i]).append(","); - } else { - sb.append(split[0]).append(":").append(TLSplit[i]); - } - } - } - return sb.toString(); - } - - public static List getKeyListSlice(String keyList) { - String[] KLSplit = keyList.split(","); - List ret = new ArrayList(); - for (int i = 0; i < KLSplit.length; i++) { - String[] split = KLSplit[i].split(":"); - if (split.length > 1 && !split[1].equals("3")) { - ret.add(split[0]); - } - } - return ret; - } - - /** - * Simple creation of tags - * - * @param spaceId - * @param storeId - * @param tailId - * @return - */ - public static String createTag(Long spaceId, Long storeId, Long tailId) { - return DEFAULT_TAGS + spaceId + UNDERLINE_SYMBOL + storeId + UNDERLINE_SYMBOL + tailId; - } - - /** - * Simply assemble a topic name - * - * @param appId - * @param appName - * @return - */ - public static String assembleTopicName(Long appId, String appName) { - StringBuilder sb = new StringBuilder(appName); - sb.append("-"); - sb.append(appId); - sb.append("-"); - sb.append("topic"); - return sb.toString(); - } - - public static String getKeyValueList(String keyList, String valueList) { - List keyListSlice = Utils.getKeyListSlice(keyList); - String[] valueS = StringUtils.split(valueList, ","); - Map map = new HashMap<>(); - for (int i = 0; i < valueS.length; i++) { - try { - int orderValue = Integer.parseInt(valueS[i]); - if (orderValue >= 0) { - map.put(orderValue, keyListSlice.get(i)); - } - } catch (Exception e) { - log.error(String.format("Data parsing exception,keyList:%s,valueList:%s", keyList, valueS), e); - } - } - return map.entrySet().stream() - .sorted(Comparator.comparing(e -> e.getKey())) - .map(Map.Entry::getValue).collect(Collectors.joining(",")); - } - - public static Map getTodayTime() { - Map map = new HashMap<>(); - Calendar calendar = Calendar.getInstance(); - calendar.set(calendar.get(Calendar.YEAR), calendar.get(Calendar.MONTH), calendar.get(Calendar.DAY_OF_MONTH), 0, 0, 0); - long start = calendar.getTime().getTime(); - map.put("start", start); - Calendar calendar1 = Calendar.getInstance(); - calendar1.set(calendar1.get(Calendar.YEAR), calendar1.get(Calendar.MONTH), calendar1.get(Calendar.DAY_OF_MONTH), 23, 59, 59); - long end = calendar1.getTime().getTime(); - map.put("end", end); - return map; - } - - public static Integer getRandomNum(Integer right) { - return ThreadLocalRandom.current().nextInt(right); - } - - - public static List generateCommonTagTopicName(String orgId) { - return IntStream.range(0, COMMON_MQ_SUFFIX.size()).mapToObj(value -> { - String suffix = COMMON_MQ_SUFFIX.get(value); - if (StringUtils.isNotBlank(orgId)) { - return String.format("%s_%s_%s", COMMON_MQ_PREFIX, orgId, suffix); - } - return String.format("%s_%s", COMMON_MQ_PREFIX, suffix); - }).collect(Collectors.toList()); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/Version.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/Version.java deleted file mode 100644 index 03fff3bb7..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/Version.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.common; - -import java.io.Serializable; - -public class Version implements Serializable { - - - @Override - public String toString() { - return "log-manager:2021-08-26:0.0.2"; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/context/MilogUserUtil.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/context/MilogUserUtil.java deleted file mode 100644 index 07dbd7296..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/context/MilogUserUtil.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.common.context; - -import com.xiaomi.mone.log.api.enums.ProjectSourceEnum; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/14 10:58 - */ -@Slf4j -public class MilogUserUtil { - - /** - * @return - */ - public static boolean isOne() { - try { - return StringUtils.equalsIgnoreCase(MoneUserContext.getCurrentUser().getZone(), ProjectSourceEnum.ONE_SOURCE.getSource()); - } catch (Exception e) { - log.error("query user zone error,is one", e); - } - return false; - } - - /** - * @return - */ - public static boolean isTwo() { - try { - return StringUtils.equalsIgnoreCase(MoneUserContext.getCurrentUser().getZone(), ProjectSourceEnum.TWO_SOURCE.getSource()); - } catch (Exception e) { - log.error("query user zone error, is two", e); - } - return false; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/context/MoneUserContext.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/context/MoneUserContext.java deleted file mode 100644 index e75501921..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/context/MoneUserContext.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.common.context; - -import com.xiaomi.mone.log.manager.model.convert.UserConvert; -import com.xiaomi.mone.log.manager.user.MoneUser; -import com.xiaomi.mone.tpc.login.vo.AuthUserVo; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/2 15:58 - */ -public class MoneUserContext { - - private static ThreadLocal currentUserHolder = new ThreadLocal<>(); - - public static void setCurrentUser(AuthUserVo user, Boolean isAdmin) { - MoneUser moneUser = UserConvert.INSTANCE.userAdapter(user); - moneUser.setIsAdmin(isAdmin); - currentUserHolder.set(moneUser); - } - - public static MoneUser getCurrentUser() { - return currentUserHolder.get(); - } - - public static void clear() { - currentUserHolder.remove(); - } - -} \ No newline at end of file diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/exception/MilogManageException.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/exception/MilogManageException.java deleted file mode 100644 index c865dec0c..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/exception/MilogManageException.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.common.exception; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/22 14:46 - */ -public class MilogManageException extends RuntimeException { - - public MilogManageException(String message, Throwable cause) { - super(message, cause); - } - - public MilogManageException(Throwable cause) { - super(cause); - } - - public MilogManageException(String message) { - super(message); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/helper/MilogAccessHelper.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/helper/MilogAccessHelper.java deleted file mode 100644 index 90cbe8dbe..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/helper/MilogAccessHelper.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.common.helper; - -import cn.hutool.core.util.ReflectUtil; -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.manager.model.bo.AccessMilogParam; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.youpin.docean.anno.Component; - -import java.lang.reflect.Field; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.SYMBOL_COMMA; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/4/19 14:29 - */ -@Component -public class MilogAccessHelper { - - - public String validParam(AccessMilogParam milogParam) { - List builder = Lists.newArrayList(); - Field[] fields = ReflectUtil.getFields(milogParam.getClass()); - for (Field field : fields) { - Object fieldValue = ReflectUtil.getFieldValue(milogParam, field); - if (null == fieldValue) { - builder.add(String.format("%s cannot be empty", field.getName())); - } - } - return builder.stream().collect(Collectors.joining(SYMBOL_COMMA)); - } - - public boolean compareSame(MilogLogTailDo newParam, MilogLogTailDo oldParam) { - Field[] newFields = ReflectUtil.getFieldsDirectly(newParam.getClass(), false); - Field[] oldFields = ReflectUtil.getFieldsDirectly(oldParam.getClass(), false); - for (int i = 0; i < newFields.length; i++) { - Object newFieldValue = ReflectUtil.getFieldValue(newParam, newFields[i]); - Object oldFieldValue = ReflectUtil.getFieldValue(oldParam, oldFields[i]); - if (!Objects.equals(newFieldValue, oldFieldValue)) { - return false; - } - } - return true; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/utils/ExportUtils.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/utils/ExportUtils.java deleted file mode 100644 index ccb4919d2..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/utils/ExportUtils.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.common.utils; - -import com.xiaomi.mone.log.manager.model.dto.LogDataDTO; - -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.Map; - -/** - * Log download utility class - */ -public class ExportUtils { - - // The HSSFWorkbook4Map method writes xls with a maximum cell length limit of 32767, which splits very long content - public static Map SplitTooLongContent(LogDataDTO logDataDto) { - int maxCellLen = 32767; - Map logOfKV = logDataDto.getLogOfKV(); - Map newLogOfKV = new LinkedHashMap<>(); - Iterator> it = logOfKV.entrySet().iterator(); - while (it.hasNext()) { - Map.Entry entry = it.next(); - String keyStr = entry.getKey(); - String valueStr = entry.getValue() == null ? "" : entry.getValue().toString(); - int entryLen = valueStr.length(); - if (entryLen > maxCellLen) { - int cnt = entryLen / maxCellLen; - if (entryLen % maxCellLen != 0) { - cnt++; - } - for (int i = 0; i < cnt; i++) { - String entryKey = String.format("%s-%d", keyStr, i); - String entryValue = ""; - int end = (i + 1) * maxCellLen; - if (end > entryLen) { - end = entryLen; - } - entryValue = valueStr.substring(i * maxCellLen, end); - newLogOfKV.put(entryKey, entryValue); - } - } else { - newLogOfKV.put(keyStr, entry.getValue()); - } - } - return newLogOfKV; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/utils/ManagerUtil.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/utils/ManagerUtil.java deleted file mode 100644 index a9fe143f8..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/utils/ManagerUtil.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.common.utils; - -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.common.Config; -import com.xiaomi.youpin.docean.plugin.nacos.NacosConfig; -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.List; - -import static com.xiaomi.mone.log.common.Constant.*; - -public class ManagerUtil { - - private static final List IGNORE_KEYS = Lists.newArrayList("keyword", "text", "ip", "date"); - - private static final String TAIL_KEY = "tail"; - - public static final String DEFAULT_SERVER_TYPE = "open"; - - private ManagerUtil() { - - } - - public static List getKeyList(String keys, String columnTypes) { - String[] keyDescryArray = keys.split(SYMBOL_COMMA); - String[] keyTypeArray = columnTypes.split(SYMBOL_COMMA); - List keyList = new ArrayList<>(); - for (int i = 0; i < keyDescryArray.length; i++) { - if (!IGNORE_KEYS.contains(keyTypeArray[i].toLowerCase())) { - continue; - } - keyList.add(keyDescryArray[i].split(SYMBOL_COLON)[0]); - } - if (!keyList.contains(TAIL_KEY)) { - keyList.add(TAIL_KEY); - } - return keyList; - } - - - public static List getKeyColonPrefix(String keys) { - String[] keyDescryArray = keys.split(SYMBOL_COMMA); - List keyList = new ArrayList<>(); - for (int i = 0; i < keyDescryArray.length; i++) { - keyList.add(keyDescryArray[i].split(SYMBOL_COLON)[0]); - } - return keyList; - } - - public static String MatchKVPrefix(String message, List keyPerfixList) { - String[] messageSplitArr = message.split(":"); - if (messageSplitArr.length != 2) { - return StringUtils.EMPTY; - } - for (String keyPrefix : keyPerfixList) { - if (messageSplitArr[0].trim().equals(keyPrefix)) { - return String.format("%s%s", keyPrefix, SYMBOL_COLON); - } - } - return StringUtils.EMPTY; - } - - public static void getConfigFromNanos() { - NacosConfig nacosConfig = new NacosConfig(); - nacosConfig.setDataId(Config.ins().get("nacos_config_dataid", "")); - nacosConfig.setGroup(Config.ins().get("nacos_config_group", DEFAULT_GROUP_ID)); - nacosConfig.setServerAddr(Config.ins().get("nacos_config_server_addr", "")); - nacosConfig.init(); - nacosConfig.forEach((k, v) -> Config.ins().set(k, v)); - } - - - /** - * Get the second last layer of the path. - * - * @param logPath - * @return - */ - public static String getPhysicsDirectory(String logPath) { - String serverType = Config.ins().get("server.type", DEFAULT_SERVER_TYPE); - if (StringUtils.equals(DEFAULT_SERVER_TYPE, serverType)) { - return StringUtils.EMPTY; - } - String[] splitPath = StringUtils.split(logPath, "/"); - if (splitPath.length > 2) { - return splitPath[splitPath.length - 2].trim(); - } - return logPath.trim(); - } - - public static boolean isOpenEnv() { - return StringUtils.equals(DEFAULT_SERVER_TYPE, Config.ins().get("server.type", DEFAULT_SERVER_TYPE)); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/validation/HeraConfigValid.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/validation/HeraConfigValid.java deleted file mode 100644 index 2d6fa16cd..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/validation/HeraConfigValid.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.common.validation; - -import com.xiaomi.mone.log.manager.dao.MilogLogTailDao; -import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; -import com.xiaomi.mone.log.manager.model.bo.LogTailParam; -import com.xiaomi.mone.log.manager.model.bo.MlogParseParam; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.mone.log.manager.service.extension.agent.MilogAgentService; -import com.xiaomi.mone.log.manager.service.extension.agent.MilogAgentServiceFactory; -import com.xiaomi.mone.log.manager.service.extension.resource.ResourceExtensionService; -import com.xiaomi.mone.log.manager.service.extension.resource.ResourceExtensionServiceFactory; -import com.xiaomi.mone.log.manager.service.extension.store.StoreExtensionService; -import com.xiaomi.mone.log.manager.service.extension.store.StoreExtensionServiceFactory; -import com.xiaomi.mone.log.manager.service.extension.tail.TailExtensionService; -import com.xiaomi.mone.log.manager.service.extension.tail.TailExtensionServiceFactory; -import com.xiaomi.mone.log.parse.LogParserFactory; -import com.xiaomi.youpin.docean.anno.Component; -import com.xiaomi.youpin.docean.plugin.config.anno.Value; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; -import java.util.List; - -@Slf4j -@Component -public class HeraConfigValid { - - @Value("$server.type") - private String serverType; - - @Resource - private MilogLogstoreDao milogLogstoreDao; - - @Resource - private MilogLogTailDao milogLogtailDao; - - private TailExtensionService tailExtensionService; - - private MilogAgentService milogAgentService; - - private StoreExtensionService storeExtensionService; - - private ResourceExtensionService resourceExtensionService; - - public void init() { - tailExtensionService = TailExtensionServiceFactory.getTailExtensionService(); - milogAgentService = MilogAgentServiceFactory.getAgentExtensionService(); - storeExtensionService = StoreExtensionServiceFactory.getStoreExtensionService(); - resourceExtensionService = ResourceExtensionServiceFactory.getResourceExtensionService(); - } - - public String verifyLogTailParam(LogTailParam param) { - if (null == param.getMilogAppId()) { - return "The selected app cannot be empty"; - } - if (null == param || StringUtils.isBlank(param.getLogPath())) { - return "The path cannot be empty"; - } - if (null == param.getSpaceId()) { - return "The space ID cannot be empty"; - } - if (null == param.getStoreId()) { - return "The store ID cannot be empty"; - } - String path = param.getLogPath(); - if (path.equals("/home/work/log/") || path.equals("/home/work/log") || path.startsWith("/home/work/log") && path.split("/").length < 4) { - return "The log path is wrong, please confirm and submit"; - } - String validMsg = tailExtensionService.validLogPath(param); - return StringUtils.isNotEmpty(validMsg) ? validMsg : StringUtils.EMPTY; - } - - public String checkParseExampleParam(MlogParseParam mlogParseParam) { - StringBuilder sb = new StringBuilder(); - if (!LogParserFactory.LogParserEnum.JSON_PARSE.getCode().equals(mlogParseParam.getParseType()) - && (null == mlogParseParam.getParseScript() || "" == mlogParseParam.getParseScript())) { - sb.append("The parsing script cannot be empty;"); - } - if (null == mlogParseParam.getMsg()) { - sb.append("Log information cannot be empty;"); - } - return sb.toString(); - } - - public boolean checkTailNameSame(String tailName, Long id, String machineRoom, Long spaceId) { - // Verify the log file with the same name - List logtailDoList = milogLogtailDao.queryTailNameExists(tailName, machineRoom, spaceId); - if (null == id) { - return CollectionUtils.isNotEmpty(logtailDoList); - } else { - if (CollectionUtils.isEmpty(logtailDoList)) { - return false; - } - MilogLogTailDo milogLogtailDo = logtailDoList.get(logtailDoList.size() - 1); - return !milogLogtailDo.getId().equals(id); - } - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/validation/OpenSourceValid.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/validation/OpenSourceValid.java deleted file mode 100644 index 517460968..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/validation/OpenSourceValid.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.common.validation; - -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.api.model.vo.MiLogMoneEnv; -import com.xiaomi.youpin.docean.anno.Component; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.util.List; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.SYMBOL_COMMA; - -/** - * @author: wtt - * @date: 2022/5/24 18:32 - * @description: Third-party application access parameter verification - */ -@Slf4j -@Component -public class OpenSourceValid { - - - public String validMiLogMoneEnv(MiLogMoneEnv logMoneEnv) { - List errorInfos = Lists.newArrayList(); - if (null == logMoneEnv.getNewAppId() || null == logMoneEnv.getOldAppId()) { - errorInfos.add("appId Cannot be empty"); - } - if (null == logMoneEnv.getNewEnvId() || null == logMoneEnv.getOldEnvId()) { - errorInfos.add("envId Cannot be empty"); - } - if (StringUtils.isBlank(logMoneEnv.getNewAppName()) || - StringUtils.isBlank(logMoneEnv.getOldAppName())) { - errorInfos.add("appName Cannot be empty"); - } - if (StringUtils.isBlank(logMoneEnv.getNewEnvName()) || - StringUtils.isBlank(logMoneEnv.getOldEnvName())) { - errorInfos.add("envName Cannot be empty"); - } - return errorInfos.stream().collect(Collectors.joining(SYMBOL_COMMA)); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/validation/ResourceValidation.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/validation/ResourceValidation.java deleted file mode 100644 index c0e7bf8b1..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/validation/ResourceValidation.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.common.validation; - -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.api.enums.LogStorageTypeEnum; -import com.xiaomi.mone.log.api.enums.MiddlewareEnum; -import com.xiaomi.mone.log.api.enums.OperateEnum; -import com.xiaomi.mone.log.api.enums.ResourceEnum; -import com.xiaomi.mone.log.api.model.bo.MiLogResource; -import com.xiaomi.youpin.docean.anno.Component; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.*; - -/** - * @author wtt - * @version 1.0 - * @description Resource management parameter verification - * @date 2022/5/11 16:51 - */ -@Slf4j -@Component -public class ResourceValidation { - - public String resourceOperateValid(MiLogResource miLogResource) { - List errorInfos = Lists.newArrayList(); - if (null == miLogResource.getOperateCode()) { - errorInfos.add("operateCode cannot be empty"); - } - boolean operateBoolean = Objects.equals(OperateEnum.UPDATE_OPERATE.getCode(), miLogResource.getOperateCode()) || - Objects.equals(OperateEnum.DELETE_OPERATE.getCode(), miLogResource.getOperateCode()); - if (operateBoolean && null == miLogResource.getId()) { - errorInfos.add("id cannot be empty"); - } - - if (null == miLogResource.getResourceCode()) { - errorInfos.add("resourceCode cannot be empty"); - } - - if (null != miLogResource.getResourceCode() && - null != miLogResource.getOperateCode() && - OperateEnum.DELETE_OPERATE.getCode().equals(miLogResource.getOperateCode())) { - return errorInfos.stream().collect(Collectors.joining(SYMBOL_COMMA)); - } - - if (MiddlewareEnum.ELASTICSEARCH.getCode().equals(miLogResource.getResourceCode())) { - if (StringUtils.isBlank(miLogResource.getConWay())) { - errorInfos.add("conWay cannot be empty"); - } - if (StringUtils.isBlank(miLogResource.getClusterName())) { - errorInfos.add("clusterName cannot be empty"); - } - if (Objects.equals(ES_CONWAY_PWD, miLogResource.getConWay()) && - (StringUtils.isBlank(miLogResource.getAk()) || StringUtils.isBlank(miLogResource.getSk()))) { -// errorInfos.add("If the connection mode is pwd, the user name and password cannot be empty"); - } - if (Objects.equals(ES_CONWAY_TOKEN, miLogResource.getConWay()) && - (StringUtils.isBlank(miLogResource.getEsToken()) || StringUtils.isBlank(miLogResource.getCatalog()) || - StringUtils.isBlank(miLogResource.getDatabase()))) { - errorInfos.add("If the connection mode is token, the token, catalog cluster, and database name cannot be empty"); - } - } - if (StringUtils.isBlank(miLogResource.getAlias())) { - errorInfos.add("alias cannot be empty"); - } - if (StringUtils.isBlank(miLogResource.getRegionEn())) { - errorInfos.add("Region code cannot be empty"); - } - - boolean esIndexExist = MiddlewareEnum.ELASTICSEARCH.getCode().equals(miLogResource.getResourceCode()) && - CollectionUtils.isEmpty(miLogResource.getMultipleEsIndex()); - if (esIndexExist && StringUtils.equalsIgnoreCase(LogStorageTypeEnum.ELASTICSEARCH.name(), miLogResource.getStorageType())) { - errorInfos.add("ES index information cannot be empty"); - } - if (ResourceEnum.MQ.getCode().equals(miLogResource.getResourceCode()) && - StringUtils.isEmpty(miLogResource.getClusterName())) { - errorInfos.add("MQ address information cannot be empty"); - } - return errorInfos.stream().collect(Collectors.joining(SYMBOL_COMMA)); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/validation/StoreSpaceAuthValid.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/validation/StoreSpaceAuthValid.java deleted file mode 100644 index 5fd6b18dd..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/validation/StoreSpaceAuthValid.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.common.validation; - -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; -import com.xiaomi.mone.log.manager.dao.MilogSpaceDao; -import com.xiaomi.mone.log.manager.model.bo.StoreSpaceAuth; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogSpaceDO; -import com.xiaomi.youpin.docean.anno.Component; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Resource; -import java.util.List; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.SYMBOL_COMMA; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/7/14 16:32 - */ -@Slf4j -@Component -public class StoreSpaceAuthValid { - - @Resource - private MilogLogstoreDao milogLogstoreDao; - - @Resource - private MilogSpaceDao milogSpaceDao; - - public String validParam(StoreSpaceAuth storeSpaceAuth) { - List errorInfos = Lists.newArrayList(); - if (null == storeSpaceAuth) { - errorInfos.add("The parameter cannot be empty"); - } - if (null == storeSpaceAuth.getStoreId()) { - errorInfos.add("The store ID cannot be empty"); - } - if (null == storeSpaceAuth.getSpaceId()) { - errorInfos.add("The space ID cannot be empty"); - } - return errorInfos.stream().collect(Collectors.joining(SYMBOL_COMMA)); - } - - public String validStoreAuthData(StoreSpaceAuth storeSpaceAuth) { - List errorInfos = Lists.newArrayList(); - MilogLogStoreDO milogLogStoreDO = milogLogstoreDao.queryById(storeSpaceAuth.getStoreId()); - if (null == milogLogStoreDO) { - errorInfos.add("The store information does not exist, please check if it is correct"); - } - MilogSpaceDO milogSpaceDO = milogSpaceDao.queryById(storeSpaceAuth.getSpaceId()); - if (null == milogSpaceDO) { - errorInfos.add("The space information does not exist, please check if it is correct"); - } - if (storeSpaceAuth.getSpaceId().equals(milogLogStoreDO.getSpaceId())) { - errorInfos.add("The store already belongs to the space and cannot be relicensed"); - } - return errorInfos.stream().collect(Collectors.joining(SYMBOL_COMMA)); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/validation/StoreValidation.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/validation/StoreValidation.java deleted file mode 100644 index 33ccc5c65..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/common/validation/StoreValidation.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.common.validation; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.xiaomi.mone.log.api.model.vo.ResourceUserSimple; -import com.xiaomi.mone.log.manager.common.context.MoneUserContext; -import com.xiaomi.mone.log.manager.common.exception.MilogManageException; -import com.xiaomi.mone.log.manager.dao.MilogMiddlewareConfigDao; -import com.xiaomi.mone.log.manager.domain.EsCluster; -import com.xiaomi.mone.log.manager.model.pojo.MilogEsClusterDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogMiddlewareConfig; -import com.xiaomi.mone.log.manager.model.vo.LogStoreParam; -import com.xiaomi.mone.log.manager.service.extension.store.StoreExtensionService; -import com.xiaomi.mone.log.manager.service.extension.store.StoreExtensionServiceFactory; -import com.xiaomi.mone.log.manager.service.impl.MilogMiddlewareConfigServiceImpl; -import com.xiaomi.youpin.docean.anno.Component; -import com.xiaomi.youpin.docean.plugin.config.anno.Value; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.SYMBOL_COMMA; - -/** - * @author: wtt - * @date: 2022/5/12 17:10 - * @description: Parameter validation in the store - */ -@Slf4j -@Component -public class StoreValidation { - - @Resource - private MilogMiddlewareConfigServiceImpl milogMiddlewareConfigService; - - private StoreExtensionService storeExtensionService; - - @Resource - private MilogMiddlewareConfigDao milogMiddlewareConfigDao; - - @Resource - private EsCluster esCluster; - - @Value("$tpc.devMode") - private String tpcDevMode; - - public void init() { - storeExtensionService = StoreExtensionServiceFactory.getStoreExtensionService(); - } - - public String logStoreParamValid(LogStoreParam storeParam) { - if (StringUtils.equals("true", tpcDevMode)) { - return ""; - } - if (null == MoneUserContext.getCurrentUser()) { - throw new MilogManageException("please go to login"); - } - List errorInfos = Lists.newArrayList(); - if (null == storeParam.getSpaceId()) { - errorInfos.add("Space information cannot be empty"); - } - if (null == storeParam || StringUtils.isBlank(storeParam.getLogstoreName())) { - errorInfos.add("logStore cannot be empty"); - } - if (StringUtils.isEmpty(storeParam.getMachineRoom())) { - errorInfos.add("Computer room information cannot be empty"); - return errorInfos.stream().collect(Collectors.joining(SYMBOL_COMMA)); - } - if (null == storeParam.getLogType()) { - errorInfos.add("Log type cannot be empty"); - } - if (StringUtils.isEmpty(storeParam.getKeyList())) { - errorInfos.add("Index columns cannot be empty"); - } - List duplicatedKeyList = getDuplicatedKeys(storeParam.getKeyList()); - if (!duplicatedKeyList.isEmpty()) { - errorInfos.add("Index column fields are duplicated, remove the duplicate fields:" + duplicatedKeyList); - } - // Additional field inspection - if (storeExtensionService.storeInfoCheck(storeParam)) { - return errorInfos.stream().collect(Collectors.joining(SYMBOL_COMMA)); - } - if (null == storeParam.getMqResourceId() || null == storeParam.getEsResourceId()) { - //Verify whether the department to which the current user belongs initializes the resource - ResourceUserSimple resourceUserSimple = milogMiddlewareConfigService.userResourceList(storeParam.getMachineRoom(), storeParam.getLogType()); - if (!resourceUserSimple.getInitializedFlag()) { - errorInfos.add(resourceUserSimple.getNotInitializedMsg()); - } - boolean resourceChosen = null == storeParam.getMqResourceId() || null == storeParam.getEsResourceId(); - if (resourceUserSimple.getInitializedFlag() && - resourceUserSimple.getShowFlag() && resourceChosen) { - errorInfos.add("Please select the required resource information first"); - } - } else { - if (null != storeParam.getMqResourceId()) { - MilogMiddlewareConfig milogMiddlewareConfig = milogMiddlewareConfigDao.queryById(storeParam.getMqResourceId()); - if (null == milogMiddlewareConfig) { - errorInfos.add("MQ resource information cannot be empty"); - } - } - if (null != storeParam.getEsResourceId()) { - MilogEsClusterDO esClusterDO = esCluster.getById(storeParam.getEsResourceId()); - if (null == esClusterDO) { - errorInfos.add("ES resource information cannot be empty"); - } - } - } - return errorInfos.stream().collect(Collectors.joining(SYMBOL_COMMA)); - } - - private List getDuplicatedKeys(String keyListStr) { - String[] keyList = keyListStr.split(","); - List duplicatedKeys = Lists.newArrayList(); - Map keyMap = Maps.newHashMap(); - for (int i = 0; i < keyList.length; i++) { - String keyName = keyList[i].split(":")[0]; - if (keyMap.containsKey(keyName)) { - duplicatedKeys.add(keyName); - } else { - keyMap.put(keyName, true); - } - } - return duplicatedKeys; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/EsDataController.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/EsDataController.java deleted file mode 100644 index 930f8bf05..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/EsDataController.java +++ /dev/null @@ -1,285 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.controller; - -import com.xiaomi.mone.log.api.model.dto.TraceLogDTO; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.dto.*; -import com.xiaomi.mone.log.manager.model.vo.*; -import com.xiaomi.mone.log.manager.service.impl.EsDataServiceImpl; -import com.xiaomi.mone.log.manager.service.impl.EsIndexTemplateServiceImpl; -import com.xiaomi.mone.log.manager.service.impl.LogCountServiceImpl; -import com.xiaomi.mone.log.manager.service.impl.MilogLogSearchSaveServiceImpl; -import com.xiaomi.youpin.docean.anno.Controller; -import com.xiaomi.youpin.docean.anno.RequestMapping; -import com.xiaomi.youpin.docean.anno.RequestParam; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Resource; -import java.io.IOException; -import java.util.List; - -@Controller -@Slf4j -public class EsDataController { - @Resource - private EsDataServiceImpl esDataService; - - @Resource - private EsIndexTemplateServiceImpl esIndexTemplateService; - - @Resource - private LogCountServiceImpl logCountService; - - @Resource - private MilogLogSearchSaveServiceImpl searchSaveService; - - @RequestMapping(path = "/es/updateIndexTemplate", method = "POST") - public Boolean updateIndexTemplate(@RequestParam(value = "updateIndexTemplateCommand") UpdateIndexTemplateCommand updateIndexTemplateCommand) throws IOException { - return esIndexTemplateService.updateIndexTemplate(updateIndexTemplateCommand); - } - - @RequestMapping(path = "/es/createIndex", method = "get") - public Boolean createIndex(@RequestParam(value = "templateName") String templateName) throws IOException { - return esIndexTemplateService.createIndex(templateName); - } - - @RequestMapping(path = "/log/export", method = "get") - public void logExport(@RequestParam(value = "logstore") String logstore, - @RequestParam(value = "storeId") Long storeId, - @RequestParam(value = "tail") String tail, - @RequestParam(value = "startTime") Long startTime, - @RequestParam(value = "endTime") Long endTime, - @RequestParam(value = "fullTextSearch") String fullTextSearch) throws Exception { - LogQuery logQuery = new LogQuery(logstore, storeId, tail, startTime, endTime, fullTextSearch, "timestamp"); - esDataService.logExport(logQuery); - } - - @RequestMapping(path = "/log/query") - public Result logQuery(LogQuery logQuery) { - return esDataService.logQuery(logQuery); - } - - @RequestMapping(path = "/log/context") - public Result logContext(LogContextQuery logContextQuery) throws Exception { - return esDataService.getDocContext(logContextQuery); - } - - @RequestMapping(path = "/log/queryRegionTraceLog") - public Result queryRegionTraceLog(RegionTraceLogQuery regionTraceLogQuery) throws IOException { - return esDataService.queryRegionTraceLog(regionTraceLogQuery); - } - - /** - * tail log collection ranking - * - * @return - */ - @RequestMapping(path = "/log/statistics/collectTop", method = "get") - public Result> collectTop() { - return logCountService.collectTop(); - } - - /** - * space log collection trends - * - * @return - */ - @RequestMapping(path = "/log/statistics/spaceCollectTop", method = "get") - public Result> spaceCollectTop() { - return logCountService.collectSpaceTop(); - } - - /** - * tail log collection trends - * - * @param tailId - * @return - * @throws IOException - */ - @RequestMapping(path = "/log/statistics/collectTrend", method = "get") - public Result> collectTrend(@RequestParam(value = "tailId") Long tailId) throws IOException { - return logCountService.collectTrend(tailId); - } - - /** - * Space log collection trends - * - * @param spaceId - * @return - * @throws IOException - */ - @RequestMapping(path = "/log/statistics/spaceCollectTrend", method = "get") - public Result> spaceCollectTrend(@RequestParam(value = "spaceId") Long spaceId) { - return logCountService.spaceCollectTrend(spaceId); - } - - /** - * Execute the script-tail log collection trend cache flush - */ - @RequestMapping(path = "/log/statistics/collectTrendRefresh", method = "get") - public void collectTrendRefresh() { - logCountService.collectTrendRefresh(); - } - - /** - * Perform a script-space log collection trend cache flush - */ - @RequestMapping(path = "/log/statistics/spaceCollectTrendRefresh", method = "get") - public void spaceCollectTrendRefresh() { - logCountService.collectSpaceTrend(); - } - - /** - * Execute script - Counts the number of logs for a specified date - * - * @param thisDay - * @throws IOException - */ - @RequestMapping(path = "/log/statistics/collectLogCount", method = "get") - public void collectLogCount(@RequestParam(value = "thisDay") String thisDay) throws IOException { - try { - logCountService.collectLogCount(thisDay); - } catch (Exception e) { - log.error("Log statistics failed,error:[{}]", e.getMessage()); - } - - } - - /** - * Perform a script - statistics log ranking cache flush - * - * @throws IOException - */ - @RequestMapping(path = "/log/statistics/collectTopCount", method = "get") - public void collectTopCount() throws IOException { - logCountService.collectTopCount(); - } - - /** - * Perform a script-space log volume cache flush - */ - @RequestMapping(path = "/log/statistics/collectSpaceTopRefresh", method = "get") - public void collectSpaceTopRefresh() { - logCountService.collectSpaceTopCount(); - } - - /** - * Execute script - Deletes log volume statistics for the specified date - * - * @param thisDay - * @throws IOException - */ - @RequestMapping(path = "/log/statistics/collectLogDelete", method = "get") - public void collectLogDelete(@RequestParam(value = "thisDay") String thisDay) throws IOException { - logCountService.collectLogDelete(thisDay); - } - - /** - * Execute script - View log statistics cache - * - * @throws IOException - */ - @RequestMapping(path = "/log/statistics/showLogCountCache", method = "get") - public void showLogCountCache() throws IOException { - logCountService.showLogCountCache(); - } - - /** - * Save the query - list display - * - * @throws IOException - */ - @RequestMapping(path = "/log/save/list", method = "get") - public Result> searchSavelList(@RequestParam(value = "storeId") Long storeId, - @RequestParam(value = "sort") Integer sort) { - return searchSaveService.list(storeId, sort); - } - - /** - * Save the query - query parameters - * - * @throws IOException - */ - @RequestMapping(path = "/log/save/detail", method = "get") - public SearchSaveDTO saveDetail(@RequestParam(value = "id") Long id) { - return searchSaveService.getById(id); - } - - /** - * Save Query - Save - * - * @throws IOException - */ - @RequestMapping(path = "/log/save/favourite") - public Result saveSearchSave(SearchSaveInsertCmd cmd) { - return searchSaveService.save(cmd); - } - - /** - * Save Query - Unfavorite - * - * @throws IOException - */ - @RequestMapping(path = "/log/save/defavourite", method = "get") - public Result defavourite(@RequestParam(value = "sort") Integer sort, @RequestParam(value = "id") Long id) { - return searchSaveService.deFavourite(sort, id); - } - - /** - * Save Query - Update - * - * @throws IOException - */ - @RequestMapping(path = "/log/save/update") - public Result saveSearchUpdate(SearchSaveUpdateCmd cmd) { - return searchSaveService.update(cmd); - } - - /** - * Save the query-swap order - */ - @RequestMapping(path = "/log/save/swap", method = "get") - public Result swap(@RequestParam(value = "idFrom") Long idFrom, @RequestParam(value = "idTo") Long idTo) { - return searchSaveService.swapOrder(idFrom, idTo); - } - - /** - * Save Query - Delete - * - * @throws IOException - */ - @RequestMapping(path = "/log/save/delete", method = "get") - public Result saveDelete(@RequestParam(value = "id") Long id) { - return searchSaveService.removeById(id); - } - - @RequestMapping(path = "/log/trace/url") - public Result getTraceAppLogUrl(TraceAppLogUrlQuery query) { - return esDataService.getTraceAppLogUrl(query); - } - - @RequestMapping(path = "/log/save/storeTree") - public Result> storeTree(KeywordPageParam keywordPageParam) { - return searchSaveService.storeTree(keywordPageParam); - } - - @RequestMapping(path = "/log/save/initOrder", method = "get") - public Result initOrder(@RequestParam(value = "key") String key) { - return searchSaveService.initOrder(key); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/LogAnalyseController.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/LogAnalyseController.java deleted file mode 100644 index e93e29e96..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/LogAnalyseController.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.controller; - -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.dto.DashboardDTO; -import com.xiaomi.mone.log.manager.model.dto.GraphDTO; -import com.xiaomi.mone.log.manager.model.dto.GraphTypeDTO; -import com.xiaomi.mone.log.manager.model.dto.LogAnalyseDataDTO; -import com.xiaomi.mone.log.manager.model.vo.*; -import com.xiaomi.mone.log.manager.service.impl.LogAnalyseService; -import com.xiaomi.youpin.docean.anno.Controller; -import com.xiaomi.youpin.docean.anno.RequestMapping; -import com.xiaomi.youpin.docean.anno.RequestParam; - -import javax.annotation.Resource; -import java.util.List; - -@Controller -public class LogAnalyseController { - @Resource - private LogAnalyseService logAnalyseService; - - @RequestMapping(path = "/log/analyse/show") - public Result logQuery(LogAnalyseQuery logAnalyseQuery) throws Exception { - return logAnalyseService.getDashboardGraph(logAnalyseQuery); - } - - @RequestMapping(path = "/log/analyse/data") - public Result data(LogAnalyseDataQuery query) throws Exception { - return logAnalyseService.data(query); - } - - @RequestMapping(path = "/log/analyse/dataPre") - public Result dataPre(LogAnalyseDataPreQuery query) throws Exception { - return logAnalyseService.dataPre(query); - } - - @RequestMapping(path = "/log/analyse/type") - public Result> type() throws Exception { - return logAnalyseService.type(); - } - - @RequestMapping(path = "/log/analyse/key", method = "get") - public Result> supportKey(@RequestParam("storeId") Long storeId) throws Exception { - return logAnalyseService.supportKey(storeId); - } - - @RequestMapping(path = "/log/analyse/ref") - public Result ref(DGRefCmd cmd) throws Exception { - return logAnalyseService.ref(cmd); - } - - @RequestMapping(path = "/log/analyse/delRef") - public Result delRef(DGRefDelCmd cmd) throws Exception { - return logAnalyseService.delRef(cmd); - } - - @RequestMapping(path = "/log/analyse/updateRef") - public Result updateRef(DGRefUpdateCmd cmd) throws Exception { - return logAnalyseService.updateRef(cmd); - } - - @RequestMapping(path = "/log/graph/create") - public Result createGraph(CreateGraphCmd cmd) throws Exception { - return logAnalyseService.createGraph(cmd); - } - - @RequestMapping(path = "/log/graph/update") - public Result createGraph(UpdateGraphCmd cmd) throws Exception { - return logAnalyseService.updateGraph(cmd); - } - - @RequestMapping(path = "/log/graph/delete") - public Result deleteGraph(Long graphId) throws Exception { - return logAnalyseService.deleteGraph(graphId); - } - - @RequestMapping(path = "/log/graph/search") - public Result> searchGraph(GraphQuery query) throws Exception { - return logAnalyseService.searchGraph(query); - } - - @RequestMapping(path = "/log/dashboard/create") - public Result createDashboard(CreateDashboardCmd cmd) throws Exception { - return logAnalyseService.createDashboard(cmd); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/LogTemplateController.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/LogTemplateController.java deleted file mode 100644 index 2096f664e..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/LogTemplateController.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.controller; - -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.dto.LogTemplateDTO; -import com.xiaomi.mone.log.manager.model.dto.LogTemplateDetailDTO; -import com.xiaomi.mone.log.manager.service.impl.LogTemplateServiceImpl; -import com.xiaomi.youpin.docean.anno.Controller; -import com.xiaomi.youpin.docean.anno.RequestMapping; -import com.xiaomi.youpin.docean.anno.RequestParam; - -import javax.annotation.Resource; -import java.util.List; - -@Controller -public class LogTemplateController { - - @Resource - private LogTemplateServiceImpl logTemplateService; - - @RequestMapping(path = "/log/logTemplate/list", method = "get") - public Result> getLogTemplateList(@RequestParam(value = "area") String area) { - return logTemplateService.getLogTemplateList(area); - } - - @RequestMapping(path = "/log/logTemplate", method = "get") - public Result getLogTemplate(@RequestParam(value = "logTemplateId") long logTemplateId) { - return logTemplateService.getLogTemplateById(logTemplateId); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MIlogAgentManageController.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MIlogAgentManageController.java deleted file mode 100644 index e39066539..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MIlogAgentManageController.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.controller; - -import com.xiaomi.mone.log.api.model.vo.AgentLogProcessDTO; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.bo.MilogAgentIpParam; -import com.xiaomi.mone.log.manager.service.extension.agent.MilogAgentService; -import com.xiaomi.mone.log.manager.service.extension.agent.MilogAgentServiceFactory; -import com.xiaomi.youpin.docean.anno.Controller; -import com.xiaomi.youpin.docean.anno.RequestMapping; -import com.xiaomi.youpin.docean.anno.RequestParam; - -import java.util.List; - -@Controller -public class MIlogAgentManageController { - - private MilogAgentService milogAgentService; - - public void init() { - milogAgentService = MilogAgentServiceFactory.getAgentExtensionService(); - } - - /** - * Log Collection Progress - * - * @param ip - * @return - */ - @RequestMapping(path = "/milog/meta/process", method = "get") - public Result> process(@RequestParam(value = "ip") String ip) { - return milogAgentService.process(ip); - } - - /** - * Agent distribution configuration - full distribution - * - * @param agentId - * @param agentIp - * @param agentMachine - * @return - */ - @RequestMapping(path = "/milog/agent/config/issue", method = "get") - public Result configIssueAgent(@RequestParam(value = "agentId") String agentId, - @RequestParam(value = "agentIp") String agentIp, - @RequestParam("agentMachine") String agentMachine) { - return milogAgentService.configIssueAgent(agentId, agentIp, agentMachine); - } - - /** - * @param agentIpParam - * @return - */ - @RequestMapping(path = "/milog/agent/deployee") - public Result agentDeploy(@RequestParam("ips") MilogAgentIpParam agentIpParam) { - return null; - } - - /** - * @param agentIpParam - * @return - */ - @RequestMapping(path = "/milog/agent/offline/batch") - public Result agentOfflineBatch(@RequestParam("param") MilogAgentIpParam agentIpParam) { - return milogAgentService.agentOfflineBatch(agentIpParam); - } - - /** - * @param ip - * @return - */ - @RequestMapping(path = "/milog/agent/upgrade") - public Result agentUpgrade(@RequestParam("ip") String ip) { - return null; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MiLogMetaManageController.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MiLogMetaManageController.java deleted file mode 100644 index 3f1818b52..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MiLogMetaManageController.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.controller; - -import com.xiaomi.mone.log.api.model.meta.LogCollectMeta; -import com.xiaomi.mone.log.manager.service.impl.MiLogMetaManageServiceImpl; -import com.xiaomi.youpin.docean.anno.Controller; -import com.xiaomi.youpin.docean.anno.RequestMapping; -import com.xiaomi.youpin.docean.anno.RequestParam; - -import javax.annotation.Resource; - -@Controller -public class MiLogMetaManageController { - @Resource - private MiLogMetaManageServiceImpl miLogMetaManageService; - - @RequestMapping(path = "/milog/meta/query", method = "get") - public LogCollectMeta queryLogCollectMeta(@RequestParam(value = "agentId") String agentId, @RequestParam(value = "agentIp") String agentIp) { - return miLogMetaManageService.queryLogCollectMeta(agentId, agentIp); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MiLogResourceController.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MiLogResourceController.java deleted file mode 100644 index 7010160c1..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MiLogResourceController.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.controller; - -import com.xiaomi.mone.log.api.model.bo.MiLogResource; -import com.xiaomi.mone.log.api.model.bo.ResourcePage; -import com.xiaomi.mone.log.api.model.vo.ResourceInfo; -import com.xiaomi.mone.log.api.model.vo.ResourceUserSimple; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.page.PageInfo; -import com.xiaomi.mone.log.manager.service.impl.MilogMiddlewareConfigServiceImpl; -import com.xiaomi.youpin.docean.anno.Controller; -import com.xiaomi.youpin.docean.anno.RequestMapping; -import com.xiaomi.youpin.docean.anno.RequestParam; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Resource; - -/** - * @author wtt - * @version 1.0 - * @description resource management controller - * @date 2022/5/10 11:03 - */ -@Slf4j -@Controller -public class MiLogResourceController { - - @Resource - private MilogMiddlewareConfigServiceImpl milogMiddlewareConfigService; - - // @UnifiedResponse - @RequestMapping(path = "/milog/resource/with/resource/code", method = "POST") - public Result> queryResourceWithTab(@RequestParam(value = "resourcePage") ResourcePage resourceCode) { - return Result.success(milogMiddlewareConfigService.queryResourceWithTab(resourceCode)); - } - - /** - * resource operation - */ - @RequestMapping(path = "/milog/resource/operate", method = "POST") - public Result resourceOperate(@RequestParam(value = "resource") MiLogResource miLogResource) { - return milogMiddlewareConfigService.resourceOperate(miLogResource); - } - - /** - * Resource details - */ - @RequestMapping(path = "/milog/resource/detail", method = "GET") - public Result resourceDetail(@RequestParam(value = "resourceCode") Integer resourceCode, - @RequestParam(value = "id") Long id) { - return Result.success(milogMiddlewareConfigService.resourceDetail(resourceCode, id)); - } - - /** - * Query whether the resource has been initialized when creating a new store - */ - @RequestMapping(path = "/milog/resource/initialized/user/dept", method = "GET") - public Result userResourceList( - @RequestParam(value = "regionCode") String regionCode, - @RequestParam(value = "logTypeCode") Integer logTypeCode) { - return Result.success(milogMiddlewareConfigService.userResourceList(regionCode,logTypeCode)); - } - - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogConfigController.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogConfigController.java deleted file mode 100644 index 96e3a3af1..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogConfigController.java +++ /dev/null @@ -1,368 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.controller; - -import com.xiaomi.mone.log.api.model.vo.TailLogProcessDTO; -import com.xiaomi.mone.log.api.model.vo.UpdateLogProcessCmd; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.MilogSpaceParam; -import com.xiaomi.mone.log.manager.model.bo.BatchQueryParam; -import com.xiaomi.mone.log.manager.model.bo.LogTailParam; -import com.xiaomi.mone.log.manager.model.bo.MlogParseParam; -import com.xiaomi.mone.log.manager.model.dto.*; -import com.xiaomi.mone.log.manager.model.page.PageInfo; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.vo.LogStoreParam; -import com.xiaomi.mone.log.manager.model.vo.QuickQueryVO; -import com.xiaomi.mone.log.manager.service.HeralogHomePageService; -import com.xiaomi.mone.log.manager.service.impl.LogProcessServiceImpl; -import com.xiaomi.mone.log.manager.service.impl.LogSpaceServiceImpl; -import com.xiaomi.mone.log.manager.service.impl.LogStoreServiceImpl; -import com.xiaomi.mone.log.manager.service.impl.LogTailServiceImpl; -import com.xiaomi.youpin.docean.anno.Controller; -import com.xiaomi.youpin.docean.anno.RequestMapping; -import com.xiaomi.youpin.docean.anno.RequestParam; - -import javax.annotation.Resource; -import java.io.IOException; -import java.util.List; -import java.util.Map; - -@Controller -public class MilogConfigController { - - @Resource - private LogSpaceServiceImpl logSpaceService; - - @Resource - private LogStoreServiceImpl logStoreService; - - @Resource - private LogTailServiceImpl logTailService; - - @Resource - private HeralogHomePageService heralogHomePageService; - - @Resource - private LogProcessServiceImpl logProcessService; - - /** - * ********* milogSpace ********** - */ - - @RequestMapping(path = "/milog/space/new") - public Result newMilogSpace(MilogSpaceParam cmd) { - return logSpaceService.newMilogSpace(cmd); - } - - @RequestMapping(path = "/milog/space/getbyid", method = "get") - public Result getMilogSpaceById(@RequestParam("id") Long id) { - return logSpaceService.getMilogSpaceById(id); - } - - @RequestMapping(path = "/milog/space/getall", method = "get") - public Result>> getMilogSpaces(@RequestParam("tenantId") Long tenantId) { - return logSpaceService.getMilogSpaces(tenantId); - } - - @RequestMapping(path = "/milog/space/getbypage", method = "get") - public Result> getMilogSpaceByPage(@RequestParam(value = "spaceName") String spaceName, - @RequestParam("tenantId") Long tenantId, - @RequestParam("page") Integer page, - @RequestParam("pageSize") Integer pageSize) { - return logSpaceService.getMilogSpaceByPage(spaceName, tenantId, page, pageSize); - } - - @RequestMapping(path = "/milog/space/delete", method = "get") - public Result deleteMilogSpace(@RequestParam("id") Long id) { - return logSpaceService.deleteMilogSpace(id); - } - - @RequestMapping(path = "/milog/space/update") - public Result updateMilogSpace(@RequestParam("param") MilogSpaceParam param) { - return logSpaceService.updateMilogSpace(param); - } - - /** - * ********* milogLogstore ********** - */ - @RequestMapping(path = "/milog/store/new") - public Result newLogStore(@RequestParam("param") LogStoreParam param) { - return logStoreService.newLogStore(param); - } - - @RequestMapping(path = "/milog/store/getbyid", method = "get") - public Result getLogStoreById(@RequestParam("id") Long id) { - return logStoreService.getLogStoreById(id); - } - - @RequestMapping(path = "/milog/store/getbyids") - public Result> getLogStoreByIds(BatchQueryParam param) throws IOException { - return logStoreService.getLogStoreByIds(param.getIds()); - } - - @RequestMapping(path = "/milog/store/getbypage", method = "get") - public Result> getLogStoreByPage(@RequestParam(value = "logstoreName") String logstoreName, - @RequestParam("spaceId") Long spaceId, - @RequestParam("page") int page, - @RequestParam("pageSize") int pageSize) { - return logStoreService.getLogStoreByPage(logstoreName, spaceId, page, pageSize); - } - - @RequestMapping(path = "/milog/store/getbyspace", method = "get") - public Result>> getLogStoreBySpaceId(@RequestParam("spaceId") Long spaceId) { - return logStoreService.getLogStoreBySpaceId(spaceId); - } - - @RequestMapping(path = "/milog/store/getall", method = "get") - public Result> getAllLogStore() { - return logStoreService.getAllLogStore(); - } - - @RequestMapping(path = "/milog/store/delete", method = "get") - public Result deleteLogStore(@RequestParam("id") Long id) { - return logStoreService.deleteLogStore(id); - } - - @RequestMapping(path = "/milog/store/log/process", method = "get") - public Result> storeLogProcess(@RequestParam("type") String type, @RequestParam("value") String value) { - return logProcessService.getStoreLogProcess(type, value); - } - - /** - * All collection information less than progress Ration progress - * - * @param progressRation progress 0.80 - * @return - */ - @RequestMapping(path = "/milog/col/process/imperfect", method = "get") - public Result> getColProcessImperfect(@RequestParam("progressRation") Double progressRation) { - return logProcessService.getColProcessImperfect(progressRation); - } - - @RequestMapping(path = "/milog/store/ips", method = "get") - public Result>> getStoreIps(@RequestParam("storeId") Long storeId) { - return logStoreService.getStoreIps(storeId); - } - - /** - * ********* milogLogtail ********** - */ - @RequestMapping(path = "/milog/tail/new") - public Result newLogTail(@RequestParam("param") LogTailParam param) { - return logTailService.newMilogLogTail(param); - } - - @RequestMapping(path = "/milog/tail/getbyid", method = "get") - public Result getLogTailById(@RequestParam("id") Long id) { - return logTailService.getMilogLogtailById(id); - } - - @RequestMapping(path = "/milog/tail/getbyids", method = "get") - public Result> getLogTailByIds(@RequestParam("ids") List ids) { - return logTailService.getMilogLogtailByIds(ids); - } - - @RequestMapping(path = "/milog/tail/getbyStoreId", method = "get") - public Result> getTailByStoreId(@RequestParam("storeId") Long storeId) { - return logTailService.getTailByStoreId(storeId); - } - - @RequestMapping(path = "/milog/tail/getbypage", method = "get") - public Result> getLogTailByPage(@RequestParam("storeId") Long storeId, - @RequestParam("page") int page, - @RequestParam("pageSize") int pageSize) throws IOException { - return logTailService.getMilogLogBypage(storeId, page, pageSize); - } - - @RequestMapping(path = "/milog/tail/getcntbystoreid", method = "get") - public Result> getLogTailCountByStoreId(@RequestParam("storeId") Long storeId) { - return logTailService.getLogTailCountByStoreId(storeId); - } - - @RequestMapping(path = "/milog/tail/update") - public Result updateLogTail(LogTailParam param) { - return logTailService.updateMilogLogTail(param); - } - - @RequestMapping(path = "/milog/tail/delete", method = "get") - public Result deleteLogTail(@RequestParam(value = "id") Long id) { - return logTailService.deleteLogTail(id); - } - - @RequestMapping(path = "/milog/tail/gettailname", method = "get") - public Result> getTailNames(@RequestParam("id") Long id, - @RequestParam("appType") Integer appType, - @RequestParam("tail") String tail) { - return logTailService.getTailNamesBystoreId(tail, appType, id); - } - - /** - * Collection rate dictionary (need to be migrated to the dictionary interface later) - * - * @return - */ - @RequestMapping(path = "/milog/tail/tailrate", method = "get") - public Result>> tailRateLimie() { - return logTailService.tailRatelimit(); - } - - /** - * ********* other ********** - */ - - @RequestMapping(path = "/milog/tail/getapp", method = "get") - public Result> getAppInfoByName(@RequestParam(value = "appName") String appName, - @RequestParam(value = "type") Integer type) { - return logTailService.getAppInfoByName(appName, type); - } - - /** - * Query all applications connected to the store - * - * @param storeId - * @return - */ - @RequestMapping(path = "/milog/tail/app/store/id", method = "get") - public Result> queryAppByStoreId(@RequestParam(value = "storeId") Long storeId) { - return logTailService.queryAppByStoreId(storeId); - } - - /** - * Get all environment information of project configuration based on app ID - * - * @param milogAppId - * @return - */ - @RequestMapping(path = "/milog/project/env/appId", method = "get") - public Result> getEnInfosByAppId(@RequestParam(value = "milogAppId") Long milogAppId, - @RequestParam(value = "deployWay") Integer deployWay, - @RequestParam(value = "machineRoom") String machineRoom) { - return logTailService.getEnInfosByAppId(milogAppId, deployWay,machineRoom); - } - - /** - * front page - */ - @RequestMapping(path = "/milog/index/access", method = "get") - public Result> getAccess() { - return heralogHomePageService.milogAccess(); - } - - @RequestMapping(path = "/milog/index/getunaccessapp", method = "get") - public Result> getUnAccessApp() { - return heralogHomePageService.unAccessAppList(); - } - - /** - * Get the store list under the space - * - * @param spaceId - * @return - */ - @RequestMapping(path = "/milog/index/getspacetree", method = "get") - public Result> getMilogSpaceTree(@RequestParam(value = "spaceId") Long spaceId) { - return heralogHomePageService.getMilogSpaceTree(spaceId); - } - - @RequestMapping(path = "/milog/index/fastaccesslogpattern", method = "get") - public Result>> fastAccessMiloglogPattern() { - return heralogHomePageService.getMiloglogAccessPattern(); - } - - // Quick access - @RequestMapping(path = "/milog/index/fastaccess") - public Result fastAccess() { - return null; - } - - - @RequestMapping(path = "/milog/app/type/tail/storeId", method = "get") - public Result> queryAppTailByStoreId(@RequestParam(value = "storeId") Long storeId) { - return logTailService.queryAppTailByStoreId(storeId); - } - - - @RequestMapping(path = "/milog/logstore/region/nameEn", method = "get") - public Result> queryLogStoreByRegionEn(@RequestParam(value = "nameEn") String nameEn) { - return logTailService.queryLogStoreByRegionEn(nameEn); - } - - /** - * Test parsing rules - * - * @param mlogParseParam - * @return - */ - @RequestMapping(path = "/milog/logtail/parse/test") - public Result parseScriptTest(@RequestParam(value = "param") MlogParseParam mlogParseParam) { - return logTailService.parseScriptTest(mlogParseParam); - } - - /** - * Parse the index content according to the example - * - * @param mlogParseParam - * @return - */ - @RequestMapping(path = "/milog/logtail/parse/example") - public Result parseExample(@RequestParam(value = "param") MlogParseParam mlogParseParam) { - return logTailService.parseExample(mlogParseParam); - } - - /** - * Quick query by application - * - * @param milogAppId - * @return - */ - @RequestMapping(path = "/milog/logtail/tail/quick/app", method = "get") - public Result> quickQueryByApp(@RequestParam(value = "milogAppId") Long milogAppId) { - return logTailService.quickQueryByApp(milogAppId); - } - - /** - * Query the space and store of the current application - * - * @param appId - * @param platFormCode - * @return - */ - @RequestMapping(path = "/milog/store/app", method = "get") - public Result queryAppStore(@RequestParam(value = "appId") Long appId, - @RequestParam(value = "platFormCode") Integer platFormCode) { - return logTailService.queryAppStore(appId, platFormCode); - } - - /** - * Query the es index list under the department where the currently created user is located - * - * @param regionCode - * @param logTypeCode - * @return - */ - @RequestMapping(path = "/milog/store/es/index", method = "get") - public Result>> queryDeptExIndexList(@RequestParam(value = "regionCode") String regionCode, - @RequestParam(value = "logTypeCode") Integer logTypeCode) { - return logStoreService.queryDeptExIndexList(regionCode, logTypeCode); - } - - - @RequestMapping(path = "/log/store/config/redistribute", method = "get") - public Result redistributeStoreConfig(@RequestParam(value = "storeId") Long storeId) { - return logStoreService.redistributeStoreConfig(storeId); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogDictionaryController.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogDictionaryController.java deleted file mode 100644 index e2f4770dc..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogDictionaryController.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.controller; - -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.bo.MilogDictionaryParam; -import com.xiaomi.mone.log.manager.model.dto.DictionaryDTO; -import com.xiaomi.mone.log.manager.service.impl.MilogDictionaryServiceImpl; -import com.xiaomi.youpin.docean.anno.Controller; -import com.xiaomi.youpin.docean.anno.RequestMapping; - -import javax.annotation.Resource; -import java.util.List; -import java.util.Map; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/26 15:29 - */ -@Controller -public class MilogDictionaryController { - - @Resource - private MilogDictionaryServiceImpl milogDictionaryService; - - @RequestMapping(path = "/milog/dictionary/list", method = "post") - public Result>>> queryDictionaryList(MilogDictionaryParam codes) { - return milogDictionaryService.queryDictionaryList(codes); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogLoginController.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogLoginController.java deleted file mode 100644 index 96b0e7ad5..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogLoginController.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.controller; - -import cn.hutool.core.bean.BeanUtil; -import com.xiaomi.mone.log.common.Config; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.common.context.MoneUserContext; -import com.xiaomi.mone.log.manager.model.vo.MilogUserVo; -import com.xiaomi.mone.log.manager.user.MoneUser; -import com.xiaomi.youpin.docean.anno.Controller; -import com.xiaomi.youpin.docean.anno.RequestMapping; -import com.xiaomi.youpin.docean.mvc.MvcContext; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/3 14:40 - */ -@Controller -public class MilogLoginController { - - - /** - * Obtain user information - only after login - * - * @return - */ - @RequestMapping(path = "/", method = "get") - public Result queryUserInfo() { - MoneUser currentUser = MoneUserContext.getCurrentUser(); - MilogUserVo milogUserVo = new MilogUserVo(); - if (null != currentUser) { - BeanUtil.copyProperties(currentUser, milogUserVo, true); - } - return Result.success(milogUserVo); - } - - /** - * User exit cleanup information - * - * @param mvcContext - * @return - */ - @RequestMapping(path = "/milog/user/logout", method = "get") - public Result logOut(MvcContext mvcContext) { - mvcContext.session().invalidate(); - MoneUserContext.clear(); - return Result.success(Config.ins().get("tpc_logout_url", "")); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogMiddlewareConfigController.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogMiddlewareConfigController.java deleted file mode 100644 index 4f71830e4..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogMiddlewareConfigController.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.controller; - -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.bo.MiddlewareAddParam; -import com.xiaomi.mone.log.manager.model.bo.MiddlewareQueryParam; -import com.xiaomi.mone.log.manager.model.bo.MiddlewareUpdateParam; -import com.xiaomi.mone.log.manager.model.page.PageInfo; -import com.xiaomi.mone.log.manager.model.pojo.MilogMiddlewareConfig; -import com.xiaomi.mone.log.manager.service.impl.MilogMiddlewareConfigServiceImpl; -import com.xiaomi.youpin.docean.anno.Controller; -import com.xiaomi.youpin.docean.anno.RequestMapping; -import com.xiaomi.youpin.docean.anno.RequestParam; - -import javax.annotation.Resource; -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description Middleware configuration controller - * @date 2021/9/22 11:41 - */ -@Controller -@Deprecated -public class MilogMiddlewareConfigController { - - @Resource - private MilogMiddlewareConfigServiceImpl milogMiddlewareConfigService; - - @RequestMapping(path = "/milog/middleware/config/page") - public Result> queryMiddlewareConfigPage(MiddlewareQueryParam middlewareQueryParam) { - PageInfo pageInfo = milogMiddlewareConfigService.queryMiddlewareConfigPage(middlewareQueryParam); - return Result.success(pageInfo); - } - - @RequestMapping(path = "/milog/middleware/config/add") - public Result addMiddlewareConfig(MiddlewareAddParam ddParam) { - return milogMiddlewareConfigService.addMiddlewareConfig(ddParam); - } - - @RequestMapping(path = "/milog/middleware/config/detail/id", method = "get") - public Result queryMiddlewareConfigById(@RequestParam(value = "id") Long id) { - return milogMiddlewareConfigService.queryMiddlewareConfigById(id); - } - - @RequestMapping(path = "/milog/middleware/config/update") - public Result updateMiddlewareConfig(MiddlewareUpdateParam updateParam) { - return milogMiddlewareConfigService.updateMiddlewareConfig(updateParam); - } - - @RequestMapping(path = "/milog/middleware/config/delete", method = "get") - public Result deleteMiddlewareConfig(@RequestParam(value = "id") Long id) { - return milogMiddlewareConfigService.deleteMiddlewareConfig(id); - } - - @RequestMapping(path = "/milog/middleware/config/list", method = "get") - public Result> queryMiddlewareConfigList() { - List milogMiddlewareConfigs = milogMiddlewareConfigService.queryMiddlewareConfigList(); - return Result.success(milogMiddlewareConfigs); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogStatisticController.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogStatisticController.java deleted file mode 100644 index c30376875..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogStatisticController.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.xiaomi.mone.log.manager.controller; - -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.StatisticsQuery; -import com.xiaomi.mone.log.manager.model.dto.EsStatisticResult; -import com.xiaomi.mone.log.manager.model.dto.EsStatisticsKeyWord; -import com.xiaomi.mone.log.manager.model.vo.LogQuery; -import com.xiaomi.mone.log.manager.service.StatisticsService; -import com.xiaomi.mone.log.manager.service.impl.EsDataServiceImpl; -import com.xiaomi.youpin.docean.anno.Controller; -import com.xiaomi.youpin.docean.anno.RequestMapping; -import com.xiaomi.youpin.docean.anno.RequestParam; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Resource; -import java.io.IOException; -import java.util.List; -import java.util.Map; - -@Controller -public class MilogStatisticController { - - private static final Logger LOGGER = LoggerFactory.getLogger(MilogStatisticController.class); - - @Resource - private StatisticsService statisticsService; - - @Resource - private EsDataServiceImpl esDataService; - - @RequestMapping(path = "/milog/statistic/es") - public Result statisticEs(@RequestParam("param") LogQuery param) throws Exception { - return esDataService.EsStatistic(param); - } - - @RequestMapping(path = "/log/queryTailStatisticsByHour") - public Result> queryTailStatisticsByHour(StatisticsQuery statisticsQuery) throws IOException { - return statisticsService.queryTailStatisticsByHour(statisticsQuery); - } - - @RequestMapping(path = "/log/queryStoreTopByDay") - public Result> queryStoreTopTailStatisticsByDay(StatisticsQuery statisticsQuery) - throws IOException { - return statisticsService.queryStoreTopTailStatisticsByDay(statisticsQuery); - } - - @RequestMapping(path = "/log/querySpaceTopStore") - public Result> querySpaceTopStore(StatisticsQuery statisticsQuery) throws IOException { - return statisticsService.querySpaceTopStoreByDay(statisticsQuery); - } - - @RequestMapping(path = "/log/store/index/field/ration") - public Result> queryEsStatisticsRation(LogQuery param) { - return statisticsService.queryEsStatisticsRation(param); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogStoreSpaceAuthController.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogStoreSpaceAuthController.java deleted file mode 100644 index 13d50a892..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogStoreSpaceAuthController.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.controller; - -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.bo.StoreSpaceAuth; -import com.xiaomi.mone.log.manager.service.impl.MilogStoreSpaceAuthServiceImpl; -import com.xiaomi.youpin.docean.anno.Controller; -import com.xiaomi.youpin.docean.anno.RequestMapping; - -import javax.annotation.Resource; - -/** - * @author wtt - * @version 1.0 - * @description store authorized - * @date 2022/7/14 16:12 - */ -@Controller -public class MilogStoreSpaceAuthController { - - @Resource - private MilogStoreSpaceAuthServiceImpl milogStoreSpaceAuthService; - - @RequestMapping(path = "/milog/store/space/auth", method = "POST") - public Result storeSpaceAuth(StoreSpaceAuth storeSpaceAuth) { - return Result.success(milogStoreSpaceAuthService.storeSpaceAuth(storeSpaceAuth)); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogStreamController.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogStreamController.java deleted file mode 100644 index 81565a9a5..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogStreamController.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.controller; - -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.service.impl.MilogStreamServiceImpl; -import com.xiaomi.youpin.docean.anno.Controller; -import com.xiaomi.youpin.docean.anno.RequestMapping; -import com.xiaomi.youpin.docean.anno.RequestParam; - -import javax.annotation.Resource; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/27 17:24 - */ -@Controller -public class MilogStreamController { - - @Resource - private MilogStreamServiceImpl milogStreamService; - - @RequestMapping(path = "/milog/execute/sql/test", method = "get") - public Result executeSql(@RequestParam(value = "sql") String sql) { - milogStreamService.executeSql(sql); - return Result.success(); - } - - /** - * Configuration is sent to stream - * - * @return - */ - @RequestMapping(path = "/milog/stream/config/issue", method = "get") - public Result configIssueStream(@RequestParam(value = "ip") String ip) { - return milogStreamService.configIssueStream(ip); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/StreamPartitionController.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/StreamPartitionController.java deleted file mode 100644 index 32513057d..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/StreamPartitionController.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.controller; - -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.Pair; -import com.xiaomi.mone.log.manager.model.bo.SpacePartitionBalance; -import com.xiaomi.mone.log.manager.model.page.PageInfo; -import com.xiaomi.mone.log.manager.model.vo.MachinePartitionParam; -import com.xiaomi.mone.log.manager.model.vo.SpaceIpParam; -import com.xiaomi.mone.log.manager.service.StreamPartitionService; -import com.xiaomi.youpin.docean.anno.Controller; -import com.xiaomi.youpin.docean.anno.RequestMapping; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; -import java.util.List; -import java.util.Objects; - -/** - * @author wtt - * @version 1.0 - * @description Consumption machine configuration, this menu can only be used by administrators - * @date 2023/9/19 14:43 - */ -@Controller -public class StreamPartitionController { - - @Resource - private StreamPartitionService streamPartitionService; - - @RequestMapping(path = "/stream/space/stream/key/list") - public Result>> queryStreamList(MachinePartitionParam partitionParam) { - if (StringUtils.isEmpty(partitionParam.getMachineRoom())) { - return Result.failParam("machineRoom 不能为空"); - } - return Result.success(streamPartitionService.queryStreamList(partitionParam)); - } - - /** - * Query the stream list under unique unique - * - * @param partitionParam - * @return - */ - @RequestMapping(path = "/stream/key/partition/list") - public Result>> queryIpPartitionBalance(MachinePartitionParam partitionParam) { - if (StringUtils.isEmpty(partitionParam.getMachineRoom())) { - return Result.failParam("machineRoom 不能为空"); - } - if (StringUtils.isEmpty(partitionParam.getUniqueKey())) { - return Result.failParam("uniqueKey 不能为空"); - } - return Result.success(streamPartitionService.queryIpPartitionBalance(partitionParam)); - } - - /** - * delete configuration - * - * @param param - * @return - */ - @RequestMapping(path = "/stream/key/space/delete") - public Result delSpaceToIp(SpaceIpParam param) { - if (null == param || StringUtils.isEmpty(param.getUniqueKey()) || Objects.isNull(param.getSpaceId())) { - return Result.failParam("参数是不能为空"); - } - return Result.success(streamPartitionService.delSpaceToIp(param)); - } - - /** - * New - * - * @param param - * @return - */ - @RequestMapping(path = "/stream/key/space/add") - public Result addSpaceToIp(SpaceIpParam param) { - if (null == param || CollectionUtils.isEmpty(param.getSpaceIds()) || - CollectionUtils.isEmpty(param.getUniqueKeys())) { - return Result.failParam("参数是不能为空"); - } - return Result.success(streamPartitionService.addSpaceToIp(param)); - } - - @RequestMapping(path = "/stream/key/space/list") - public Result>> findUnIncludedSpaceList(SpaceIpParam param) { - if (null == param || StringUtils.isEmpty(param.getMachineRoom())) { - return Result.failParam("machineRoom 不能为空"); - } - if (StringUtils.isEmpty(param.getUniqueKey())) { - return Result.failParam("参数uniqueKey是不能为空"); - } - return Result.success(streamPartitionService.findUnIncludedSpaceList(param)); - } - - /** - * space dimension query - * - * @param partitionParam - * @return - */ - @RequestMapping(path = "/stream/space/list") - public Result> querySpacePartitionBalance(MachinePartitionParam partitionParam) { - if (StringUtils.isEmpty(partitionParam.getMachineRoom())) { - return Result.failParam("machineRoom 不能为空"); - } - return Result.success(streamPartitionService.querySpacePartitionBalance(partitionParam)); - } - - @RequestMapping(path = "/stream/space/key/list") - public Result>> queryAllUniqueKeyList(SpaceIpParam param) { - if (null == param || StringUtils.isEmpty(param.getMachineRoom())) { - return Result.failParam("machineRoom 不能为空"); - } - if (Objects.isNull(param.getSpaceId())) { - return Result.failParam("spaceId 不能为空"); - } - return Result.success(streamPartitionService.queryAllUniqueKeyList(param)); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/interceptor/HttpRequestInterceptor.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/interceptor/HttpRequestInterceptor.java deleted file mode 100644 index 3cb140321..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/interceptor/HttpRequestInterceptor.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.controller.interceptor; - -import com.google.common.collect.Sets; -import com.xiaomi.hera.trace.context.TraceIdUtil; -import com.xiaomi.mone.log.common.Config; -import com.xiaomi.mone.log.manager.common.context.MoneUserContext; -import com.xiaomi.mone.log.manager.common.exception.MilogManageException; -import com.xiaomi.mone.log.manager.domain.Tpc; -import com.xiaomi.mone.tpc.login.filter.DoceanReqUserFilter; -import com.xiaomi.mone.tpc.login.util.ConstUtil; -import com.xiaomi.mone.tpc.login.vo.AuthUserVo; -import com.xiaomi.youpin.docean.Ioc; -import com.xiaomi.youpin.docean.anno.Component; -import com.xiaomi.youpin.docean.aop.AopContext; -import com.xiaomi.youpin.docean.aop.EnhanceInterceptor; -import com.xiaomi.youpin.docean.mvc.ContextHolder; -import com.xiaomi.youpin.docean.mvc.MvcContext; -import joptsimple.internal.Strings; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; -import java.lang.reflect.Method; -import java.util.*; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.SYMBOL_COMMA; -import static com.xiaomi.mone.log.manager.common.ManagerConstant.SPACE_PAGE_URL; -import static com.xiaomi.mone.log.manager.common.ManagerConstant.TPC_HOME_URL_HEAD; - -/** - * @Author goodjava@qq.com - * @Date 2021/9/2 15:18 - */ -@Slf4j -@Component -public class HttpRequestInterceptor extends EnhanceInterceptor { - - @Resource - private DoceanReqUserFilter doceanReqUserFilter; - - public HttpRequestInterceptor() { - doceanReqUserFilter = new DoceanReqUserFilter(); - Config config = Config.ins(); - Map map = new HashMap<>(); - map.put(ConstUtil.devMode, config.get("tpc.devMode", "false")); - map.put(ConstUtil.innerAuth, "false"); - map.put(ConstUtil.authTokenUrl, config.get("auth_token_url", "http://127.0.0.1:8098/login/token/parse")); - map.put(ConstUtil.ignoreUrl, "/alert/get"); - map.put(ConstUtil.loginUrl, config.get("tpc_login_url", "")); - map.put(ConstUtil.logoutUrl, config.get("tpc_logout_url", "")); - - doceanReqUserFilter.init(map); - } - - private static final Integer MAX_LENGTH = 3000; - - - private String filterUrls = Config.ins().get("filter_urls", Strings.EMPTY); - - private static final Set TPC_HEADERS_URLS = Sets.newHashSet(); - - private List filterUrlList; - - { - filterUrlList = Arrays.stream(filterUrls.split(SYMBOL_COMMA)).distinct().collect(Collectors.toList()); - TPC_HEADERS_URLS.add(SPACE_PAGE_URL); - } - - @Override - public void before(AopContext aopContext, Method method, Object[] args) { - /** - * User information will not be available in the context - */ - if (filterUrlList.contains(method.getName())) { - return; - } - MvcContext mvcContext = ContextHolder.getContext().get(); - saveUserInfoThreadLocal(mvcContext); - } - - private void saveUserInfoThreadLocal(MvcContext mvcContext) { - AuthUserVo userVo = (AuthUserVo) mvcContext.session().getAttribute(ConstUtil.TPC_USER); - if (null == userVo && !doceanReqUserFilter.doFilter(mvcContext)) { - throw new MilogManageException("please go to login"); - } - userVo = (AuthUserVo) mvcContext.session().getAttribute(ConstUtil.TPC_USER); - Tpc tpc = Ioc.ins().getBean(Tpc.class); - MoneUserContext.setCurrentUser(userVo, tpc.isAdmin(userVo.getAccount(), userVo.getUserType())); - } - - @Override - public Object after(AopContext context, Method method, Object res) { - solveResHeaders(); - clearThreadLocal(); - return super.after(context, method, res); - } - - private void solveResHeaders() { - MvcContext mvcContext = ContextHolder.getContext().get(); - mvcContext.getResHeaders().put("traceId", TraceIdUtil.traceId() == null ? StringUtils.EMPTY : TraceIdUtil.traceId()); - if (TPC_HEADERS_URLS.contains(mvcContext.getPath())) { - mvcContext.getResHeaders().put(TPC_HOME_URL_HEAD, Config.ins().get(TPC_HOME_URL_HEAD, "https://127.0.0.1")); - } - } - - @Override - public void exception(AopContext context, Method method, Throwable ex) { - log.error("data exception,", ex); - clearThreadLocal(); - super.exception(context, method, ex); - } - - private void clearThreadLocal() { - MoneUserContext.clear(); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/LogStreamBalanceConfigDao.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/LogStreamBalanceConfigDao.java deleted file mode 100644 index f53aa1270..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/LogStreamBalanceConfigDao.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.dao; - -import com.xiaomi.mone.log.manager.model.pojo.LogStreamBalanceConfig; -import com.xiaomi.youpin.docean.anno.Service; -import org.nutz.dao.Cnd; -import org.nutz.dao.impl.NutDao; - -import javax.annotation.Resource; -import java.util.List; - -import static com.xiaomi.mone.log.common.Constant.EQUAL_OPERATE; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/10/16 15:48 - */ -@Service -public class LogStreamBalanceConfigDao { - - @Resource - private NutDao dao; - - public List queryConfigEnabled() { - return dao.query(LogStreamBalanceConfig.class, Cnd.where("status", EQUAL_OPERATE, 0)); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogAppMiddlewareRelDao.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogAppMiddlewareRelDao.java deleted file mode 100644 index 8e84a4b8d..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogAppMiddlewareRelDao.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.dao; - -import com.xiaomi.mone.log.manager.model.dto.MilogAppConfigTailDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogAppMiddlewareRel; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.nutz.dao.Chain; -import org.nutz.dao.Cnd; -import org.nutz.dao.Sqls; -import org.nutz.dao.entity.Record; -import org.nutz.dao.impl.NutDao; -import org.nutz.dao.sql.Sql; - -import javax.annotation.Resource; -import java.util.LinkedList; -import java.util.List; - -import static com.xiaomi.mone.log.common.Constant.EQUAL_OPERATE; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/23 11:15 - */ -@Service -@Slf4j -public class MilogAppMiddlewareRelDao { - - @Resource - private NutDao dao; - - public void insertUpdate(MilogAppMiddlewareRel middlewareRel) { - Cnd cnd = Cnd.where("milog_app_id", EQUAL_OPERATE, middlewareRel.getMilogAppId()) - .and("middleware_id", EQUAL_OPERATE, middlewareRel.getMiddlewareId()) - .and("tail_id", EQUAL_OPERATE, middlewareRel.getTailId()); - List middlewareRels = dao.query(MilogAppMiddlewareRel.class, cnd); - if (CollectionUtils.isEmpty(middlewareRels)) { - dao.insert(middlewareRel); - } else { - MilogAppMiddlewareRel rel = middlewareRels.get(middlewareRels.size() - 1); - if (null != rel.getConfig().getBatchSendSize() && null == middlewareRel.getConfig().getBatchSendSize()) { - middlewareRel.getConfig().setBatchSendSize(rel.getConfig().getBatchSendSize()); - } - middlewareRel.setId(rel.getId()); - dao.update(middlewareRel); - } - } - - public List queryByCondition(Long milogAppId, Long middlewareId, Long tailId) { - Cnd cnd = Cnd.NEW(); - if (null != milogAppId) { - cnd.and("milog_app_id", EQUAL_OPERATE, milogAppId); - } - if (null != middlewareId) { - cnd.and("middleware_id", EQUAL_OPERATE, middlewareId); - } - if (null != tailId) { - cnd.and("tail_id", EQUAL_OPERATE, tailId); - } - List middlewareRels = dao.query(MilogAppMiddlewareRel.class, cnd); - return middlewareRels; - } - - public boolean updateTopicRelMqConfig(Long id, MilogAppMiddlewareRel.Config config) { - int ret = dao.update(MilogAppMiddlewareRel.class, Chain.make("config", config), Cnd.where("id", "=", id)); - if (ret == 1) { - return true; - } - return false; - } - - public void deleteRel(Long milogAppId, Long tailId) { - List milogAppMiddlewareRels = queryByCondition(milogAppId, null, tailId); - if (CollectionUtils.isNotEmpty(milogAppMiddlewareRels)) { - milogAppMiddlewareRels.forEach(middlewareRel -> dao.delete(MilogAppMiddlewareRel.class, middlewareRel.getId())); - } - } - - public List queryByAMilogAppId(Long milogAppId) { - Sql sql = Sqls.queryEntity("SELECT\n" + - "\tmr.middleware_id AS middlewareId,\n" + - "\tmc.type AS type,\n" + - "\tmc.alias AS middlewareName,\n" + - "\tml.id AS tailId,\n" + - "\tml.tail AS tailName,\n" + - "\tml.creator AS tailCreator,\n" + - "\tml.ctime AS tailCreateTime,\n" + - "\tml.utime AS tailUpdateTime,\n" + - "\tml.updater AS tailUpdater,\n" + - "\tmr.config AS mqConfig \n" + - "FROM\n" + - "\tmilog_app_middleware_rel mr\n" + - "\tLEFT JOIN milog_middleware_config mc ON mr.middleware_id = mc.id\n" + - "\tLEFT JOIN milog_logstail ml ON mr.tail_id = ml.id \n" + - "WHERE\n" + - "\tmr.milog_app_id = @milogAppId"); - sql.params().set("milogAppId", milogAppId); - sql.setEntity(dao.getEntity(MilogAppConfigTailDTO.ConfigTailDTO.class)); - dao.execute(sql); - return sql.getList(MilogAppConfigTailDTO.ConfigTailDTO.class); - } - - public void update(MilogAppMiddlewareRel milogAppMiddlewareRel) { - dao.update(milogAppMiddlewareRel); - } - - public void delete(Long id) { - dao.delete(MilogAppMiddlewareRel.class, id); - } - - public List getAppRelByLimit(int offset, int rows) { - String sqlString = String.format("select * from milog_app_middleware_rel limit %d,%d", offset, rows); - Sql sql = Sqls.queryEntity(sqlString); - sql.setEntity(dao.getEntity(MilogAppMiddlewareRel.class)); - dao.execute(sql); - return sql.getList(MilogAppMiddlewareRel.class); - } - - public Integer queryCountByTopicName(String topicName) { - Sql sql = Sqls.queryRecord("SELECT count(1) as count FROM `milog_app_middleware_rel` where config like @topicName "); - sql.params().set("topicName", "%" + topicName + "%"); - LinkedList records = (LinkedList) dao.execute(sql).getResult(); - int access = records.get(0).getInt("count"); - return access; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogAppTopicRelDao.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogAppTopicRelDao.java deleted file mode 100644 index 33782ea44..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogAppTopicRelDao.java +++ /dev/null @@ -1,302 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.dao; - -import org.apache.ozhera.app.api.response.AppBaseInfo; -import org.apache.ozhera.app.api.service.HeraAppService; -import com.xiaomi.mone.log.api.enums.ProjectTypeEnum; -import com.xiaomi.mone.log.manager.model.pojo.MilogAppTopicRelDO; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.plugin.dubbo.anno.Reference; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.nutz.dao.Cnd; -import org.nutz.dao.Condition; -import org.nutz.dao.Sqls; -import org.nutz.dao.impl.NutDao; -import org.nutz.dao.pager.Pager; -import org.nutz.dao.sql.Sql; - -import javax.annotation.Resource; -import java.util.List; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.EQUAL_OPERATE; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/8 19:33 - */ -@Service -@Slf4j -public class MilogAppTopicRelDao { - - @Resource - private NutDao dao; - - @Reference(interfaceClass = HeraAppService.class, group = "$dubbo.env.group", check = false) - private HeraAppService heraAppService; - - public MilogAppTopicRelDO insert(MilogAppTopicRelDO milogAppTopicRel) { - return dao.insert(milogAppTopicRel); - } - - public List queryAppInfo(Long appId, String appName, Long tenantId, String source) { - return queryAppInfo(appId, appName, tenantId, source, null); - } - - public List queryAppInfo(Long appId, String appName, Long tenantId, String source, Integer appType) { - Cnd cnd = Cnd.where("app_id", EQUAL_OPERATE, appId); - if (StringUtils.isNotEmpty(appName)) { - cnd.andEX("app_name", EQUAL_OPERATE, appName); - } - if (null != tenantId) { - cnd.andEX("tenant_id", EQUAL_OPERATE, tenantId); - } - if (null != appType) { - cnd.andEX("type", EQUAL_OPERATE, appType); - } - if (StringUtils.isNotEmpty(source)) { - cnd.andEX("source", EQUAL_OPERATE, source); - } - return dao.query(MilogAppTopicRelDO.class, cnd); - } - - public void deleteAppTopicRelDb(Long appId, String appName, Long tenantId, String source) { - List milogAppTopicRels = queryAppInfo(appId, appName, tenantId, source); - if (CollectionUtils.isNotEmpty(milogAppTopicRels)) { - dao.delete(MilogAppTopicRelDO.class, milogAppTopicRels.get(0).getId()); - } - } - - public MilogAppTopicRelDO queryByAppId(Long appId, String source) { - Cnd cnd = Cnd.where("app_id", EQUAL_OPERATE, appId); - if (StringUtils.isNotEmpty(source)) { - cnd.andEX("source", EQUAL_OPERATE, source); - } - List milogAppTopicRels = dao.query(MilogAppTopicRelDO.class, cnd); - if (CollectionUtils.isNotEmpty(milogAppTopicRels)) { - return milogAppTopicRels.get(milogAppTopicRels.size() - 1); - } - return null; - } - - public MilogAppTopicRelDO queryByAppId(Long appId, Integer type) { - Cnd cnd = Cnd.where("app_id", EQUAL_OPERATE, appId); - if (null != type) { - cnd.andEX("type", EQUAL_OPERATE, type); - } - List milogAppTopicRels = dao.query(MilogAppTopicRelDO.class, cnd); - if (CollectionUtils.isNotEmpty(milogAppTopicRels)) { - return milogAppTopicRels.get(milogAppTopicRels.size() - 1); - } - return null; - } - - public List queryByMilogAppIds(List milogAppIds) { - Cnd cnd = Cnd.where("id", "IN", milogAppIds); - return dao.query(MilogAppTopicRelDO.class, cnd); - } - - public List queryAppInfoByName(String appName, Integer type) { - Cnd cnd = Cnd.where("type", EQUAL_OPERATE, type); - if (StringUtils.isNotEmpty(appName)) { - cnd.and("app_name", "like", "%" + appName + "%"); - } - List ret = dao.query(MilogAppTopicRelDO.class, cnd); - return ret; - } - - public List queryAppInfoByName(String appName, String source, Integer type) { - Cnd cnd = Cnd.NEW(); - if (StringUtils.isNotEmpty(appName)) { - Cnd.where("app_name", "like", "%" + appName + "%"); - } - if (null != type) { - cnd.and("type", EQUAL_OPERATE, type); - } else { - cnd.and("source", EQUAL_OPERATE, source); - } - List ret = dao.query(MilogAppTopicRelDO.class, cnd); - return ret; - } - - public List queryAppInfoByNameSource(String appName, String source, Integer type) { - Cnd cnd = Cnd.NEW(); - if (StringUtils.isNotEmpty(appName)) { - Cnd.where("app_name", "like", "%" + appName + "%"); - } - if (StringUtils.isNotBlank(source)) { - cnd.and("source", EQUAL_OPERATE, source); - } - if (null != type) { - cnd.and("type", EQUAL_OPERATE, type); - } - List ret = dao.query(MilogAppTopicRelDO.class, cnd); - return ret; - } - - public List queryAppTopicList(Condition cnd, Pager pager) { - if (null == pager) { - return dao.query(MilogAppTopicRelDO.class, cnd); - } - return dao.query(MilogAppTopicRelDO.class, cnd, pager); - } - - public void updateTopicName(Long appId, String topicNameSimple) { - } - - public MilogAppTopicRelDO queryById(Long id) { - AppBaseInfo appBaseInfo = heraAppService.queryById(id); - MilogAppTopicRelDO milogAppTopicRelDO = new MilogAppTopicRelDO(); - milogAppTopicRelDO.setId(appBaseInfo.getId().longValue()); - milogAppTopicRelDO.setAppId(Long.valueOf(appBaseInfo.getBindId())); - milogAppTopicRelDO.setAppName(appBaseInfo.getAppName()); - if (CollectionUtils.isNotEmpty(appBaseInfo.getTreeIds())) { - milogAppTopicRelDO.setTreeIds(appBaseInfo.getTreeIds().stream().map(Integer::longValue).collect(Collectors.toList())); - } - milogAppTopicRelDO.setSource(appBaseInfo.getPlatformName()); - milogAppTopicRelDO.setType(appBaseInfo.getPlatformType()); - return milogAppTopicRelDO; - } - - public void updateAppTopicRelMqConfigById(Long id, String existTopic) { - } - - public Integer queryAppTopicPageCount(Condition cnd) { - return dao.count(MilogAppTopicRelDO.class, cnd); - } - - public void delTopicRecordAll() { - dao.delete(MilogAppTopicRelDO.class); - } - - public List queryAllMilogAppList() { - return queryAppTopicList(Cnd.NEW(), null); - } - - public List queryAllAccessMilogAppList() { - Sql sql = Sqls.queryEntity("SELECT\n" + - "\tmt.* \n" + - "FROM\n" + - "\tmilog_app_topic_rel mt\n" + - "\tLEFT JOIN ( SELECT DISTINCT milog_app_id FROM milog_logstail ) ml ON mt.id = ml.milog_app_id \n" + - "WHERE ml.milog_app_id IS NOT NULL"); - sql.setEntity(dao.getEntity(MilogAppTopicRelDO.class)); - dao.execute(sql); - return sql.getList(MilogAppTopicRelDO.class); - } - - - public List queryAppsExistInMachine(String ip) { - Sql sql = Sqls.queryEntity("SELECT mt.* FROM milog_app_topic_rel mt LEFT JOIN( SELECT DISTINCT ml.app_id,ms.source FROM milog_logstail ml LEFT JOIN milog_space ms ON ml.space_id = ms.id WHERE JSON_CONTAINS( ml.ips, JSON_ARRAY( @ip)) ) ma ON mt.app_id = ma.app_id and mt.source = ma.source WHERE ma.app_id IS NOT NULL"); - sql.params().set("ip", ip); - sql.setEntity(dao.getEntity(MilogAppTopicRelDO.class)); - dao.execute(sql); - return sql.getList(MilogAppTopicRelDO.class); - } - - public MilogAppTopicRelDO queryByIpAndAppid(Long appId, String ip) { - Sql sql = Sqls.queryEntity("SELECT mt.* FROM milog_app_topic_rel mt LEFT JOIN( SELECT DISTINCT ml.app_id,ms.source FROM milog_logstail ml LEFT JOIN milog_space ms ON ml.space_id = ms.id WHERE JSON_CONTAINS( ml.ips, JSON_ARRAY( @ip)) ) ma ON mt.app_id = ma.app_id and mt.source = ma.source WHERE ma.app_id =@appId"); - sql.params().set("ip", ip); - sql.params().set("appId", appId); - sql.setEntity(dao.getEntity(MilogAppTopicRelDO.class)); - dao.execute(sql); - return sql.getList(MilogAppTopicRelDO.class).get(0); - } - - public int getAppCount() { - return dao.count(MilogAppTopicRelDO.class); - } - - public List getMioneAppAll(String source) { - Cnd cnd = Cnd.where("source", EQUAL_OPERATE, source).and("type", EQUAL_OPERATE, ProjectTypeEnum.MIONE_TYPE.getCode()); - return dao.query(MilogAppTopicRelDO.class, cnd); - } - - public MilogAppTopicRelDO queryIsExists(MilogAppTopicRelDO milogAppTopicRel) { - Cnd cnd = Cnd.where("app_id", EQUAL_OPERATE, milogAppTopicRel.getAppId()) - .and("source", EQUAL_OPERATE, milogAppTopicRel.getSource()) - .and("type", EQUAL_OPERATE, milogAppTopicRel.getType()); - List appTopicRels = dao.query(MilogAppTopicRelDO.class, cnd); - if (CollectionUtils.isNotEmpty(appTopicRels)) { - return appTopicRels.get(appTopicRels.size() - 1); - } - return null; - } - - public void insertNoExists(MilogAppTopicRelDO milogAppTopicRel) { - MilogAppTopicRelDO appTopicRel = queryIsExists(milogAppTopicRel); - if (null == appTopicRel) { - dao.insert(milogAppTopicRel); - } else { - if ((null != milogAppTopicRel.getNodeIPs() && milogAppTopicRel.getNodeIPs().size() > 0) && - (null == appTopicRel.getNodeIPs() || appTopicRel.getNodeIPs().size() == 0 || milogAppTopicRel.getNodeIPs().size() != appTopicRel.getNodeIPs().size())) { - appTopicRel.setNodeIPs(milogAppTopicRel.getNodeIPs()); - dao.update(appTopicRel); - } - } - - } - - public void update(MilogAppTopicRelDO milogAppTopicRel) { - dao.update(milogAppTopicRel); - } - - public MilogAppTopicRelDO queryByIamTreeId(Long iamTreeId) { - Cnd cnd = Cnd.where("iam_tree_id", EQUAL_OPERATE, iamTreeId); - List appTopicRels = dao.query(MilogAppTopicRelDO.class, cnd); - if (CollectionUtils.isNotEmpty(appTopicRels)) { - return appTopicRels.get(appTopicRels.size() - 1); - } - return null; - } - - public void deleteByAppIds(Long appId, String source) { - MilogAppTopicRelDO appTopicRel = queryByAppId(appId, source); - if (null != appTopicRel) { - dao.delete(MilogAppTopicRelDO.class, appTopicRel.getId()); - } - } - - public List queryAppInfoByChinaCondition(Long appId, Long iamTreeId) { - Cnd cnd = Cnd.where("app_id", EQUAL_OPERATE, appId); - if (null != iamTreeId) { - cnd.andEX("iam_tree_id", EQUAL_OPERATE, iamTreeId); - } - List appTopicRels = dao.query(MilogAppTopicRelDO.class, cnd); - return appTopicRels; - } - - public List queryByIds(List ids) { - Cnd cnd = Cnd.where("id", "IN", ids); - return dao.query(MilogAppTopicRelDO.class, cnd); - } - - public List queryAppbyNameSource(String appName, String source) { - Cnd cnd = Cnd.where("app_name", EQUAL_OPERATE, appName) - .and("source", EQUAL_OPERATE, source); - return dao.query(MilogAppTopicRelDO.class, cnd); - } - - public List queryAppbyName(String appName) { - Cnd cnd = Cnd.where("app_name", EQUAL_OPERATE, appName); - return dao.query(MilogAppTopicRelDO.class, cnd); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogLockDao.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogLockDao.java deleted file mode 100644 index 9371bd8f2..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogLockDao.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.dao; - -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.nutz.dao.Chain; -import org.nutz.dao.Cnd; -import org.nutz.dao.impl.NutDao; - -import javax.annotation.Resource; - -/** - * @author zhangping17 - * @date 2021-10-25 - */ -@Service -@Slf4j -public class MilogLockDao { - - @Resource - private NutDao dao; - - public int updateLock(String code, int status){ - Cnd cnd = Cnd.where("code", "=", code); - if (status == 0){ - cnd = cnd.and("status", "=", 1); - } else { - cnd = cnd.and("status", "=", 0); - } - int res = dao.update("milog_lock", Chain.make("status", status).add("utime", System.currentTimeMillis()), cnd); - return res; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogLogProcessDao.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogLogProcessDao.java deleted file mode 100644 index 3a9435a58..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogLogProcessDao.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.dao; - -import com.xiaomi.mone.log.manager.model.pojo.MilogLogProcessDO; -import com.xiaomi.youpin.docean.anno.Service; -import org.nutz.dao.Cnd; -import org.nutz.dao.impl.NutDao; - -import javax.annotation.Resource; -import java.util.List; - -@Service -public class MilogLogProcessDao { - @Resource - private NutDao dao; - - /** - * Get the log details monitored by the agent - * - * @param agentId - * @return - */ - public List queryByAgentId(long agentId) { - return dao.query(MilogLogProcessDO.class, Cnd.where("agent_id", "IN", agentId)); - } - - public List queryByIp(String ip) { - return dao.query(MilogLogProcessDO.class, Cnd.where("ip", "=", ip)); - } - - /** - * Save in batches - * - * @param doList - */ - public List bulkSave(List doList) { - return dao.insert(doList); - } - - /** - * delete log collection progress - * - * @param tailId - */ - public void deleteByTailId(Long tailId) { - dao.clear(MilogLogProcessDO.class, Cnd.where("tailId", "=", tailId)); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogLogTailDao.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogLogTailDao.java deleted file mode 100644 index a2d0d6261..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogLogTailDao.java +++ /dev/null @@ -1,396 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.dao; - -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.api.enums.MachineTypeEnum; -import com.xiaomi.mone.log.api.enums.ProjectTypeEnum; -import com.xiaomi.mone.log.api.model.meta.FilterDefine; -import com.xiaomi.mone.log.manager.common.context.MoneUserContext; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.nutz.dao.Chain; -import org.nutz.dao.Cnd; -import org.nutz.dao.FieldFilter; -import org.nutz.dao.Sqls; -import org.nutz.dao.entity.Record; -import org.nutz.dao.impl.NutDao; -import org.nutz.dao.pager.Pager; -import org.nutz.dao.sql.Sql; - -import javax.annotation.Resource; -import java.sql.Connection; -import java.sql.ResultSet; -import java.time.Instant; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.EQUAL_OPERATE; - -@Service -@Slf4j -public class MilogLogTailDao { - @Resource - private NutDao dao; - - public MilogLogTailDo add(MilogLogTailDo mt) { - return dao.insert(mt); - } - - public boolean update(MilogLogTailDo mt) { - return 1 == dao.update(mt); - } - - public boolean updateMilogLogtail(Long id, Integer parseType, String parseScript, String logPath, String valueList, Long appId, Long envId, String envName, - String tail, List ips, List confs, - Long milogAppId, String appName, String logSplitExpress, Integer deployWay, String deploySpace, String firstLineReg) { -// int ret = dao.update(MilogLogTailDo.class, Chain.make("space_id", spaceId).add("app_id", appId).add("parse_type",parseType).add("log_path",logPath).add("value_list",valueList), Cnd.where("id", "=", id)); - Chain chain = Chain.make("parse_type", parseType).add("parse_script", parseScript) - .add("log_path", logPath).add("value_list", valueList) - .add("env_id", envId).add("env_name", envName) - .add("tail", tail).add("ips", ips) - .add("filter", confs).add("milog_app_id", milogAppId).add("app_name", appName) - .add("deploy_way", deployWay) - .add("deploy_space", deploySpace) - .add("first_line_reg", firstLineReg); - if (null != appId) { - chain.add("app_id", appId); - } - chain.add("log_split_express", logSplitExpress); - chain.add("utime", Instant.now().toEpochMilli()); - chain.add("updater", MoneUserContext.getCurrentUser().getUser()); - int ret = dao.update(MilogLogTailDo.class, chain, Cnd.where("id", EQUAL_OPERATE, id)); - if (ret != 1) { - return false; - } else { - return true; - } - } - - public boolean deleteMilogLogtail(Long id) { - int ret = dao.clear(MilogLogTailDo.class, Cnd.where("id", EQUAL_OPERATE, id)); - if (ret != 1) { - return false; - } else { - return true; - } - } - - public List queryTailByAppName(String appName) { - Cnd cnd = Cnd.NEW(); - if (!StringUtils.isNotEmpty(appName)) { - cnd = cnd.and("app_name", EQUAL_OPERATE, appName); - } - return dao.query(MilogLogTailDo.class, cnd); - } - - public List getMilogLogtail(List ids) { - return dao.query(MilogLogTailDo.class, Cnd.where("id", "in", ids)); - } - - public List getMilogLogtailByStoreId(Long storeId) { - return dao.query(MilogLogTailDo.class, Cnd.where("store_id", EQUAL_OPERATE, storeId)); - } - - public List getMilogLogtailByStoreIds(List ids) { - return dao.query(MilogLogTailDo.class, Cnd.where("store_id", "in", ids).orderBy("ctime", "desc")); - } - - public List getMilogLogtailByPage(Long storeId, int page, int pagesize) { - Cnd cnd = Cnd.where("store_id", EQUAL_OPERATE, storeId); - return dao.query(MilogLogTailDo.class, cnd.orderBy("ctime", "desc"), new Pager(page, pagesize)); - } - - public int getTailCount(Long storeId) { - Cnd cnd = Cnd.where("store_id", EQUAL_OPERATE, storeId); - return dao.count(MilogLogTailDo.class, cnd); - } - - public List getMilogLogtailByPage(String appName, int page, int pagesize) { - Cnd cnd = Cnd.NEW(); - if (!StringUtils.isEmpty(appName)) { - cnd = cnd.and("app_name", "like", "%" + appName + "%"); - } - List ret = dao.query(MilogLogTailDo.class, cnd.orderBy("ctime", "desc"), new Pager(page, pagesize)); - return ret; - } - - public Map> getMilogLogtailByAppId(List appIds) { - Map> ret = new HashMap<>(); - for (Long appId : appIds) { - List logtails = dao.query(MilogLogTailDo.class, Cnd.where("app_id", EQUAL_OPERATE, appId)); - if (logtails != null && logtails.size() > 0) { - ret.put(appId, logtails); - } - } - return ret; - } - - - public List queryByAppIds(Long[] appIds) { - return dao.query(MilogLogTailDo.class, Cnd.where("app_id", "in", appIds)); - } - - public List queryByAppIdAgentIp(Long milogAppId, String agentIp) { - Sql sql = Sqls.queryEntity("SELECT * FROM milog_logstail WHERE milog_app_id = @milogAppId AND JSON_CONTAINS( ips, JSON_ARRAY( @ip))"); - sql.params().set("milogAppId", milogAppId); - sql.params().set("ip", agentIp); - sql.setEntity(dao.getEntity(MilogLogTailDo.class)); - dao.execute(sql); - return sql.getList(MilogLogTailDo.class); - } - - public List queryByAppAndEnv(Long appId, Long envId) { - return dao.query(MilogLogTailDo.class, Cnd.where("app_id", EQUAL_OPERATE, appId).and("env_id", EQUAL_OPERATE, envId)); - } - - public List queryByMilogAppAndEnv(Long milogAppId, Long envId) { - return dao.query(MilogLogTailDo.class, Cnd.where("milog_app_id", EQUAL_OPERATE, milogAppId).and("env_id", EQUAL_OPERATE, envId)); - } - - public List queryByMilogAppAndEnvId(Long milogAppId, Long envId) { - Cnd cnd = Cnd.where("milog_app_id", EQUAL_OPERATE, milogAppId); - if (null != envId) { - cnd.and("env_id", EQUAL_OPERATE, envId); - } - return dao.query(MilogLogTailDo.class, cnd); - } - - public List queryByMilogAppAndEnvK8s(Long milogAppId, Long envId, - Integer deploy_way) { - return dao.query(MilogLogTailDo.class, - Cnd.where("milog_app_id", EQUAL_OPERATE, milogAppId) - .and("env_id", EQUAL_OPERATE, envId) - .and("deploy_way", EQUAL_OPERATE, deploy_way)); - } - - public MilogLogTailDo queryById(Long id) { - if (null == id) { - return null; - } - return dao.fetch(MilogLogTailDo.class, id); - } - - public List queryAllIds() { - String idKey = "id"; - List records = dao.query("milog_logstail", null, null, idKey); - if (CollectionUtils.isNotEmpty(records)) { - return records.stream().map(record -> record.getLong(idKey)).collect(Collectors.toList()); - } - return Lists.newArrayList(); - } - - public List getLogTailByMilogAppId(Long milogAppId) { - return dao.query(MilogLogTailDo.class, Cnd.where("milog_app_id", EQUAL_OPERATE, milogAppId)); - } - - public MilogLogTailDo getTailByName(String tailName, Integer appType) { - return dao.fetch(MilogLogTailDo.class, Cnd.where("tail", EQUAL_OPERATE, tailName) - .and("app_type", EQUAL_OPERATE, appType)); - } - - public List queryByAppType(Integer appType) { - return dao.query(MilogLogTailDo.class, Cnd.where("app_type", EQUAL_OPERATE, appType)); - } - - public List getMilogLogtailByIdsAndName(List ids, String tail, Integer appType) { - Cnd cnd = Cnd.where("store_id", "in", ids); - if (!StringUtils.isEmpty(tail)) { - cnd.and("tail", "like", "%" + tail + "%"); - } - if (null != appType) { - cnd.and("app_type", EQUAL_OPERATE, appType); - } - return dao.query(MilogLogTailDo.class, cnd); - } - - /** - * Get the logtail of ip configuration - * - * @param ip - * @return - */ - public List queryByIp(String ip) { - Sql sql = Sqls.queryEntity("SELECT * FROM milog_logstail WHERE JSON_CONTAINS(`ips`, JSON_ARRAY( @ip )) and collection_ready = 1"); - sql.params().set("ip", ip); - sql.setEntity(dao.getEntity(MilogLogTailDo.class)); - dao.execute(sql); - return sql.getList(MilogLogTailDo.class); - } - - public int appCount() { - Sql sql = Sqls.queryRecord("SELECT count(DISTINCT app_id) as access from milog_logstail"); - LinkedList records = (LinkedList) dao.execute(sql).getResult(); - int access = records.get(0).getInt("access"); - return access; - } - - public List getAll(String source) { - Sql sql = Sqls.queryEntity("SELECT ml.* FROM milog_logstail ml LEFT JOIN milog_app_topic_rel mt ON ml.app_id = mt.app_id WHERE mt.source = @source"); - sql.setEntity(dao.getEntity(MilogLogTailDo.class)); - sql.params().set("source", source); - dao.execute(sql); - return sql.getList(MilogLogTailDo.class); - } - - public String queryTailNameByAppIdAndName(String appId, String ip) { - Sql sql = Sqls.queryString("SELECT tail FROM milog_logstail WHERE app_id=@appId and JSON_CONTAINS(ips, '[\"" + ip + "\"]');"); - sql.params().set("appId", appId); - dao.execute(sql); - return sql.getString(); - } - - public List queryAppIdByStoreId(Long storeId) { - return dao.query(MilogLogTailDo.class, Cnd.where("store_id", EQUAL_OPERATE, storeId)); - } - - public List getLogTailByLimit(int offset, int rows) { - String sqlString = String.format("select * from milog_logstail limit %d,%d", offset, rows); - Sql sql = Sqls.queryEntity(sqlString); - sql.setEntity(dao.getEntity(MilogLogTailDo.class)); - dao.execute(sql); - return sql.getList(MilogLogTailDo.class); - } - - public List queryStoreIdByRegionNameEN(String nameEn) { - Sql sql = Sqls.queryEntity("SELECT * FROM `milog_logstail` where JSON_CONTAINS(motor_rooms, JSON_OBJECT(\"nameEn\", @nameEn))"); - sql.params().set("nameEn", nameEn); - sql.setEntity(dao.getEntity(MilogLogTailDo.class)); - dao.execute(sql); - return sql.getList(MilogLogTailDo.class); - } - - public List queryTailNameExists(String tailName, String machineRoom, Long spaceId) { - Sql sql = Sqls.queryEntity("SELECT la.* FROM milog_logstail la LEFT JOIN milog_logstore lt ON la.store_id = lt.id WHERE la.tail = @tailName AND lt.machine_room = @machineRoom and la.space_id = @spaceId"); - sql.params().set("tailName", tailName); - sql.params().set("machineRoom", machineRoom); - sql.params().set("spaceId", spaceId); - sql.setEntity(dao.getEntity(MilogLogTailDo.class)); - dao.execute(sql); - return sql.getList(MilogLogTailDo.class); - } - - public List queryTailsByStoreId(Long storeId) { - return dao.query(MilogLogTailDo.class, Cnd.where("store_id", EQUAL_OPERATE, storeId)); - } - - public MilogLogTailDo queryTailByMilogAppIdAndIps(Long milogAppId, List ips) { - List MilogLogTailDoList = dao.query(MilogLogTailDo.class, Cnd.where("milog_app_id", EQUAL_OPERATE, milogAppId)); - if (CollectionUtils.isNotEmpty(MilogLogTailDoList)) { - return MilogLogTailDoList.stream().filter(MilogLogTailDo -> CollectionUtils.isEqualCollection(MilogLogTailDo.getIps(), ips)).findFirst().get(); - } - return null; - } - - public Long queryMinTailCountStoreId(Long spaceId, List storeIdList) { - Sql sql = Sqls.create("SELECT t.mKey FROM( SELECT count( id) AS mValue, store_id AS mKey FROM milog_logstail WHERE space_id = @spaceId AND store_id IN (" + storeIdList.stream().map(String::valueOf).collect(Collectors.joining(",")) + ") GROUP BY store_id ) t ORDER BY t.mValue LIMIT 1"); - sql.params().set("spaceId", spaceId); - sql.setCallback((Connection conn, ResultSet rs, Sql sql1) -> { - Long minCountStoreId = null; - while (rs.next()) { - minCountStoreId = rs.getLong(1); - } - return minCountStoreId; - }); - Long object = dao.execute(sql).getObject(Long.class); - return object; - } - - public MilogLogTailDo queryServerlessTailByFuncId(Long spaceId, Long storeId, Long milogAppId, Long funcId) { - List logTailDos = dao.query(MilogLogTailDo.class, Cnd.where("space_id", EQUAL_OPERATE, spaceId) -// .and("store_id", EQUAL_OPERATE, storeId) - .and("milog_app_id", EQUAL_OPERATE, milogAppId) - .and("env_id", EQUAL_OPERATE, funcId)); - if (CollectionUtils.isNotEmpty(logTailDos)) { - return logTailDos.get(logTailDos.size() - 1); - } - return null; - } - - public List queryTailsByAppAndStores(Long appId, List storeIds) { - return dao.query(MilogLogTailDo.class, Cnd.where("app_id", EQUAL_OPERATE, appId) - .and("app_type", EQUAL_OPERATE, ProjectTypeEnum.MIONE_TYPE) - .and("store_id", "in", storeIds)); - } - - - /** - * Query the logTail for all included matrix type apps - */ - public List queryTailsByStores(List storeIds) { - return dao.query(MilogLogTailDo.class, Cnd.where("store_id", "in", storeIds)); - } - - public List queryAllAppId() { - Sql sql = Sqls.create("SELECT DISTINCT milog_app_id FROM milog_logstail"); - sql.setCallback((Connection conn, ResultSet rs, Sql sql1) -> { - List appBaseIds = Lists.newArrayList(); - while (rs.next()) { - appBaseIds.add(rs.getInt(1)); - } - return appBaseIds; - }); - return dao.execute(sql).getList(Integer.class); - } - - public List queryTailWithAppIdNull() { - return dao.query(MilogLogTailDo.class, Cnd.where("app_id", "is", null)); - } - - public List queryByAppId(Long appId) { - return dao.query(MilogLogTailDo.class, Cnd.where("app_id", "=", appId)); - } - - public List queryByAppId(Long appId, Long milogAppId) { - return dao.query(MilogLogTailDo.class, Cnd.where("app_id", EQUAL_OPERATE, appId) - .and("milog_app_id", EQUAL_OPERATE, milogAppId)); - } - - public List queryByIds(List tailIds) { - return dao.query(MilogLogTailDo.class, Cnd.where("id", "in", tailIds)); - } - - public void updateIps(MilogLogTailDo milogLogtailDo) { - dao.update(milogLogtailDo, FieldFilter.create(MilogLogTailDo.class, "ips|utime|updater")); - } - - public List queryAppTypeTailByAppId(Long serviceId, Integer typeCode) { - return dao.query(MilogLogTailDo.class, Cnd.where("app_id", EQUAL_OPERATE, serviceId) - .and("app_type", EQUAL_OPERATE, typeCode) - .and("machine_type", EQUAL_OPERATE, MachineTypeEnum.PHYSICAL_MACHINE.getType())); - } - - public List queryByNames(Long storeId, List nameList) { - Cnd cnd = Cnd.where("store_id", EQUAL_OPERATE, storeId).and("tail", "in", nameList); - return dao.query(MilogLogTailDo.class, cnd); - } - - public List queryByCondition(Long spaceId, Long storeId, String tailName, Long heraAppId, Long envId, - String logPath) { - return dao.query(MilogLogTailDo.class, Cnd.where("space_id", EQUAL_OPERATE, spaceId) - .and("store_id", EQUAL_OPERATE, storeId) - .and("tail", EQUAL_OPERATE, tailName) - .and("milog_app_id", EQUAL_OPERATE, heraAppId) - .and("env_id", EQUAL_OPERATE, envId) - .and("log_path", EQUAL_OPERATE, logPath)); - } -} \ No newline at end of file diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogLogstoreDao.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogLogstoreDao.java deleted file mode 100644 index faea463bf..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogLogstoreDao.java +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.dao; - -import cn.hutool.core.bean.BeanUtil; -import com.xiaomi.mone.log.api.enums.LogTypeEnum; -import com.xiaomi.mone.log.manager.domain.ClusterIndexVO; -import com.xiaomi.mone.log.manager.model.bo.MilogLogstoreBo; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogSpaceDO; -import com.xiaomi.youpin.docean.anno.Service; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.nutz.dao.Chain; -import org.nutz.dao.Cnd; -import org.nutz.dao.Sqls; -import org.nutz.dao.impl.NutDao; -import org.nutz.dao.pager.Pager; -import org.nutz.dao.sql.Sql; - -import javax.annotation.Resource; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.EQUAL_OPERATE; -import static com.xiaomi.mone.log.common.Constant.LIKE_OPERATE; - -@Service -public class MilogLogstoreDao { - - @Resource - private NutDao dao; - - public boolean newMilogLogStore(MilogLogStoreDO ml) { - MilogLogStoreDO ret = dao.insert(ml); - if (ret == null) { - return false; - } else { - return true; - } - } - - public MilogLogStoreDO insert(MilogLogStoreDO logStoreDO) { - return dao.insert(logStoreDO); - } - - public boolean updateMilogLogStore(Long id, Long spaceId, String logstoreName, - Integer storePeriod, Integer shardCnd, String keyList, - Integer logType, String machineRoom, String updater) { - Long utime = System.currentTimeMillis(); - int ret = dao.update(MilogLogStoreDO.class, Chain.make("space_id", spaceId) - .add("logstoreName", logstoreName).add("store_period", storePeriod) - .add("shard_cnt", shardCnd).add("key_list", keyList) - .add("machine_room", machineRoom).add("updater", updater) - .add("log_type", logType).add("utime", utime), Cnd.where("id", EQUAL_OPERATE, id)); - if (ret != 1) { - return false; - } else { - return true; - } - } - - public boolean updateMilogLogStore(MilogLogStoreDO ml) { - return 1 == dao.update(ml) ? true : false; - } - - - public boolean deleteMilogSpace(Long id) { - int ret = dao.clear(MilogLogStoreDO.class, Cnd.where("id", EQUAL_OPERATE, id)); - if (ret != 1) { - return false; - } else { - return true; - } - } - - public List getMilogLogstore(List ids) { - return dao.query(MilogLogStoreDO.class, Cnd.where("id", "in", ids)); - } - - public List getMilogLogstoreBySpaceId(Long spaceId) { - return dao.query(MilogLogStoreDO.class, Cnd.where("space_id", EQUAL_OPERATE, spaceId).orderBy("ctime", "desc")); - } - - public List getMilogLogstoreBySpaceId(List spaceIdList) { - return dao.query(MilogLogStoreDO.class, Cnd.where("space_id", "in", spaceIdList).orderBy("ctime", "desc")); - } - - public List getMilogLogstoreBySpaceIdRegion(Long spaceId, String region) { - return dao.query(MilogLogStoreDO.class, Cnd.where("space_id", EQUAL_OPERATE, spaceId).and("machine_room", EQUAL_OPERATE, region)); - } - - public Map getMilogLogstoreByPage(String logsotreName, Long spaceId, int page, int pagesize) { - Cnd cnd = Cnd.where("space_id", "=", spaceId); - if (StringUtils.isNotEmpty(logsotreName)) { - cnd = cnd.and("logstoreName", LIKE_OPERATE, "%" + logsotreName + "%"); - } - Map result = new HashMap<>(); - List ret = dao.query(MilogLogStoreDO.class, cnd.orderBy("utime", "desc"), new Pager(page, pagesize)); - if (CollectionUtils.isNotEmpty(ret)) { - List logstoreBos = ret.stream().map(milogLogstoreDO -> { - MilogLogstoreBo milogLogstoreBo = new MilogLogstoreBo(); - BeanUtil.copyProperties(milogLogstoreDO, milogLogstoreBo); - milogLogstoreBo.setLogTypeText(LogTypeEnum.queryNameByType(milogLogstoreDO.getLogType())); - return milogLogstoreBo; - }).collect(Collectors.toList()); - result.put("list", logstoreBos); - } - - result.put("total", dao.count(MilogLogStoreDO.class, cnd)); - result.put("page", page); - result.put("pageSize", pagesize); - return result; - } - - public Map getAllMilogLogstore(String source) { - List ret = dao.query(MilogLogStoreDO.class, null); - ret = ret.stream().filter(milogLogstoreDO -> { - MilogSpaceDO milogSpace = dao.fetch(MilogSpaceDO.class, milogLogstoreDO.getSpaceId()); - if (milogSpace.getSource().equals(source)) { - return true; - } - return false; - }).collect(Collectors.toList()); - Map result = new HashMap<>(); - result.put("list", ret); - return result; - } - - public List getAllMilogLogstore() { - return dao.query(MilogLogStoreDO.class, null); - } - - public MilogLogStoreDO getByName(String logstoreName) { - List queryByName = dao.query(MilogLogStoreDO.class, Cnd.where("logstoreName", EQUAL_OPERATE, logstoreName)); - return queryByName == null || queryByName.isEmpty() ? null : queryByName.get(0); - } - - public MilogLogStoreDO queryById(Long id) { - return dao.fetch(MilogLogStoreDO.class, id); - } - - public String getAppIndex(Long appId, String ip) { - String sqlStr = "SELECT s.es_index FROM milog_logstore s, milog_logstail t WHERE t.store_id = s.id and t.app_id = @appId AND JSON_CONTAINS(t.ips, '\"" + ip + "\"')"; - Sql sql = Sqls.create(sqlStr); - sql.params().set("appId", appId); - sql.setCallback(Sqls.callback.str()); - dao.execute(sql); - return sql.getString(); - } - - public boolean verifyExistByName(String logstoreName, Long id, Long spaceId) { - Cnd cnd = Cnd.where("logstoreName", EQUAL_OPERATE, logstoreName); - if (null != id) { - cnd.andNot("id", EQUAL_OPERATE, id); - } - if (null != spaceId) { - cnd.and("space_id", EQUAL_OPERATE, spaceId); - } - int count = dao.count(MilogLogStoreDO.class, cnd); - return count > 0; - } - - public List queryStoreBySpaceStoreNameVague(Long spaceId, String storeNamePrefix, String machineRoom) { - Cnd cnd = Cnd.where("machine_room", EQUAL_OPERATE, machineRoom); - cnd.and("space_id", EQUAL_OPERATE, spaceId); - if (StringUtils.isNotBlank(storeNamePrefix)) { - cnd.and("logstoreName", LIKE_OPERATE, String.format("%s%s", storeNamePrefix.trim(), "%")); - } - cnd.orderBy("ctime", "desc"); - return dao.query(MilogLogStoreDO.class, cnd); - } - - public MilogLogStoreDO queryStoreBySpaceStoreName(Long spaceId, String storeName, String machineRoom) { - Cnd cnd = Cnd.where("machine_room", EQUAL_OPERATE, machineRoom); - cnd.and("space_id", EQUAL_OPERATE, spaceId); - if (StringUtils.isNotBlank(storeName)) { - cnd.and("logstoreName", EQUAL_OPERATE, storeName); - } - List logStoreDOS = dao.query(MilogLogStoreDO.class, cnd); - if (CollectionUtils.isNotEmpty(logStoreDOS)) { - return logStoreDOS.get(logStoreDOS.size() - 1); - } - return null; - } - - public List queryByIds(List storeIds) { - Cnd cnd = Cnd.where("id", "in", storeIds); - return dao.query(MilogLogStoreDO.class, cnd); - } - - public List queryAll() { - return dao.query(MilogLogStoreDO.class, Cnd.NEW()); - } - - public List queryByEsInfo(String regionEn, Long esClusterId) { - return dao.query(MilogLogStoreDO.class, Cnd.where("machine_room", EQUAL_OPERATE, regionEn).and("es_cluster_id", EQUAL_OPERATE, esClusterId)); - } - - public List queryByLogType(Integer type) { - return dao.query(MilogLogStoreDO.class, Cnd.where("log_type", EQUAL_OPERATE, type)); - } - - public void deleteById(Long id) { - dao.delete(MilogLogStoreDO.class, id); - } - - public List queryBySpaceIdNamed(Long spaceId, String logStoreName) { - Cnd cnd = Cnd.where("space_id", EQUAL_OPERATE, spaceId); - cnd.and("logstoreName", EQUAL_OPERATE, logStoreName); - return dao.query(MilogLogStoreDO.class, cnd); - } - - public List queryClusterIndexByAppId(Long appId) { - Sql sql = Sqls.queryEntity("SELECT DISTINCT se.es_cluster_id as clusterId, se.es_index as indexName FROM milog_logstail sl LEFT JOIN milog_logstore se ON sl.store_id = se.id WHERE sl.app_id = @appId;"); - sql.params().set("appId", appId); - sql.setEntity(dao.getEntity(ClusterIndexVO.class)); - dao.execute(sql); - return sql.getList(ClusterIndexVO.class); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogMachineDao.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogMachineDao.java deleted file mode 100644 index 57fb532f2..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogMachineDao.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.dao; - -import com.xiaomi.mone.log.api.enums.AppTypeEnum; -import com.xiaomi.mone.log.common.Constant; -import com.xiaomi.mone.log.manager.model.bo.MachineQueryParam; -import com.xiaomi.mone.log.manager.model.pojo.MiLogMachine; -import com.xiaomi.mone.log.manager.model.pojo.MilogSpaceDO; -import com.xiaomi.youpin.docean.anno.Service; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.nutz.dao.Cnd; -import org.nutz.dao.impl.NutDao; -import org.nutz.dao.impl.sql.NutSql; -import org.nutz.dao.pager.Pager; - -import javax.annotation.Resource; -import java.util.List; - -import static com.xiaomi.mone.log.common.Constant.EQUAL_OPERATE; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/16 11:45 - */ -@Service -public class MilogMachineDao { - - @Resource - private NutDao dao; - - public List queryMachineByType(AppTypeEnum appTypeEnum) { - return dao.query(MiLogMachine.class, Cnd.where("type", EQUAL_OPERATE, appTypeEnum.getType())); - } - - public MiLogMachine insert(MiLogMachine miLogMachine) { - return dao.insert(miLogMachine); - } - - public void deleteMachineInfo(Long id) { - dao.delete(MiLogMachine.class, id); - } - - public List queryMachinePage(MachineQueryParam param) { - return dao.query(MiLogMachine.class, assembleParam(param).orderBy("utime", "desc"), - new Pager(param.getPageNum(), param.getPageSize())); - } - - public Integer queryMachinePageCount(MachineQueryParam param) { - - return dao.count(MilogSpaceDO.class, assembleParam(param)); - } - - private Cnd assembleParam(MachineQueryParam param) { - Cnd cnd = Cnd.NEW(); - if (null != param.getId()) { - cnd.where().and("space_name", EQUAL_OPERATE, param.getId()); - } - if (null != param.getType()) { - cnd.where().and("type", EQUAL_OPERATE, param.getType()); - } - if (StringUtils.isNotEmpty(param.getIp())) { - cnd.where().and("ip", EQUAL_OPERATE, param.getIp()); - } - return cnd; - } - - public MiLogMachine queryById(Long id) { - Cnd cnd = Cnd.where("id", Constant.EQUAL_OPERATE, id); - List miLogMachines = dao.query(MiLogMachine.class, cnd); - if (CollectionUtils.isNotEmpty(miLogMachines)) { - return miLogMachines.get(miLogMachines.size() - 1); - } - return null; - } - - public void executeSql(String sql) { - dao.execute(new NutSql(sql)); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogMiddlewareConfigDao.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogMiddlewareConfigDao.java deleted file mode 100644 index 11cba5d14..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogMiddlewareConfigDao.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.dao; - -import com.xiaomi.mone.log.api.enums.MachineRegionEnum; -import com.xiaomi.mone.log.api.enums.MiddlewareEnum; -import com.xiaomi.mone.log.common.Constant; -import com.xiaomi.mone.log.manager.model.pojo.MilogMiddlewareConfig; -import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionServiceFactory; -import com.xiaomi.mone.log.manager.service.extension.resource.ResourceExtensionServiceFactory; -import com.xiaomi.youpin.docean.anno.Service; -import org.apache.commons.collections.CollectionUtils; -import org.nutz.dao.Cnd; -import org.nutz.dao.Condition; -import org.nutz.dao.impl.NutDao; -import org.nutz.dao.pager.Pager; - -import javax.annotation.Resource; -import java.util.List; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.EQUAL_OPERATE; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/22 12:03 - */ -@Service -public class MilogMiddlewareConfigDao { - - @Resource - private NutDao dao; - - public List queryMiddlewareConfigByCondition(Condition cnd, Pager pager) { - if (null == pager) { - return dao.query(MilogMiddlewareConfig.class, cnd); - } - return dao.query(MilogMiddlewareConfig.class, cnd, pager); - } - - public Integer queryMiddlewareConfigCountByCondition(Condition cnd) { - return dao.count(MilogMiddlewareConfig.class, cnd); - } - - public void addMiddlewareConfig(MilogMiddlewareConfig addParamToAddEntity) { - dao.insert(addParamToAddEntity); - } - - public MilogMiddlewareConfig queryById(Long id) { - return dao.fetch(MilogMiddlewareConfig.class, id); - } - - public void updateMiddlewareConfig(MilogMiddlewareConfig middlewareConfig) { - dao.update(middlewareConfig); - } - - public void deleteMiddlewareConfig(Long id) { - dao.delete(MilogMiddlewareConfig.class, id); - } - - public List queryCurrentMontorRoomMQ(String montorRoomEn) { - List middlewareConfigs = dao.query(MilogMiddlewareConfig.class, Cnd.where("region_en", EQUAL_OPERATE, montorRoomEn.trim())); - if (CollectionUtils.isNotEmpty(middlewareConfigs)) { - return middlewareConfigs.stream().filter(middlewareConfig -> CommonExtensionServiceFactory.getCommonExtensionService().middlewareEnumValid(middlewareConfig.getType())).collect(Collectors.toList()); - } - return null; - } - - public MilogMiddlewareConfig queryDefaultMiddlewareConfig() { - List defaultConfigList = dao.query(MilogMiddlewareConfig.class, Cnd.where("region_en", EQUAL_OPERATE, MachineRegionEnum.CN_MACHINE.getEn()).and("is_default", EQUAL_OPERATE, 1)); - if (CollectionUtils.isNotEmpty(defaultConfigList)) { - return defaultConfigList.stream().filter(middlewareConfig -> CommonExtensionServiceFactory.getCommonExtensionService().middlewareEnumValid(middlewareConfig.getType())).findFirst().orElse(null); - } - return null; - } - - public MilogMiddlewareConfig queryCurrentEnvNacos(String enName) { - Cnd cnd = Cnd.where("type", EQUAL_OPERATE, MiddlewareEnum.NCOS.getCode()).and("region_en", EQUAL_OPERATE, enName); - List milogMiddlewareConfigs = dao.query(MilogMiddlewareConfig.class, cnd); - if (CollectionUtils.isNotEmpty(milogMiddlewareConfigs)) { - return milogMiddlewareConfigs.get(milogMiddlewareConfigs.size() - 1); - } - return null; - } - - public List queryAll() { - return dao.query(MilogMiddlewareConfig.class, null); - } - - public MilogMiddlewareConfig queryDefaultMqMiddlewareConfigMotorRoom(String motorRooman) { - Cnd cnd = Cnd.where("type", "in", ResourceExtensionServiceFactory.getResourceExtensionService().getResourceCode()) - .and("region_en", EQUAL_OPERATE, motorRooman); - List milogMiddlewareConfigs = dao.query(MilogMiddlewareConfig.class, cnd); - if (CollectionUtils.isNotEmpty(milogMiddlewareConfigs) && milogMiddlewareConfigs.size() == 1) { - return milogMiddlewareConfigs.get(milogMiddlewareConfigs.size() - 1); - } - if (CollectionUtils.isNotEmpty(milogMiddlewareConfigs) && milogMiddlewareConfigs.size() > 1) { - return milogMiddlewareConfigs.stream().filter(milogMiddlewareConfig -> Constant.YES.intValue() == milogMiddlewareConfig.getIsDefault().intValue()).findFirst().get(); - } - return null; - } - - public MilogMiddlewareConfig queryNacosRegionByNameServer(String nameServer) { - Cnd cnd = Cnd.where("type", EQUAL_OPERATE, MiddlewareEnum.NCOS.getCode()).and("name_server", EQUAL_OPERATE, nameServer); - List milogMiddlewareConfigs = dao.query(MilogMiddlewareConfig.class, cnd); - if (CollectionUtils.isNotEmpty(milogMiddlewareConfigs)) { - return milogMiddlewareConfigs.get(milogMiddlewareConfigs.size() - 1); - } - return null; - } - - public List queryByResourceCode(Integer resourceCode, String regionCode) { - Cnd cnd = Cnd.where("type", EQUAL_OPERATE, resourceCode); - cnd.and("region_en", EQUAL_OPERATE, regionCode); - cnd.orderBy("utime", "desc"); - return dao.query(MilogMiddlewareConfig.class, cnd); - } - - public List queryByResourceCodes(List resourceCodes, String regionCode) { - Cnd cnd = Cnd.where("type", "in", resourceCodes); - cnd.and("region_en", EQUAL_OPERATE, regionCode); - return dao.query(MilogMiddlewareConfig.class, cnd); - } - - public List selectByAlias(String alias) { - Cnd cnd = Cnd.where("alias", EQUAL_OPERATE, alias); - return dao.query(MilogMiddlewareConfig.class, cnd); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogRegionAvailableZoneDao.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogRegionAvailableZoneDao.java deleted file mode 100644 index e0b1e18a5..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogRegionAvailableZoneDao.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.dao; - -import com.xiaomi.mone.log.manager.model.pojo.MilogRegionAvailableZoneDO; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.nutz.dao.Cnd; -import org.nutz.dao.impl.NutDao; - -import javax.annotation.Resource; -import java.util.List; - -/** - * @author zhangping17 - * @date 2021-10-15 Region，AZ query - */ -@Service -@Slf4j -public class MilogRegionAvailableZoneDao { - - @Resource - private NutDao dao; - - /** - * According to zone English name or region and zone information - * - * @param zoneNameENs - * @return - */ - public List listMilogRegionAvailableZones(List zoneNameENs) { - List milogRegionAvailableZoneDOList = null; - Cnd cnd = null; - if (zoneNameENs != null && !zoneNameENs.isEmpty()) { - cnd = Cnd.where("zone_name_en", "in", zoneNameENs); - } - milogRegionAvailableZoneDOList = dao.query(MilogRegionAvailableZoneDO.class, cnd); - return milogRegionAvailableZoneDOList; - } - - public void insert(MilogRegionAvailableZoneDO milogRegionAvailableZoneDO) { - dao.insert(milogRegionAvailableZoneDO); - } - - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogSpaceDao.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogSpaceDao.java deleted file mode 100644 index 312f525ab..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogSpaceDao.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.dao; - -import com.xiaomi.mone.log.manager.common.context.MoneUserContext; -import com.xiaomi.mone.log.manager.model.convert.MilogSpaceConvert; -import com.xiaomi.mone.log.manager.model.dto.MilogSpaceDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogSpaceDO; -import com.xiaomi.mone.log.manager.user.MoneUser; -import com.xiaomi.youpin.docean.anno.Service; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.nutz.dao.Chain; -import org.nutz.dao.Cnd; -import org.nutz.dao.impl.NutDao; -import org.nutz.dao.pager.Pager; - -import javax.annotation.Resource; -import java.time.Instant; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static com.xiaomi.mone.log.common.Constant.EQUAL_OPERATE; - -@Service -public class MilogSpaceDao { - - @Resource - private NutDao dao; - - /** - * new build - * - * @param ms - * @return - */ - public MilogSpaceDO newMilogSpace(MilogSpaceDO ms) { - MilogSpaceDO ret = dao.insert(ms); - return ret; - } - - public MilogSpaceDO insert(MilogSpaceDO spaceDO) { - return dao.insert(spaceDO); - } - - public MilogSpaceDO queryById(Long id) { - return dao.fetch(MilogSpaceDO.class, id); - } - - public List queryBySpaceName(String spaceName) { - return dao.query(MilogSpaceDO.class, Cnd.where("space_name", EQUAL_OPERATE, spaceName)); - } - - /** - * update - * - * @param id - * @param tenantId - * @param spaceName - * @param description - * @return - */ - public boolean updateMilogSPace(Long id, Long tenantId, String spaceName, String description) { - Chain chain = Chain.make("tenant_id", tenantId).add("space_name", spaceName).add("description", description); - chain.add("utime", Instant.now().toEpochMilli()); - int ret = dao.update(MilogSpaceDO.class, chain, Cnd.where("id", "=", id)); - if (ret != 1) { - return false; - } else { - return true; - } - } - - public boolean update(MilogSpaceDO milogSpace) { - return dao.update(milogSpace) == 1; - } - - /** - * delete - * - * @param id - */ - public boolean deleteMilogSpace(Long id) { - int ret = dao.clear(MilogSpaceDO.class, Cnd.where("id", "=", id)); - if (ret != 1) { - return false; - } else { - return true; - } - } - - /** - * search - * - * @param ids - * @return - */ - public List getSpaceByIdList(List ids) { - return dao.query(MilogSpaceDO.class, Cnd.where("id", "in", ids)); - } - - public List getMilogSpaces(String limitDeptId) { - Cnd cnd = Cnd.NEW(); - MoneUser currentUser = MoneUserContext.getCurrentUser(); - if (!currentUser.getIsAdmin() && StringUtils.isNotEmpty(currentUser.getZone())) { - cnd.and("perm_dept_id", "like", "%" + limitDeptId + "%").or("creator", "=", "system"); - } - cnd.orderBy("ctime", "desc"); - return dao.query(MilogSpaceDO.class, cnd); - } - - public MilogSpaceDO getMilogSpaceById(Long id) { - List milogSpaces = dao.query(MilogSpaceDO.class, Cnd.where("id", "=", id)); - if (CollectionUtils.isNotEmpty(milogSpaces)) { - return milogSpaces.get(milogSpaces.size() - 1); - } - return null; - } - - public boolean verifyExistByName(String spaceName) { - int count = dao.count(MilogSpaceDO.class, Cnd.where("space_name", "=", spaceName)); - return count > 0; - } - - public boolean verifyExistByName(String spaceName, Long id) { - List ret = dao.query(MilogSpaceDO.class, Cnd.where("space_name", "=", spaceName)); - for (int i = 0; i < ret.size(); i++) { - if (!ret.get(i).getId().equals(id)) { - return true; - } - } - return false; - } - - public Map getMilogSpaceByPage(String spaceName, List permSpaceIdList, int page, int pagesize) { - Cnd cnd; - if (StringUtils.isNotEmpty(spaceName)) { - cnd = Cnd.where("space_name", "like", "%" + spaceName + "%"); - } else { - cnd = Cnd.NEW(); - } - MoneUser currentUser = MoneUserContext.getCurrentUser(); - if (!currentUser.getIsAdmin() && StringUtils.isNotEmpty(currentUser.getZone())) { - cnd.and("id", "in", permSpaceIdList).or("creator", "=", "system"); - } - List ret = dao.query(MilogSpaceDO.class, cnd.orderBy("ctime", "desc"), new Pager(page, pagesize)); - List dtoList = MilogSpaceConvert.INSTANCE.fromDOList(ret); - Map result = new HashMap<>(); - result.put("list", dtoList); - result.put("total", dao.count(MilogSpaceDO.class, cnd)); - result.put("page", page); - result.put("pageSize", pagesize); - return result; - } - - /** - * query all - * - * @return - */ - public List getAll() { - return dao.query(MilogSpaceDO.class, null); - } - - public List queryByIds(List spaceIds) { - Cnd cnd = Cnd.where("id", "in", spaceIds); - return dao.query(MilogSpaceDO.class, cnd); - } - - public List queryByName(String name) { - Cnd cnd = Cnd.NEW(); - if (StringUtils.isNotEmpty(name)) { - cnd = Cnd.where("space_name", "like", "%" + name + "%"); - } - return this.dao.query(MilogSpaceDO.class, cnd); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogStoreSpaceAuthDao.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogStoreSpaceAuthDao.java deleted file mode 100644 index 778b323e1..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/dao/MilogStoreSpaceAuthDao.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.dao; - -import com.xiaomi.mone.log.manager.model.pojo.MilogStoreSpaceAuth; -import com.xiaomi.youpin.docean.anno.Service; -import org.apache.commons.collections.CollectionUtils; -import org.nutz.dao.Cnd; -import org.nutz.dao.impl.NutDao; - -import javax.annotation.Resource; -import java.util.List; - -import static com.xiaomi.mone.log.common.Constant.EQUAL_OPERATE; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/7/14 16:13 - */ -@Service -public class MilogStoreSpaceAuthDao { - - @Resource - private NutDao dao; - - public MilogStoreSpaceAuth queryByStoreSpace(Long storeId, Long spaceId) { - Cnd cnd = Cnd.NEW(); - if (null != storeId) { - cnd.where().and("store_id", EQUAL_OPERATE, storeId); - } - if (null != spaceId) { - cnd.where().and("space_id", EQUAL_OPERATE, spaceId); - } - List milogStoreSpaceAuths = dao.query(MilogStoreSpaceAuth.class, cnd); - if (CollectionUtils.isNotEmpty(milogStoreSpaceAuths)) { - return milogStoreSpaceAuths.get(milogStoreSpaceAuths.size() - 1); - } - return null; - } - - public void add(MilogStoreSpaceAuth storeSpaceAuth) { - dao.insert(storeSpaceAuth); - } - - public int update(MilogStoreSpaceAuth milogStoreSpaceAuth) { - return dao.update(milogStoreSpaceAuth); - } - - public List queryStoreIdsBySpaceId(Long spaceId) { - return dao.query(MilogStoreSpaceAuth.class, Cnd.where("space_id", EQUAL_OPERATE, spaceId).orderBy("ctime", "desc")); - } - - public List queryBySpaceId(List spaceIdList) { - return dao.query(MilogStoreSpaceAuth.class, Cnd.where("space_id", "in", spaceIdList).orderBy("ctime", "desc")); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/AnalyseLog.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/AnalyseLog.java deleted file mode 100644 index f445e7dc1..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/AnalyseLog.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.domain; - -import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; -import com.xiaomi.mone.log.manager.domain.analyse.AggrCalcu; -import com.xiaomi.mone.log.manager.mapper.MilogAnalyseGraphMapper; -import com.xiaomi.mone.log.manager.model.bo.CalcuAggrParam; -import com.xiaomi.mone.log.manager.model.dto.LogAnalyseDataDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogAnalyseGraphDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.vo.LogAnalyseDataPreQuery; -import com.xiaomi.mone.log.manager.model.vo.LogAnalyseDataQuery; -import com.xiaomi.mone.log.manager.model.vo.LogQuery; -import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionService; -import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionServiceFactory; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.plugin.es.EsService; -import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.builder.SearchSourceBuilder; - -import javax.annotation.Resource; -import java.io.IOException; -import java.util.concurrent.TimeUnit; - -@Slf4j -@Service -public class AnalyseLog { - @Resource - private MilogLogstoreDao logstoreDao; - - @Resource - private EsCluster esCluster; - - @Resource - private MilogAnalyseGraphMapper graphMapper; - - @Resource - private AggrCalcu aggrCalcu; - - private CommonExtensionService commonExtensionService; - - public void init() { - commonExtensionService = CommonExtensionServiceFactory.getCommonExtensionService(); - } - - - public LogAnalyseDataDTO getData(LogAnalyseDataPreQuery query) throws IOException { - return getData(query.getStoreId(), query.getFieldName(), query.getTypeCode(), query.getGraphParam(), query.getStartTime(), query.getEndTime()); - } - - public LogAnalyseDataDTO getData(LogAnalyseDataQuery query) throws IOException { - MilogAnalyseGraphDO graph = graphMapper.selectById(query.getGraphId()); - return getData(graph.getStoreId(), graph.getFieldName(), graph.getGraphType(), graph.getGraphParam(), query.getStartTime(), query.getEndTime()); - } - - private LogAnalyseDataDTO getData(Long storeId, String fieldName, Integer graphType, String graphParam, Long startTime, Long endTime) throws IOException { - if (storeId == null || fieldName == null) { - return null; - } - MilogLogStoreDO logStore = logstoreDao.queryById(storeId); - EsService esService = esCluster.getEsService(logStore.getEsClusterId()); - String esIndex = commonExtensionService.getSearchIndex(logStore.getId(), logStore.getEsIndex()); - - // Filter criteria - LogQuery logQuery = new LogQuery(); - logQuery.setStoreId(logStore.getId()); - logQuery.setStartTime(startTime); - logQuery.setEndTime(endTime - 1000); - logQuery.setLogstore(logStore.getLogstoreName()); - BoolQueryBuilder boolQueryBuilder = commonExtensionService.commonRangeQuery(logQuery); - - AggregationBuilder aggrs = aggrCalcu.getAggr(new CalcuAggrParam(graphType, graphParam, fieldName, startTime, endTime)); - if (aggrs == null) { - return null; - } - - // Construct the query object - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(boolQueryBuilder); - searchSourceBuilder.size(0); - searchSourceBuilder.aggregation(aggrs); - searchSourceBuilder.timeout(new TimeValue(1, TimeUnit.MINUTES)); - SearchRequest request = new SearchRequest(esIndex); - request.source(searchSourceBuilder); - - // query - SearchResponse response = esService.search(request); - - return aggrCalcu.formatRes(graphType, response); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/ClusterIndexVO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/ClusterIndexVO.java deleted file mode 100644 index 20f70f182..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/ClusterIndexVO.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.domain; - -import lombok.*; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/12/28 20:32 - */ -@Data -@Builder -@EqualsAndHashCode -@NoArgsConstructor -@AllArgsConstructor -public class ClusterIndexVO { - private Long clusterId; - private String indexName; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/Down.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/Down.java deleted file mode 100644 index 1a922a290..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/Down.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.domain; - -import com.xiaomi.youpin.docean.mvc.ContextHolder; -import com.xiaomi.youpin.docean.mvc.MvcContext; -import com.xiaomi.youpin.docean.mvc.MvcRequest; -import com.xiaomi.youpin.docean.mvc.MvcResponse; -import com.xiaomi.youpin.docean.mvc.download.Download; -import io.netty.channel.ChannelHandlerContext; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -public class Down { - public static void down(String path) throws IOException { - ChannelHandlerContext handlerContext = ContextHolder.getContext().get().getHandlerContext(); - ContextHolder context = ContextHolder.getContext(); - MvcContext mvcContext = context.get(); - MvcResponse mvcResponse = new MvcResponse(); - mvcResponse.setCtx(handlerContext); - MvcRequest mvcRequest = new MvcRequest(); - Map param = new HashMap<>(); - param.put("name", path); - mvcRequest.setParams(param); - Download.download(mvcContext, mvcRequest, mvcResponse); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/EsCluster.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/EsCluster.java deleted file mode 100644 index b44cdca0a..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/EsCluster.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.domain; - -import com.xiaomi.mone.log.common.Constant; -import com.xiaomi.mone.log.manager.bootstrap.LogStoragePlugin; -import com.xiaomi.mone.log.manager.common.context.MoneUserContext; -import com.xiaomi.mone.log.manager.mapper.MilogEsClusterMapper; -import com.xiaomi.mone.log.manager.model.pojo.MilogEsClusterDO; -import com.xiaomi.mone.log.manager.service.extension.store.StoreExtensionService; -import com.xiaomi.mone.log.manager.service.extension.store.StoreExtensionServiceFactory; -import com.xiaomi.youpin.docean.Ioc; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.common.StringUtils; -import com.xiaomi.youpin.docean.plugin.es.EsService; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Resource; -import java.util.List; -import java.util.Objects; - -@Service -@Slf4j -public class EsCluster { - @Resource - private MilogEsClusterMapper esClusterMapper; - - @Resource - private LogStoragePlugin logStoragePlugin; - - private StoreExtensionService storeExtensionService; - - public void init() { - storeExtensionService = StoreExtensionServiceFactory.getStoreExtensionService(); - } - - /** - * Get the ES client - * - * @param esClusterId - * @return - */ - public EsService getEsService(Long esClusterId) { - if (esClusterId == null) { - return null; - } - - String beanName = Constant.LOG_STORAGE_SERV_BEAN_PRE + esClusterId; - - if (isBeanInitialized(esClusterId, beanName)) { - return Ioc.ins().getBean(beanName); - } - - return null; - } - - private boolean isBeanInitialized(Long esClusterId, String beanName) { - if (Ioc.ins().containsBean(beanName)) { - return true; - } - - logStoragePlugin.initializeLogStorage(getById(esClusterId)); - return Ioc.ins().containsBean(beanName); - } - - /** - * Get the ES client - * - * @return - */ - public EsService getEsService() { - MilogEsClusterDO curEsCluster = this.getCurEsCluster(); - if (curEsCluster == null) { - return null; - } - if (Ioc.ins().containsBean(Constant.LOG_STORAGE_SERV_BEAN_PRE + curEsCluster.getId())) { - return Ioc.ins().getBean(Constant.LOG_STORAGE_SERV_BEAN_PRE + curEsCluster.getId()); - } else { - return null; - } - } - - /** - * Obtain the ES client corresponding to the current user - * - * @return - */ - public MilogEsClusterDO getCurEsCluster() { - List esClusterList = esClusterMapper.selectByTag(MoneUserContext.getCurrentUser().getZone()); - MilogEsClusterDO cluster = esClusterList == null || esClusterList.isEmpty() ? null : esClusterList.get(0); - log.info("[EsCluster.getCurEsCluster] user is {}, cluster is {}", MoneUserContext.getCurrentUser(), cluster.getName()); - return cluster; - } - - public MilogEsClusterDO getById(Long id) { - return esClusterMapper.selectById(id); - } - - public MilogEsClusterDO getByRegion(String region) { - if (StringUtils.isEmpty(region)) { - return null; - } - MilogEsClusterDO esClusterDO = esClusterMapper.selectByRegion(region); - return esClusterDO; - } - - // Obtain the supported ES clusters in your region - public MilogEsClusterDO getByArea4China(String area) { - if (StringUtils.isEmpty(area)) { - return null; - } - List clusterList = esClusterMapper.selectByArea(area, storeExtensionService.getMangerEsLabel()); - if (clusterList == null || clusterList.isEmpty()) { - return null; - } - if (clusterList.size() > 1) { - String zone = MoneUserContext.getCurrentUser().getZone(); - for (MilogEsClusterDO clusterDO : clusterList) { - if (Objects.equals(zone, clusterDO.getTag())) { - return clusterDO; - } - } - } - return clusterList.get(0); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/EsIndexTemplate.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/EsIndexTemplate.java deleted file mode 100644 index b8b56d645..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/EsIndexTemplate.java +++ /dev/null @@ -1,328 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.domain; - - -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.xiaomi.mone.log.api.enums.LogTypeEnum; -import com.xiaomi.mone.log.common.Es; -import com.xiaomi.mone.log.manager.common.exception.MilogManageException; -import com.xiaomi.mone.log.manager.mapper.MilogEsClusterMapper; -import com.xiaomi.mone.log.manager.mapper.MilogEsIndexMapper; -import com.xiaomi.mone.log.manager.model.dto.EsInfoDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogEsClusterDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogEsIndexDO; -import com.xiaomi.mone.log.manager.model.vo.CreateIndexTemplatePropertyCommand; -import com.xiaomi.mone.log.manager.model.vo.CreateOrUpdateLogStoreCmd; -import com.xiaomi.mone.log.manager.model.vo.UpdateIndexTemplateCommand; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.common.StringUtils; -import com.xiaomi.youpin.docean.plugin.es.EsService; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.elasticsearch.action.admin.indices.alias.Alias; -import org.elasticsearch.client.indices.CreateIndexRequest; -import org.elasticsearch.client.indices.CreateIndexResponse; -import org.elasticsearch.client.indices.IndexTemplatesExistRequest; -import org.elasticsearch.client.indices.PutIndexTemplateRequest; - -import javax.annotation.Resource; -import java.io.IOException; -import java.util.*; - -import static com.xiaomi.mone.log.manager.common.Utils.getRandomNum; - -@Service -@Slf4j -public class EsIndexTemplate { - - @Resource - private EsCluster esCluster; - - @Resource - private MilogEsIndexMapper esIndexMapper; - - @Resource - private MilogEsClusterMapper milogEsClusterMapper; - - /** - * Create an index template - * - * @param command - * @return - * @throws IOException - */ - public boolean createIndexTemplate(CreateOrUpdateLogStoreCmd command) throws IOException { - // Verify that the template exists - if (existTemplate(command.getEsIndex())) { - return false; - } - PutIndexTemplateRequest request = new PutIndexTemplateRequest(command.getEsIndex()); - // Match the name - request.patterns(Arrays.asList(command.getEsIndex() + "*")); - // Set up an alias alias - request.alias(new Alias(command.getEsIndex())); - // Priority - request.order(20); - // Version number - request.version(0); - // Template settings - buildSetting(request, command); - // Template mapping - buildMapping(request, command); - EsService esService = esCluster.getEsService(null); - return esService.createIndexTemplate(request) && createIndex(command.getEsIndex()); - } - - public boolean createIndex(String indexName) throws IOException { - if (existTemplate(indexName) == false) { - return false; - } - CreateIndexRequest request = new CreateIndexRequest(indexName + Es.indexPostfix()); - EsService esService = esCluster.getEsService(null); - CreateIndexResponse response = esService.createIndex(request); - return response.isAcknowledged(); - } - - /** - * Build the setting - * - * @param request - * @param command - */ - public void buildSetting(PutIndexTemplateRequest request, CreateOrUpdateLogStoreCmd command) { -// request.settings(Settings.builder() -// // Number of shards -// .put("index.number_of_shards" , command.getShardCnt()) -// // ILM name -// .put("index.lifecycle.name", command.getStorePeriod() + "Del") -// // Rollback index aliases -//// .put("index.lifecycle.rollover_alias", createIndexTemplateCommand.getIndexTemplateName() + indexTemplate.getIndexPostfix()) -// ); - } - - /** - * Build the setting - * - * @param request - * @param updateIndexTemplateCommand - */ - public void buildSetting(PutIndexTemplateRequest request, UpdateIndexTemplateCommand updateIndexTemplateCommand) { -// request.settings(Settings.builder() -// // Number of shards -// .put("index.number_of_shards" , updateIndexTemplateCommand.getIndexShards()) -// .put("index.number_of_replicas", updateIndexTemplateCommand.getIndexReplicas()) -// // ILM name -// .put("index.lifecycle.name",updateIndexTemplateCommand.getLifecycle()) - // Rollback index aliases -// .put("index.lifecycle.rollover_alias", updateIndexTemplateCommand.getIndexTemplateName() + indexTemplate.getIndexPostfix()) -// ); - } - - /** - * Build mapping - * - * @param request - * @param command - */ - public void buildMapping(PutIndexTemplateRequest request, CreateOrUpdateLogStoreCmd command) { - String[] keyArray = command.getKeyList().split(","); - String[] keyTypeArray = command.getColumnTypeList().split(","); - Map mapping = new HashMap<>(); - Map properties = new HashMap<>(); - Map property; - for (int i = 0; i < keyArray.length; i++) { - property = new HashMap<>(); - property.put("type", keyTypeArray[i]); - properties.put(keyArray[i], property); - } - mapping.put("properties", properties); - request.mapping(mapping); - } - - /** - * Build mapping - * - * @param request - * @param updateIndexTemplateCommand - */ - public void buildMapping(PutIndexTemplateRequest request, UpdateIndexTemplateCommand updateIndexTemplateCommand) { - List propertyList = updateIndexTemplateCommand.getPropertyList(); - if (propertyList == null || propertyList.isEmpty()) { - return; - } - Map mapping = new HashMap<>(); - Map properties = new HashMap<>(); - Map property; - for (CreateIndexTemplatePropertyCommand p : propertyList) { - property = new HashMap<>(); - property.put("type", p.getType()); - properties.put(p.getName(), property); - } - mapping.put("properties", properties); - request.mapping(mapping); - } - - /** - * Whether the template exists - * - * @param templateName - * @return - * @throws IOException - */ - public boolean existTemplate(String templateName) throws IOException { - if (StringUtils.isEmpty(templateName)) { - return false; - } - IndexTemplatesExistRequest request = new IndexTemplatesExistRequest(templateName); - EsService esService = esCluster.getEsService(null); - return esService.existsTemplate(request); - } - - /** - * Gets the index of this type under the region - * - * @return - */ - public String getAreaTypeIndex(String area, Integer logType) { - MilogEsClusterDO cluster = esCluster.getByArea4China(area); - if (cluster == null) { - return ""; - } - return getClusterTypeIndex(cluster.getId(), logType); - } - - /** - * Obtain the index of this type under the data center - * - * @return - */ - public String getRegionTypeIndex(String region, Integer logType) { - MilogEsClusterDO esClusterDO = esCluster.getByRegion(region); - if (esClusterDO == null) { - return ""; - } - return getClusterTypeIndex(esClusterDO.getId(), logType); - } - - /** - * Obtain the index of this type under the ES cluster - * - * @param logType - * @return - */ - public String getClusterTypeIndex(Long esClusterId, Integer logType) { - if (esClusterId == null || logType == null) { - return ""; - } - Map params = new HashMap<>(); - params.put("cluster_id", esClusterId); - params.put("log_type", logType); - List indexList = esIndexMapper.selectByMap(params); - return indexChooseAlgo(indexList); - } - - /** - * Index selection logic - * - * @param indexList - * @return - */ - private String indexChooseAlgo(List indexList) { - return randomChoose(indexList); - } - - /** - * Randomly select an index - * - * @param indexList - * @return - */ - private String randomChoose(List indexList) { - if (indexList == null || indexList.isEmpty()) { - return ""; - } - return indexList.get(new Random().nextInt(indexList.size())).getIndexName(); - } - - /** - * Get ES information - * - * @param machineRoom - * @param logType - * @return - */ - public EsInfoDTO getEsInfo(String machineRoom, Integer logType) { - MilogEsClusterDO cluster; - // There is no machine room, it is processed according to domestic logic - if (StringUtils.isEmpty(machineRoom)) { - cluster = esCluster.getCurEsCluster(); - } else { - cluster = esCluster.getByArea4China(machineRoom); - } - if (cluster == null) { - return null; - } - String index = getClusterTypeIndex(cluster.getId(), logType); - return new EsInfoDTO(cluster.getId(), index); - } - - public EsInfoDTO getEsInfo(Long clusterId, Integer logType, String exIndex) { - QueryWrapper queryWrapper = new QueryWrapper() - .eq("cluster_id", clusterId) - .eq("log_type", logType); - List milogEsIndexDOS = esIndexMapper.selectList(queryWrapper); - if (CollectionUtils.isEmpty(milogEsIndexDOS)) { - MilogEsClusterDO milogEsClusterDO = milogEsClusterMapper.selectById(clusterId); - throw new MilogManageException(String.format( - "ES:%s,log type:%s The index does not exist, go to the Resource Management page to configure it first", milogEsClusterDO.getName(), LogTypeEnum.queryNameByType(logType))); - } - if (StringUtils.isNotEmpty(exIndex)) { - Optional doOptional = milogEsIndexDOS.stream() - .filter(indexDO -> Objects.equals(exIndex, indexDO.getIndexName())) - .findFirst(); - if (doOptional.isPresent()) { - return new EsInfoDTO(doOptional.get().getClusterId(), exIndex); - } - } - String indexName = indexChooseAlgo(milogEsIndexDOS); - return new EsInfoDTO(clusterId, indexName); - } - - public EsInfoDTO getEsInfoOtherDept(Long clusterId, Integer logType, String exIndex) { - QueryWrapper queryWrapper = new QueryWrapper() - .eq("cluster_id", clusterId) - .eq("log_type", logType); - List milogEsIndexDOS = esIndexMapper.selectList(queryWrapper); - if (CollectionUtils.isEmpty(milogEsIndexDOS)) { - MilogEsClusterDO milogEsClusterDO = milogEsClusterMapper.selectById(clusterId); - throw new MilogManageException(String.format( - "ES:%s,log type:%s The index does not exist, go to the Resource Management page to configure it first", milogEsClusterDO.getName(), LogTypeEnum.queryNameByType(logType))); - } - if (StringUtils.isNotEmpty(exIndex)) { - Optional doOptional = milogEsIndexDOS.stream() - .filter(indexDO -> Objects.equals(exIndex, indexDO.getIndexName())) - .findFirst(); - if (doOptional.isPresent()) { - return new EsInfoDTO(doOptional.get().getClusterId(), exIndex); - } - } - // Choose one at random - MilogEsIndexDO milogEsIndexDO = milogEsIndexDOS.get(getRandomNum(milogEsIndexDOS.size())); - return new EsInfoDTO(clusterId, milogEsIndexDO.getIndexName()); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/LogProcess.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/LogProcess.java deleted file mode 100644 index 3d3fce516..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/LogProcess.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.domain; - -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.api.model.vo.AgentLogProcessDTO; -import com.xiaomi.mone.log.api.model.vo.TailLogProcessDTO; -import com.xiaomi.mone.log.api.model.vo.UpdateLogProcessCmd; -import com.xiaomi.mone.log.api.service.LogProcessCollector; -import com.xiaomi.mone.log.manager.dao.MilogLogTailDao; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.common.StringUtils; -import com.xiaomi.youpin.docean.plugin.dubbo.anno.Reference; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Resource; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -@Service -@Slf4j -public class LogProcess { - - @Resource - private MilogLogTailDao logtailDao; - - @Getter - private Map> tailProgressMap = new ConcurrentHashMap<>(256); - - @Reference(interfaceClass = LogProcessCollector.class, group = "$dubbo.env.group", check = false, timeout = 14000) - private LogProcessCollector logProcessCollector; - - /** - * Update log collection progress - * - * @param cmd - */ - public void updateLogProcess(UpdateLogProcessCmd cmd) { - log.debug("[LogProcess.updateLogProcess] cmd:{} ", cmd); - if (cmd == null || StringUtils.isEmpty(cmd.getIp())) { - return; - } - tailProgressMap.put(cmd.getIp(), cmd.getCollectList()); - } - - /** - * Get the progress of agent log collection - * - * @param ip - * @return - */ - public List getAgentLogProcess(String ip) { - return logProcessCollector.getAgentLogProcess(ip); - } - - /** - * Get the log collection progress of tail - * - * @param tailId - * @return - */ - public List getTailLogProcess(Long tailId, String targetIp) { - if (tailId == null) { - return Lists.newArrayList(); - } - MilogLogTailDo logTail = logtailDao.queryById(tailId); - if (null == logTail) { - return Lists.newArrayList(); - } - return logProcessCollector.getTailLogProcess(tailId, logTail.getTail(), targetIp); - } - - /** - * Get the log collection progress of the store - * - * @param storeId - * @return - */ - public List getStoreLogProcess(Long storeId, String targetIp) { - if (storeId == null) { - return new ArrayList<>(); - } - List logtailList = logtailDao.getMilogLogtailByStoreId(storeId); - List dtoList = new ArrayList<>(); - List processList; - for (MilogLogTailDo milogLogtailDo : logtailList) { - processList = getTailLogProcess(milogLogtailDo.getId(), targetIp); - if (!processList.isEmpty()) { - dtoList.addAll(processList); - } - } - return dtoList; - } - - public List getColProcessImperfect(Double progressRation) { - return logProcessCollector.getColProcessImperfect(progressRation); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/LogStore.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/LogStore.java deleted file mode 100644 index 323b965f6..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/LogStore.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.domain; - -import com.baomidou.mybatisplus.core.toolkit.Wrappers; -import com.xiaomi.mone.log.api.model.vo.ResourceUserSimple; -import com.xiaomi.mone.log.manager.mapper.MilogEsClusterMapper; -import com.xiaomi.mone.log.manager.model.dto.EsInfoDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogEsClusterDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogMiddlewareConfig; -import com.xiaomi.mone.log.manager.model.vo.LogStoreParam; -import com.xiaomi.mone.log.manager.service.impl.MilogMiddlewareConfigServiceImpl; -import com.xiaomi.youpin.docean.anno.Service; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; -import java.util.List; - -@Service -public class LogStore { - @Resource - private MilogMiddlewareConfigServiceImpl resourceConfigService; - - @Resource - private MilogMiddlewareConfigServiceImpl milogMiddlewareConfigService; - - @Resource - private MilogEsClusterMapper milogEsClusterMapper; - - @Resource - private EsIndexTemplate esIndexTemplate; - - /** - * ES resource binding - */ - public void storeResourceBinding(MilogLogStoreDO ml, LogStoreParam cmd) { - ResourceUserSimple resourceUserConfig = resourceConfigService.userResourceList(cmd.getMachineRoom(), cmd.getLogType()); - if (resourceUserConfig.getInitializedFlag()) { - //Select the ES cluster - if (null == cmd.getEsResourceId()) { - List esClusterDOS = milogEsClusterMapper.selectList(Wrappers.lambdaQuery()); - cmd.setEsResourceId(esClusterDOS.get(esClusterDOS.size() - 1).getId()); - } - EsInfoDTO esInfo = esIndexTemplate.getEsInfo(cmd.getEsResourceId(), cmd.getLogType(), null); - cmd.setEsIndex(esInfo.getIndex()); - ml.setEsClusterId(esInfo.getClusterId()); - if (StringUtils.isEmpty(cmd.getEsIndex())) { - ml.setEsIndex(esInfo.getIndex()); - } else { - ml.setEsIndex(cmd.getEsIndex()); - } - if (null == cmd.getMqResourceId()) { - MilogMiddlewareConfig milogMiddlewareConfig = milogMiddlewareConfigService.queryMiddlewareConfigDefault(cmd.getMachineRoom()); - ml.setMqResourceId(milogMiddlewareConfig.getId()); - cmd.setMqResourceId(milogMiddlewareConfig.getId()); - } - } - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/LogTail.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/LogTail.java deleted file mode 100644 index 25a9c9e3e..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/LogTail.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.domain; - -import com.xiaomi.mone.log.manager.dao.MilogLogTailDao; -import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; -import com.xiaomi.mone.log.manager.mapper.MilogLogTemplateMapper; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.mone.log.manager.service.bind.LogTypeProcessor; -import com.xiaomi.mone.log.manager.service.bind.LogTypeProcessorFactory; -import com.xiaomi.mone.log.manager.service.extension.tail.TailExtensionService; -import com.xiaomi.mone.log.manager.service.extension.tail.TailExtensionServiceFactory; -import com.xiaomi.mone.log.manager.service.impl.LogTailServiceImpl; -import com.xiaomi.youpin.docean.anno.Service; -import org.apache.commons.collections.CollectionUtils; - -import javax.annotation.Resource; -import java.util.List; - -@Service -public class LogTail { - @Resource - private MilogLogTailDao milogLogtailDao; - - @Resource - private LogTailServiceImpl logTailService; - - @Resource - private MilogLogstoreDao logStoreDao; - - @Resource - private MilogLogTemplateMapper milogLogTemplateMapper; - - @Resource - private LogTypeProcessorFactory logTypeProcessorFactory; - - private LogTypeProcessor logTypeProcessor; - - private TailExtensionService tailExtensionService; - - public void init() { - logTypeProcessorFactory.setMilogLogTemplateMapper(milogLogTemplateMapper); - logTypeProcessor = logTypeProcessorFactory.getLogTypeProcessor(); - tailExtensionService = TailExtensionServiceFactory.getTailExtensionService(); - } - - public void handleStoreTail(Long storeId) { - MilogLogStoreDO milogLogStoreDO = logStoreDao.queryById(storeId); - - boolean supportedConsume = logTypeProcessor.supportedConsume(milogLogStoreDO.getLogType()); - - List milogLogtailDos = milogLogtailDao.queryTailsByStoreId(storeId); - if (CollectionUtils.isNotEmpty(milogLogtailDos)) { - milogLogtailDos.forEach(milogLogtailDo -> tailExtensionService.updateSendMsg(milogLogtailDo, milogLogtailDo.getIps(), supportedConsume)); - } - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/LogTemplate.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/LogTemplate.java deleted file mode 100644 index 468e7cb4a..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/LogTemplate.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.domain; - -import com.xiaomi.youpin.docean.anno.Service; - -@Service -public class LogTemplate { - - /** - * Build an app log template - */ - public void createAppTemplate() { - // template - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/SearchLog.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/SearchLog.java deleted file mode 100644 index f73d396e0..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/SearchLog.java +++ /dev/null @@ -1,388 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.domain; - -import com.xiaomi.mone.log.manager.dao.MilogLogTailDao; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.mone.log.manager.model.vo.LogContextQuery; -import com.xiaomi.mone.log.manager.model.vo.LogQuery; -import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionService; -import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionServiceFactory; -import com.xiaomi.mone.log.manager.service.extension.common.DefaultCommonExtensionService; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.common.DoceanConfig; -import com.xiaomi.youpin.docean.common.StringUtils; -import com.xiaomi.youpin.docean.plugin.es.antlr4.common.util.EsQueryUtils; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.poi.hssf.usermodel.HSSFWorkbook; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.builder.SearchSourceBuilder; - -import javax.annotation.Resource; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.time.Instant; -import java.time.ZoneId; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -@Slf4j -@Service -public class SearchLog { - - @Resource - private MilogLogTailDao tailDao; - - private CommonExtensionService commonExtensionService; - - public void init() { - commonExtensionService = CommonExtensionServiceFactory.getCommonExtensionService(); - } - - /** - * Gets the query parameters - * - * @param logQuery - * @param keyList - * @return - */ - public BoolQueryBuilder getQueryBuilder(LogQuery logQuery, List keyList) { - // process tail id - processTailNamesToIds(logQuery); - - BoolQueryBuilder boolQueryBuilder = buildCommonBuilder(logQuery); - if (StringUtils.isEmpty(logQuery.getFullTextSearch())) { - return boolQueryBuilder; - } - SearchSourceBuilder searchSourceBuilder = EsQueryUtils.getSearchSourceBuilder(logQuery.getFullTextSearch()); - if (searchSourceBuilder != null) { - boolQueryBuilder.filter(searchSourceBuilder.query()); - } - return boolQueryBuilder; - } - - private BoolQueryBuilder buildCommonBuilder(LogQuery logQuery) { - BoolQueryBuilder boolQueryBuilder = commonExtensionService.commonRangeQuery(logQuery); - // Support tail multi-selection - BoolQueryBuilder tailQueryBuilder = QueryBuilders.boolQuery(); - if (CollectionUtils.isNotEmpty(logQuery.getTailIds())) { - for (Long tailId : logQuery.getTailIds()) { - tailQueryBuilder.should(commonExtensionService.multipleChooseBuilder(DefaultCommonExtensionService.QueryTypeEnum.ID, logQuery.getStoreId(), tailId.toString())); - } - } else { - if (StringUtils.isNotEmpty(logQuery.getTail())) { - String[] tailLimitArray = logQuery.getTail().split(","); - for (String tail : tailLimitArray) { - tailQueryBuilder.should(commonExtensionService.multipleChooseBuilder(DefaultCommonExtensionService.QueryTypeEnum.TEXT, logQuery.getStoreId(), tail)); - } - } - } - tailQueryBuilder.minimumShouldMatch(1); - boolQueryBuilder.filter(tailQueryBuilder); - return boolQueryBuilder; - } - - /** - * Gets the query parameters for the matrix es data - * - * @param logQuery - * @param keyList - * @return - * @throws Exception - */ - public BoolQueryBuilder getMatrixQueryBuilder(LogQuery logQuery, List keyList) throws Exception { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - Instant fromStartUnixTimestamp = Instant.ofEpochMilli((logQuery.getStartTime())); - //Declare the time zone, otherwise the default is UTC 0 time zone - String startTime = fromStartUnixTimestamp.atZone(ZoneId.of("+08:00")).toString(); - Instant fromEndUnixTimestamp = Instant.ofEpochMilli((logQuery.getEndTime())); - //Declare the time zone, otherwise the default is UTC 0 time zone - String endTime = fromEndUnixTimestamp.atZone(ZoneId.of("+08:00")).toString(); - boolQueryBuilder.filter(QueryBuilders.rangeQuery("alpha_timestamp").from(startTime).to(endTime)); - if (StringUtils.isEmpty(logQuery.getFullTextSearch())) { - return boolQueryBuilder; - } -// BoolQueryBuilder fullTextSearchBuilder = buildTextQuery(logQuery.getFullTextSearch(), keyList); - SearchSourceBuilder searchSourceBuilder = EsQueryUtils.getSearchSourceBuilder(logQuery.getFullTextSearch()); - if (searchSourceBuilder != null) { - boolQueryBuilder.filter(searchSourceBuilder.query()); - } - return boolQueryBuilder; - } - - public static BoolQueryBuilder buildTextQuery(String querytext, List keyList) { - List mustQueryTextList = new ArrayList<>(); - List mustNotQueryTextList = new ArrayList<>(); - queryAnalyse(querytext, mustQueryTextList, mustNotQueryTextList); - BoolQueryBuilder boolQueryBuilder = queryDispatchAndBuild(mustQueryTextList, mustNotQueryTextList, keyList); - return boolQueryBuilder; - } - - private static BoolQueryBuilder queryDispatchAndBuild(List mustQueryTextList, List mustNotQueryTextList, List keyList) { - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - QueryBuilder thisQueryBuilder; - boolean isGrantQuery; - String queryText; - int i = 0; - while (i < mustQueryTextList.size() + mustNotQueryTextList.size()) { - isGrantQuery = i < mustQueryTextList.size(); - queryText = isGrantQuery ? mustQueryTextList.get(i) : mustNotQueryTextList.get(i - mustQueryTextList.size()); - i++; - // Specific keyword searches - if (queryText.startsWith("\"")) { - queryText = queryText.substring(1, queryText.length() - 1); - thisQueryBuilder = precisionQueryBuilder(queryText); - queryBuilder = isGrantQuery ? queryBuilder.must(thisQueryBuilder) : queryBuilder.mustNot(thisQueryBuilder); - continue; - } - // k-v search - if (queryText.contains(":")) { - int kvApartIndex = queryText.indexOf(":"); - String key = queryText.substring(0, kvApartIndex).trim(); - String value = kvApartIndex == queryText.length() ? "" : queryText.substring(kvApartIndex + 1).trim(); - // Specific keyword searches... - if (value.startsWith("\"") && value.endsWith("\"")) { - value = value.substring(1, value.length() - 1); - thisQueryBuilder = kvPrecisionQueryBuilder(key, value); - queryBuilder = isGrantQuery ? queryBuilder.must(thisQueryBuilder) : queryBuilder.mustNot(thisQueryBuilder); - continue; - } else { - // K-V normal search - thisQueryBuilder = kvMatchQueryBuilder(key, value); - queryBuilder = isGrantQuery ? queryBuilder.must(thisQueryBuilder) : queryBuilder.mustNot(thisQueryBuilder); - continue; - } - } - // Participle search - thisQueryBuilder = multiMatchQueryBuilder(queryText, keyList); - queryBuilder = isGrantQuery ? queryBuilder.must(thisQueryBuilder) : queryBuilder.mustNot(thisQueryBuilder); - } - return queryBuilder; - } - - private static void queryAnalyse(String querytext, List mustQueryTextList, List mustNotQueryTextList) { - boolean isGrantQuery; // Determine whether the search term is added not - do { - querytext = querytext.trim(); - isGrantQuery = true; - // If the search term is added with NOT, then all search conditions after NOT are NOT logic - if (isGrantQuery && (querytext.trim().toLowerCase().startsWith("not "))) { - isGrantQuery = false; - // Remove "not" from the search statement - querytext = querytext.substring(4); - } - int endIndex = getEndIndex(querytext); - String thisQuerytext = querytext.substring(0, endIndex); - if (isGrantQuery) { - mustQueryTextList.add(thisQuerytext); - } else { - mustNotQueryTextList.add(thisQuerytext); - } - querytext = querytext.substring(endIndex).trim(); - // Remove the "and" from the search statement. - if (querytext.toLowerCase().startsWith("and ")) { - querytext = querytext.substring(4); - } - } while (StringUtils.isNotEmpty(querytext)); - } - - private static QueryBuilder kvMatchQueryBuilder(String key, String value) { - key = key.trim(); - value = value.trim(); - if ("logLevel".equals(key) || "level".equals(key) && ("INFO".equalsIgnoreCase(value) || "WARN".equalsIgnoreCase(value))) { - value = String.format("%-5s", value); - } - return QueryBuilders.matchQuery(key, value); - } - - private static QueryBuilder multiMatchQueryBuilder(String querytext, List keyList) { - BoolQueryBuilder textQueryBuilder = QueryBuilders.boolQuery(); - // All plus regular expression queries - textQueryBuilder.should(regexpQuery(querytext)); - QueryBuilder queryBuilder = QueryBuildChain.doChain(querytext, keyList); - textQueryBuilder.should(queryBuilder); - textQueryBuilder.minimumShouldMatch(1); - return textQueryBuilder; - } - - // Regular expression queries - private static QueryBuilder regexpQuery(String querytext) { - return QueryBuilders.regexpQuery("message", querytext.toLowerCase()); - } - - // Precise + prefix query - private static QueryBuilder precisionQueryBuilder(String querytext) { - BoolQueryBuilder phraseQueryBuilder = QueryBuilders.boolQuery(); - phraseQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery("message", querytext)); - phraseQueryBuilder.should(QueryBuilders.termQuery("traceId", querytext)); - phraseQueryBuilder.minimumShouldMatch(1); - return phraseQueryBuilder; - } - - // kv exact + prefix query - private static QueryBuilder kvPrecisionQueryBuilder(String key, String value) { - return QueryBuilders.matchPhrasePrefixQuery(key, value); - } - - public static class QueryBuildChain { - private static final String[] simpleQueryStringSymble = {"+", "|", "-"}; - private static final String[] simpleQueryStringEndSymble = {"*"}; - private static final String[] wildcardQuerySymble = {"*", "?"}; - - private static QueryBuilder doChain(String querytext, List keyList) { - return simpleQueryString(querytext, keyList); - } - - // Simple statement queries - private static QueryBuilder simpleQueryString(String querytext, List keyList) { - for (String symble : simpleQueryStringSymble) { - if (querytext.contains(symble)) { - return QueryBuilders.simpleQueryStringQuery(querytext); - } - } - for (String symble : simpleQueryStringEndSymble) { - if (querytext.endsWith(symble)) { - return QueryBuilders.simpleQueryStringQuery(querytext); - } - } - return wildcardQuery(querytext, keyList); - } - - // Fuzzy queries - private static QueryBuilder wildcardQuery(String querytext, List keyList) { - for (String symble : wildcardQuerySymble) { - if (querytext.contains(symble)) { - return QueryBuilders.wildcardQuery("message", querytext); - } - } - return multiMatchQuery(querytext, keyList); - } - - // Multi-field queries - private static QueryBuilder multiMatchQuery(String querytext, List keyList) { - return QueryBuilders.multiMatchQuery(querytext, keyList.toArray(new String[keyList.size()])); - } - } - - private static int getEndIndex(String querytext) { - querytext = querytext.toLowerCase(); - // Phrase query Returns the coordinates of the second quotation mark - if (querytext.startsWith("\"")) { - return querytext.substring(1).indexOf("\"") + 2; - } - // Multi-criteria queries - int endIndex = querytext.indexOf("and "); - if (endIndex == -1 || querytext.substring(0, endIndex).contains("not ")) { - endIndex = querytext.indexOf("not "); - } - if (endIndex == -1) { - return querytext.length(); // Single-condition queries - } - // The v of a k-v query contains "and", for example: a: "1 and b" - String thisQueryText = querytext.substring(0, endIndex); - if (thisQueryText.contains(":")) { - String queryValue = querytext.substring(querytext.indexOf(":") + 1); - if (queryValue.startsWith("\"")) { - endIndex = queryValue.substring(1).indexOf("\"") + 2 + thisQueryText.indexOf(":") + 1; - } - } - return endIndex; - } - - public boolean isLegalParam(LogContextQuery param) { - if (param == null || StringUtils.isEmpty(param.getLogstore()) || StringUtils.isEmpty(param.getIp()) || StringUtils.isEmpty(param.getFileName()) || param.getLineNumber() == null || StringUtils.isEmpty(param.getTimestamp()) || param.getType() == null || param.getPageSize() == null) { - return false; - } - return true; - } - - public void downLogFile(HSSFWorkbook excel, String fileName) throws IOException { - File file = null; - FileOutputStream fos = null; - try { - file = new File(DoceanConfig.ins().get("download_file_path", "/tmp") + File.separator + fileName); - file.createNewFile(); - fos = new FileOutputStream(file); - excel.write(fos); - Down.down(fileName); - } catch (Exception e) { - log.error("downLogFile error,fileName:{}", fileName, e); - } finally { - if (excel != null) { - excel.close(); - } - if (fos != null) { - fos.close(); - } - if (file != null) { - file.delete(); - } - } - } - - public String esHistogramInterval(Long duration) { - duration = duration / 1000; - if (duration > 24 * 60 * 60) { - duration = duration / 100; - return duration + "s"; - } else if (duration > 12 * 60 * 60) { - duration = duration / 80; - return duration + "s"; - } else if (duration > 6 * 60 * 60) { - duration = duration / 60; - return duration + "s"; - } else if (duration > 60 * 60) { - duration = duration / 50; - return duration + "s"; - } else if (duration > 30 * 60) { - duration = duration / 40; - return duration + "s"; - } else if (duration > 10 * 60) { - duration = duration / 30; - return duration + "s"; - } else if (duration > 5 * 60) { - duration = duration / 25; - return duration + "s"; - } else if (duration > 3 * 60) { - duration = duration / 20; - return duration + "s"; - } else if (duration > 60) { - duration = duration / 15; - return duration + "s"; - } else if (duration > 10) { - duration = duration / 10; - return duration + "s"; - } else { - return ""; - } - } - - private void processTailNamesToIds(LogQuery logQuery) { - if (StringUtils.isNotEmpty(logQuery.getTail())) { - List tailNames = List.of(logQuery.getTail().split(",")); - List logTailDos = tailDao.queryByNames(logQuery.getStoreId(), tailNames); - List tailIds = logTailDos.stream() - .map(MilogLogTailDo::getId) - .collect(Collectors.toList()); - logQuery.setTailIds(tailIds); - } - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/Store.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/Store.java deleted file mode 100644 index 9b320f7a0..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/Store.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.domain; - -import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; -import com.xiaomi.mone.log.manager.dao.MilogStoreSpaceAuthDao; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogStoreSpaceAuth; -import com.xiaomi.youpin.docean.anno.Service; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -@Service -public class Store { - @Resource - private MilogLogstoreDao logstoreDao; - - @Resource - private MilogStoreSpaceAuthDao storeSpaceAuthDao; - - public List getStoreList(List spaceIdList, String keyword) { - List storeList = logstoreDao.getMilogLogstoreBySpaceId(spaceIdList); - Set storeIdSet = storeList.stream().filter(store -> { - if (StringUtils.isNotEmpty(keyword)) { - return store.getLogstoreName().contains(keyword); - } - return true; - }).map(MilogLogStoreDO::getId).collect(Collectors.toSet()); - List storeAuthList = storeSpaceAuthDao.queryBySpaceId(spaceIdList); - if (storeAuthList != null && !storeAuthList.isEmpty()) { - List authStoreIdList = storeAuthList.stream().filter(auth -> !storeIdSet.contains(auth.getStoreId())).map(MilogStoreSpaceAuth::getStoreId).collect(Collectors.toList()); - List authStoreList = logstoreDao.queryByIds(authStoreIdList); - storeList.addAll(authStoreList); - } - return storeList; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/Tpc.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/Tpc.java deleted file mode 100644 index 7601c6c15..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/Tpc.java +++ /dev/null @@ -1,295 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.domain; - -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.manager.common.context.MoneUserContext; -import com.xiaomi.mone.log.manager.common.exception.MilogManageException; -import com.xiaomi.mone.log.manager.dao.MilogSpaceDao; -import com.xiaomi.mone.log.manager.model.MilogSpaceParam; -import com.xiaomi.mone.log.manager.model.pojo.MilogSpaceDO; -import com.xiaomi.mone.log.manager.user.MoneUser; -import com.xiaomi.mone.tpc.api.service.NodeFacade; -import com.xiaomi.mone.tpc.api.service.NodeUserFacade; -import com.xiaomi.mone.tpc.api.service.UserOrgFacade; -import com.xiaomi.mone.tpc.common.enums.NodeStatusEnum; -import com.xiaomi.mone.tpc.common.enums.NodeTypeEnum; -import com.xiaomi.mone.tpc.common.enums.OutIdTypeEnum; -import com.xiaomi.mone.tpc.common.enums.UserTypeEnum; -import com.xiaomi.mone.tpc.common.param.*; -import com.xiaomi.mone.tpc.common.vo.NodeVo; -import com.xiaomi.mone.tpc.common.vo.OrgInfoVo; -import com.xiaomi.mone.tpc.common.vo.PageDataVo; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.common.StringUtils; -import com.xiaomi.youpin.docean.plugin.config.anno.Value; -import com.xiaomi.youpin.docean.plugin.dubbo.anno.Reference; -import com.xiaomi.youpin.infra.rpc.Result; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; - -import javax.annotation.Resource; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.GSON; - -@Service -@Slf4j -public class Tpc { - @Resource - private MilogSpaceDao milogSpaceDao; - - @Reference(interfaceClass = NodeFacade.class, group = "$tpc_dubbo_group", check = false, version = "1.0", timeout = 15000) - private NodeFacade tpcService; - - @Reference(interfaceClass = NodeUserFacade.class, group = "$tpc_dubbo_group", check = false, version = "1.0", timeout = 15000) - private NodeUserFacade tpcUserService; - - @Reference(interfaceClass = UserOrgFacade.class, group = "$tpc_dubbo_group", check = false, version = "1.0", timeout = 15000) - private UserOrgFacade userOrgFacade; - - @Value("${tpc_node_code}") - private String tpcNodeCode; - - private Long tpcPId; - - public boolean isAdmin(String account, Integer userType) { - handleRemoteTpcId(tpcNodeCode, account, userType); - NodeQryParam param = new NodeQryParam(); - param.setAccount(account); - param.setUserType(userType); - param.setId(tpcPId); - param.setType(NodeTypeEnum.PRO_TYPE.getCode()); - Result nodeVoResult = tpcService.get(param); - NodeVo node = nodeVoResult.getData(); - log.debug("get user isAdmin,param:{},result:{}", GSON.toJson(param), GSON.toJson(node)); - return node.isCurrentMgr() || node.isTopMgr() || node.isParentMgr(); - } - - public List getUserPermSpaceId() { - Result> res = getUserPermSpace(null, 1, 100); - if (res == null || res.getData() == null || res.getData().getList() == null) { - return null; - } - return res.getData().getList().stream().map(NodeVo::getOutId).collect(Collectors.toList()); - } - - public Result> getUserPermSpace(String spaceName, Integer page, Integer pageSize) { - MoneUser currentUser = MoneUserContext.getCurrentUser(); - if (null == currentUser) { - log.info("user not login"); - throw new MilogManageException("please go to login"); - } - handleRemoteTpcId(tpcNodeCode, currentUser.getUser(), currentUser.getUserType()); - NodeQryParam param = new NodeQryParam(); - param.setPager(true); - param.setPage(page); - param.setPageSize(pageSize); - param.setParentId(tpcPId); - param.setAccount(currentUser.getUser()); - param.setType(NodeTypeEnum.PRO_SUB_GROUP.getCode()); - param.setUserType(currentUser.getUserType()); - if (StringUtils.isNotEmpty(spaceName)) { - param.setNodeName(spaceName); - } - param.setStatus(NodeStatusEnum.ENABLE.getCode()); - // The admin user queries all - param.setMyNode(!currentUser.getIsAdmin()); - return tpcService.orgNodelist(param); - } - - public List queryUserListNode(String spaceName, String userName) { - handleRemoteTpcId(tpcNodeCode, userName, UserTypeEnum.CAS_TYPE.getCode()); - - boolean isAdmin = isAdmin(userName, UserTypeEnum.CAS_TYPE.getCode()); - - List nodeVoList = Lists.newArrayList(); - int page = 1; - Integer pageSize = 100; - while (true) { - NodeQryParam param = createNodeQryParam(spaceName, userName, page, isAdmin, pageSize); - Result> pageDataVoResult = tpcService.orgNodelist(param); - - Optional.ofNullable(pageDataVoResult) - .map(Result::getData) - .map(PageDataVo::getList) - .ifPresentOrElse(nodeVoList::addAll, () -> log.info("No more nodes found. Breaking out of the loop.")); - - if (pageDataVoResult == null || pageDataVoResult.getData() == null || CollectionUtils.isEmpty(pageDataVoResult.getData().getList())) { - break; - } - - page++; - } - - return nodeVoList; - } - - private NodeQryParam createNodeQryParam(String spaceName, String userName, int page, boolean isAdmin, Integer pageSize) { - NodeQryParam param = new NodeQryParam(); - param.setPager(true); - param.setPage(page); - param.setPageSize(pageSize); - param.setParentId(tpcPId); - param.setAccount(userName); - param.setType(NodeTypeEnum.PRO_SUB_GROUP.getCode()); - param.setUserType(UserTypeEnum.CAS_TYPE.getCode()); - param.setStatus(NodeStatusEnum.ENABLE.getCode()); - param.setMyNode(!isAdmin); - if (StringUtils.isNotEmpty(spaceName)) { - param.setNodeName(spaceName); - } - return param; - } - - /** - * set tpc id from tpc server - */ - public void handleRemoteTpcId(String tpcNodeCode, String account, Integer userType) { - if (StringUtils.isEmpty(tpcNodeCode)) { - throw new MilogManageException("tpc_node_code is empty,please check config file"); - } - if (null == tpcPId) { - Result nodeVoResult = tpcService.getByNodeCode(getNodeParam(account, userType)); - tpcPId = nodeVoResult.getData().getId(); - } - if (null == tpcPId) { - throw new MilogManageException("query tpc id by tpc server error,tpc code:" + tpcNodeCode); - } - } - - private NodeQryParam getNodeParam(String account, Integer userType) { - NodeQryParam nodeParam = new NodeQryParam(); - nodeParam.setAccount(account); - nodeParam.setUserType(userType); - nodeParam.setType(NodeTypeEnum.PRO_TYPE.getCode()); - nodeParam.setNodeCode(tpcNodeCode); - return nodeParam; - } - - public boolean hasPerm(MoneUser user, Long spaceId) { - NodeQryParam param = new NodeQryParam(); - param.setAccount(user.getUser()); - param.setUserType(user.getUserType()); - param.setOutId(spaceId); - param.setOutIdType(OutIdTypeEnum.SPACE.getCode()); - Result nodeVoResult = tpcService.getByOutId(param); - NodeVo node = nodeVoResult.getData(); - return node.isTopMgr() || node.isParentMgr() || node.isCurrentMgr(); - } - - public Result deleteSpaceTpc(Long id, String account, Integer userType) { - NodeDeleteParam delete = new NodeDeleteParam(); - delete.setOutId(id); - delete.setOutIdType(OutIdTypeEnum.SPACE.getCode()); - delete.setAccount(account); - delete.setUserType(userType); - return tpcService.delete(delete); - } - - public Result saveSpacePerm(MilogSpaceDO spaceDO, String account) { - NodeAddParam nodeAddParam = new NodeAddParam(); - handleRemoteTpcId(tpcNodeCode, account, UserTypeEnum.CAS_TYPE.getCode()); - nodeAddParam.setParentNodeId(tpcPId); - nodeAddParam.setType(NodeTypeEnum.PRO_SUB_GROUP.getCode()); - nodeAddParam.setNodeName(spaceDO.getSpaceName()); - nodeAddParam.setDesc(spaceDO.getDescription()); - nodeAddParam.setOutId(spaceDO.getId()); - nodeAddParam.setOutIdType(OutIdTypeEnum.SPACE.getCode()); - nodeAddParam.setAccount(account); - nodeAddParam.setUserType(UserTypeEnum.CAS_TYPE.getCode()); - - return tpcService.add(nodeAddParam); - } - - public Result updateSpaceTpc(MilogSpaceParam param, String accout) { - NodeEditParam edit = new NodeEditParam(); - edit.setNodeName(param.getSpaceName()); - edit.setDesc(param.getDescription()); - edit.setOutId(param.getId()); - edit.setOutIdType(OutIdTypeEnum.SPACE.getCode()); - edit.setAccount(accout); - edit.setUserType(UserTypeEnum.CAS_TYPE.getCode()); - return tpcService.edit(edit); - } - - /** - * add space member - * - * @param spaceId - * @param userAccount - * @param userType - * @param memberCode - */ - public void addSpaceMember(Long spaceId, String userAccount, Integer userType, Integer memberCode) { - NodeUserAddParam add = new NodeUserAddParam(); - add.setOutId(spaceId); - add.setOutIdType(OutIdTypeEnum.SPACE.getCode()); - add.setMemberAcc(userAccount); - add.setMemberAccType(UserTypeEnum.CAS_TYPE.getCode()); - add.setType(memberCode); - add.setAccount(userAccount); - add.setUserType(userType); - tpcUserService.add(add); - } - - public NodeVo getByOuterId(Long id, Integer outType) { - MoneUser currentUser = MoneUserContext.getCurrentUser(); - NodeQryParam param = new NodeQryParam(); - param.setPager(false); - param.setAccount(currentUser == null ? "system" : currentUser.getUser()); - param.setUserType(UserTypeEnum.CAS_TYPE.getCode()); - param.setType(NodeTypeEnum.PRO_SUB_GROUP.getCode()); - if (null != currentUser) { - param.setUserType(currentUser.getUserType()); - } - param.setStatus(NodeStatusEnum.ENABLE.getCode()); - param.setOutIdType(OutIdTypeEnum.SPACE.getCode()); - param.setOutId(id); - param.setMyNode(false); - Result nodeVoResult = tpcService.getByOutId(param); - NodeVo data = nodeVoResult.getData(); - return data; - } - - public NodeVo getSpaceByOuterId(Long id) { - return getByOuterId(id, OutIdTypeEnum.SPACE.getCode()); - } - - public String getSpaceLastOrg(Long id) { - NodeVo spaceNode = this.getSpaceByOuterId(id); - if (spaceNode == null || spaceNode.getOrgInfoVo() == null) { - return ""; - } - String spaceOrg = spaceNode.getOrgInfoVo().getNamePath(); - return spaceOrg.lastIndexOf('/') == -1 ? "Public space" : spaceOrg.substring(spaceOrg.lastIndexOf('/') + 1); - } - - public OrgInfoVo getOrg(String account, Integer userType) { - NullParam param = new NullParam(); - param.setAccount(account); - param.setUserType(userType); - Result res = userOrgFacade.getOrgByAccount(param); - if (res == null || res.getCode() != 0) { - log.warn("Failed to find user department,account:[{}], userType:[{}], res:[{}]", account, userType, res); - return new OrgInfoVo(); - } - return res.getData(); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/TraceLog.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/TraceLog.java deleted file mode 100644 index 021f6a3c0..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/TraceLog.java +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.domain; - -import cn.hutool.core.date.DateTime; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; -import com.google.common.collect.Lists; -import com.google.gson.Gson; -import com.xiaomi.mone.log.api.model.dto.TraceLogDTO; -import com.xiaomi.mone.log.common.Constant; -import com.xiaomi.mone.log.manager.dao.MilogLogTailDao; -import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; -import com.xiaomi.mone.log.manager.mapper.MilogEsIndexMapper; -import com.xiaomi.mone.log.manager.model.pojo.MilogEsIndexDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.common.StringUtils; -import com.xiaomi.youpin.docean.plugin.es.EsService; -import lombok.Data; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import javax.annotation.Resource; -import java.io.IOException; -import java.util.*; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -@Slf4j -@Service -public class TraceLog { - private static final String TIME_STAMP = "timestamp"; - @Resource - private EsCluster esCluster; - - @Resource - private MilogEsIndexMapper esIndexMapper; - - @Resource - private MilogLogstoreDao logStoreDao; - - @Resource - private MilogLogTailDao logTailDao; - - public TraceLogDTO getTraceLog(Long appId, String traceId, String region, String generationTime, String level) throws IOException { - if (StringUtils.isEmpty(traceId)) { - return null; - } - SearchSourceBuilder qb = new SearchSourceBuilder(); - BoolQueryBuilder queryBuilder = getBoolQueryBuilder(appId, traceId, generationTime, level); - qb.query(queryBuilder); - List indexList; - - List clusterIndexVOS; - if (null != appId) { - clusterIndexVOS = logStoreDao.queryClusterIndexByAppId(appId); - } else { - if (StringUtils.isEmpty(region)) { - // Region is empty to query all regions in the country - indexList = esIndexMapper.selectAreaIndexList("cn"); - } else { - indexList = esIndexMapper.selectRegionIndexList(region); - } - if (indexList == null || indexList.isEmpty()) { - return TraceLogDTO.emptyData(); - } - clusterIndexVOS = indexList.stream() - .map(MilogEsIndexDO::toClusterIndexVO).distinct().collect(Collectors.toList()); - } - return EsAsyncSearch(clusterIndexVOS, qb); - } - - @NotNull - private BoolQueryBuilder getBoolQueryBuilder(Long appId, String traceId, String generationTime, String level) { - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - List tailIds = queryTailIdByAppId(appId); - queryBuilder.filter(QueryBuilders.termQuery("traceId", traceId)); - if (StringUtils.isNotBlank(level)) { - queryBuilder.filter(QueryBuilders.termQuery("level", level)); - } - if(CollectionUtils.isNotEmpty(tailIds)){ - queryBuilder.filter(QueryBuilders.termsQuery("tailId", tailIds)); - } - if (StringUtils.isNotEmpty(generationTime)) { - long time = new DateTime(generationTime).getTime(); - long startTime = time - TimeUnit.HOURS.toMillis(1); - long endTime = time + TimeUnit.HOURS.toMillis(1); - queryBuilder.filter(QueryBuilders.rangeQuery("timestamp").from(startTime).to(endTime)); - } - return queryBuilder; - } - - private List queryTailIdByAppId(Long appId) { - List logTailDos = logTailDao.queryByAppId(appId); - if (CollectionUtils.isNotEmpty(logTailDos)) { - return logTailDos.stream().map(MilogLogTailDo::getId).collect(Collectors.toList()); - } - return Lists.newArrayList(); - } - - public TraceLogDTO EsAsyncSearch(List indexList, SearchSourceBuilder qb) { - CountDownLatch countDownLatch = new CountDownLatch(indexList.size()); - AsyncSearchObj asyncSearchObj = new AsyncSearchObj(countDownLatch); - for (ClusterIndexVO esIndexDO : indexList) { - EsService esService = esCluster.getEsService(esIndexDO.getClusterId()); - if (esService == null) { - countDownLatch.countDown(); - log.warn("[Esdata.getTraceLog] es client [{}] is not generated", Constant.LOG_STORAGE_SERV_BEAN_PRE + esIndexDO.getClusterId()); - continue; - } - SearchRequest searchRequest = new SearchRequest(esIndexDO.getIndexName()); - searchRequest.source(qb); - esService.searchAsync(searchRequest, asyncSearchObj.getListenerStack().pop()); - } - try { - countDownLatch.await(10, TimeUnit.SECONDS); - } catch (InterruptedException e) { - log.error("Log query error, query trace log error countdownlatch timeout, error is [{}]", e.getMessage(), e); - } - Set logSet = asyncSearchObj.getLogSet(); - return new TraceLogDTO(sortedLogWithTime(logSet)); - } - - @Nullable - private Set sortedLogWithTime(Set logSet) { - if (CollectionUtils.isNotEmpty(logSet)) { - logSet = logSet.stream().sorted((o1, o2) -> { - try { - JSONObject obj1 = JSON.parseObject(String.valueOf(o1)); - JSONObject obj2 = JSON.parseObject(String.valueOf(o2)); - if (obj1.getLong(TIME_STAMP) == null) { - return 1; - } - if (obj2.getLong(TIME_STAMP) == null) { - return -1; - } - return obj2.getLong(TIME_STAMP).compareTo(obj1.getLong(TIME_STAMP)); - } catch (Exception e) { - log.error("compare exception", e); - } - return 0; - }).collect(Collectors.toCollection(LinkedHashSet::new)); - } - return logSet; - } - - @Data - private class AsyncSearchObj { - private Set logSet; - private Stack> listenerStack = new Stack<>(); - - public AsyncSearchObj(CountDownLatch countDownLatch) { - this.logSet = Collections.synchronizedSet(new TreeSet<>(new Comparator() { - Gson gson = new Gson(); - - @Override - public int compare(String o1, String o2) { - if (gson.fromJson(o1, Map.class).get("timestamp") == null) { - return 1; - } - if (gson.fromJson(o2, Map.class).get("timestamp") == null) { - return -1; - } - int res = (int) (Double.parseDouble(String.valueOf(gson.fromJson(o1, Map.class).get("timestamp"))) - Double.parseDouble((String.valueOf(gson.fromJson(o2, Map.class).get("timestamp"))))); - if (res == 0) { - // return (int) (Double.parseDouble(String.valueOf(gson.fromJson(o1, Map.class).get("_id"))) - Double.parseDouble((String.valueOf(gson.fromJson(o2, Map.class).get("_id"))))); - return 1; - } - return res; - } - })); - for (int i = 0; i < countDownLatch.getCount(); i++) { - ActionListener listener = new ActionListener() { - @SneakyThrows - @Override - public void onResponse(SearchResponse searchResponse) { - SearchHit[] hits = searchResponse.getHits().getHits(); - if (hits == null || hits.length == 0) { - countDownLatch.countDown(); - return; - } - for (SearchHit hit : hits) { - logSet.add(hit.getSourceAsString()); - } - countDownLatch.countDown(); - } - - @Override - public void onFailure(Exception e) { - countDownLatch.countDown(); - log.error("[Esdata.getTraceLog] search has failure, error is [{}]", e.getMessage()); - } - }; - listenerStack.push(listener); - } - } - } - -} \ No newline at end of file diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/analyse/AggrCalcu.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/analyse/AggrCalcu.java deleted file mode 100644 index 5aa7b7d02..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/analyse/AggrCalcu.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.domain.analyse; - -import com.xiaomi.mone.log.manager.model.bo.CalcuAggrParam; -import com.xiaomi.mone.log.manager.model.dto.LogAnalyseDataDTO; -import com.xiaomi.youpin.docean.Ioc; -import com.xiaomi.youpin.docean.anno.Service; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.search.aggregations.AggregationBuilder; - -@Service -public class AggrCalcu { - - private AggrCalcuStrategy getStrategy(Integer graphType) { - if (graphType == null) { - return null; - } - switch (graphType) { - case 1: - case 2: - case 3: - case 4: - case 5: - case 6: - case 7: - case 8: - case 9: - return Ioc.ins().getBean(FieldStrategy.class.getName()); - case 10: - return Ioc.ins().getBean(DateGroupStrategy.class.getName()); - default: - return null; - } - } - - public AggregationBuilder getAggr(CalcuAggrParam param) { - AggrCalcuStrategy strategy = getStrategy(param.getGraphType()); - if (strategy == null) { - return null; - } - return strategy.getAggr(param); - } - - public LogAnalyseDataDTO formatRes(Integer graphType, SearchResponse response) { - AggrCalcuStrategy strategy = getStrategy(graphType); - if (strategy == null) { - return null; - } - return strategy.formatRes(response); - } - - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/analyse/AggrCalcuStrategy.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/analyse/AggrCalcuStrategy.java deleted file mode 100644 index 8c09a9e82..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/analyse/AggrCalcuStrategy.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.domain.analyse; - -import com.xiaomi.mone.log.manager.model.bo.CalcuAggrParam; -import com.xiaomi.mone.log.manager.model.dto.LogAnalyseDataDTO; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.search.aggregations.AggregationBuilder; - -public interface AggrCalcuStrategy { - - AggregationBuilder getAggr(CalcuAggrParam param); - - LogAnalyseDataDTO formatRes(SearchResponse response); - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/analyse/DateGroupStrategy.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/analyse/DateGroupStrategy.java deleted file mode 100644 index 90712fcc0..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/analyse/DateGroupStrategy.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.domain.analyse; - -import com.xiaomi.mone.log.manager.model.bo.CalcuAggrParam; -import com.xiaomi.mone.log.manager.model.dto.LogAnalyseDataDTO; -import com.xiaomi.mone.log.manager.model.dto.LogAnalyseDataDateDTO; -import com.xiaomi.youpin.docean.anno.Service; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; -import org.elasticsearch.search.aggregations.bucket.histogram.LongBounds; -import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; - -import java.util.*; - -@Service -public class DateGroupStrategy implements AggrCalcuStrategy { - - @Override - public AggregationBuilder getAggr(CalcuAggrParam param) { - // The timestamp parameter is taken to the second bit to offset the bucketing error - Long startTime = param.getStartTime() / 1000 * 1000; - Long endTime = param.getEndTime() / 1000 * 1000; - - int interval = (int) Math.ceil((endTime - startTime) / 1000 / Double.parseDouble(param.getGraphParam())); - String offset = startTime % (interval * 1000) + "ms"; - // Time binning aggregation - DateHistogramAggregationBuilder dateGroupAggs = AggregationBuilders - .dateHistogram("dateAggs") - .field("timestamp") - .minDocCount(0) - .offset(offset) - .format("yyyy-MM-dd HH:mm:ss") - .timeZone(TimeZone.getTimeZone("GMT+8").toZoneId()) - .fixedInterval(new DateHistogramInterval(interval + "s")) - .extendedBounds(new LongBounds(startTime, endTime - 1000)); - // Field aggregation - AggregationBuilder topAggs = AggregationBuilders - .terms("fieldAggs") - .size(4) - .field(param.getBead()) - .executionHint("map"); - // Combine aggregations - dateGroupAggs.subAggregation(topAggs); - - return dateGroupAggs; - } - - @Override - public LogAnalyseDataDTO formatRes(SearchResponse response) { - if (response == null || response.getAggregations() == null) { - return null; - } - Histogram dateAggs = response.getAggregations().get("dateAggs"); - if (dateAggs == null) { - return null; - } - // Solve the term and data do not correspond - the first traversal, encapsulated into intermediate data - Map> ferryMap = new LinkedHashMap<>(); - Map ferryTermMap; - Set termKeySet = new HashSet<>(); - for (int i = 0; i < dateAggs.getBuckets().size(); i++) { - Histogram.Bucket bucket = dateAggs.getBuckets().get(i); - ParsedStringTerms filedAggs = bucket.getAggregations().get("fieldAggs"); - if (filedAggs == null) { - return null; - } - ferryTermMap = new HashMap<>(); - for (int j = 0; j < filedAggs.getBuckets().size(); j++) { - Terms.Bucket filedAggsBucket = filedAggs.getBuckets().get(j); - ferryTermMap.put(filedAggsBucket.getKeyAsString(), String.valueOf(filedAggsBucket.getDocCount())); - // Collect all the keys - termKeySet.add(filedAggsBucket.getKeyAsString()); - } - ferryMap.put(bucket.getKeyAsString(), ferryTermMap); - } - // Solve the term and data do not correspond - secondary traversal, encapsulated into dto - List> date = new ArrayList<>(); - List ferryDate; - for (Map.Entry> entry : ferryMap.entrySet()) { - ferryDate = new ArrayList<>(); - ferryDate.add(entry.getKey()); - for (String term : termKeySet) { - ferryDate.add(entry.getValue().containsKey(term) ? entry.getValue().get(term) : "0"); - } - date.add(ferryDate); - } - return new LogAnalyseDataDateDTO(date, termKeySet); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/analyse/FieldStrategy.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/analyse/FieldStrategy.java deleted file mode 100644 index 8e12eec4b..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/analyse/FieldStrategy.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.domain.analyse; - -import com.xiaomi.mone.log.manager.model.bo.CalcuAggrParam; -import com.xiaomi.mone.log.manager.model.dto.LogAnalyseDataDTO; -import com.xiaomi.mone.log.manager.model.dto.LogAnalyseDataFieldDTO; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.common.StringUtils; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; - -import java.util.*; - -import static com.xiaomi.mone.log.common.Constant.GSON; - -@Service -public class FieldStrategy implements AggrCalcuStrategy { - - @Override - public AggregationBuilder getAggr(CalcuAggrParam param) { - if (StringUtils.isEmpty(param.getBead())) { - return null; - } - - AggregationBuilder groupAggBuilder = AggregationBuilders.terms("group_aggs") - .size(Integer.MAX_VALUE) - .field(param.getBead()) - // If you know that the number of aggregation results in this layer is very small, set it to map to improve performance. - .executionHint("map"); - - return groupAggBuilder; - } - - @Override - public LogAnalyseDataDTO formatRes(SearchResponse response) { - List> dataArray = new ArrayList<>(); - Map ferray; - if (Objects.nonNull(response) && Objects.equals(response.status(), RestStatus.OK) - && Objects.nonNull(response.getAggregations()) && Objects.nonNull(response.getAggregations().get("group_aggs"))) { - Terms groupResult = response.getAggregations().get("group_aggs"); - for (Terms.Bucket groupBucket : groupResult.getBuckets()) { - ferray = new HashMap<>(); - Object key = groupBucket.getKey(); - long docCount = groupBucket.getDocCount(); - ferray.put("field", key); - ferray.put("count", docCount); - dataArray.add(ferray); - } - } - return new LogAnalyseDataFieldDTO(GSON.toJson(dataArray)); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/job/EsIndexJob.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/job/EsIndexJob.java deleted file mode 100644 index d3e85e693..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/job/EsIndexJob.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.job; - -import cn.hutool.core.thread.ThreadUtil; -import com.xiaomi.mone.log.manager.service.EsIndexOperateService; -import com.xiaomi.youpin.docean.anno.Component; -import com.xiaomi.youpin.docean.plugin.config.anno.Value; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Resource; -import java.util.Objects; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2024/1/19 17:05 The index needs to be created in advance. If it is like wtt-test-data-2023-01-19@000001, it is convenient to use the ilm plug-in of the index to manage the life cycle. - */ -@Component -@Slf4j -public class EsIndexJob { - - private static final String OPEN_TYPE = "open"; - - @Value("$server.type") - private String serverType; - - @Resource - private EsIndexOperateService esIndexOperateService; - - public void init() { - if (!Objects.equals(OPEN_TYPE, serverType)) { - return; - } - log.info("EsIndexJob execute"); - ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor( - ThreadUtil.newNamedThreadFactory("es-index", false) - ); - long initDelay = 1; - long intervalTime = 2; - scheduledExecutor.scheduleAtFixedRate(() -> { - try { - esIndexOperateService.createIndexPre(); - } catch (Exception e) { - log.error("EsIndexJob execute error", e); - } - }, initDelay, TimeUnit.HOURS.toMinutes(intervalTime), TimeUnit.MINUTES); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/job/TailLogCountJob.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/job/TailLogCountJob.java deleted file mode 100644 index 52bd2ccd0..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/job/TailLogCountJob.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.job; - -import cn.hutool.core.thread.ThreadUtil; -import com.xiaomi.mone.log.manager.service.impl.LogCountServiceImpl; -import com.xiaomi.mone.log.utils.DateUtils; -import com.xiaomi.youpin.docean.anno.Component; -import com.xiaomi.youpin.docean.plugin.config.anno.Value; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Resource; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -@Component -@Slf4j -public class TailLogCountJob { - @Resource - private LogCountServiceImpl logCountService; - - @Value("$job_start_flag") - public String jobStartFlag; - - public void init() { - if (!Boolean.parseBoolean(jobStartFlag)) { - return; - } - ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor( - ThreadUtil.newNamedThreadFactory("log-tailLogCountJob", false) - ); - long initDelay = 0; - long intervalTime = 1; - scheduledExecutor.scheduleAtFixedRate(() -> { - statisticsAll(); - }, initDelay, intervalTime, TimeUnit.HOURS); - } - - public void statisticsAll() { - try { - log.info("Statistics log scheduled task starts execution"); - String thisDay = DateUtils.getDaysAgo(1); - if (!logCountService.isLogtailCountDone(thisDay)) { - logCountService.collectLogCount(thisDay); - logCountService.deleteHistoryLogCount(); - } - logCountService.collectTopCount(); - logCountService.collectSpaceTopCount(); - logCountService.collectSpaceTrend(); - log.info("Statistics log scheduled task execution completed"); - } catch (Exception e) { - log.error("Statistical log timing task failed", e); - } - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogAnalyseDashboardGraphRefMapper.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogAnalyseDashboardGraphRefMapper.java deleted file mode 100644 index f2e66c9c0..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogAnalyseDashboardGraphRefMapper.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.xiaomi.mone.log.manager.model.pojo.MilogAnalyseDashboardGraphRefDO; -import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; - -import java.util.List; - -/** - *

- * Mapper - *

- * - * @author wanghaoyang - * @since 2022-08-18 - */ -@Mapper -public interface MilogAnalyseDashboardGraphRefMapper extends BaseMapper { - - List getByDashboardId(@Param(value = "dashboardId") Long dashboardId); - - void deleteGraphRef(@Param(value = "graphId") Long graphId); - - Long isRefed(@Param(value = "dashboardId") Long dashboardId, @Param(value = "graphId") Long graphId); - - int delRef(@Param(value = "dashboardId") Long dashboardId, @Param(value = "graphId") Long graphId); - - MilogAnalyseDashboardGraphRefDO getRef(@Param(value = "dashboardId") Long dashboardId, @Param(value = "graphId") Long graphId); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogAnalyseDashboardMapper.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogAnalyseDashboardMapper.java deleted file mode 100644 index 0337305e9..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogAnalyseDashboardMapper.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.xiaomi.mone.log.manager.model.pojo.MilogAnalyseDashboardDO; -import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; - -import java.util.List; - -/** - *

- * Mapper - *

- * - * @author wanghaoyang - * @since 2022-08-18 - */ -@Mapper -public interface MilogAnalyseDashboardMapper extends BaseMapper { - - List getByStoreId(@Param(value = "storeId") Long storeId); - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogAnalyseGraphMapper.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogAnalyseGraphMapper.java deleted file mode 100644 index 53429eb11..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogAnalyseGraphMapper.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.xiaomi.mone.log.manager.model.dto.DashboardGraphDTO; -import com.xiaomi.mone.log.manager.model.dto.GraphDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogAnalyseGraphDO; -import com.xiaomi.mone.log.manager.model.vo.GraphQuery; -import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; - -import java.util.List; - -/** - *

- * Mapper - *

- * - * @author wanghaoyang - * @since 2022-08-18 - */ -@Mapper -public interface MilogAnalyseGraphMapper extends BaseMapper { - - List getDashboardGraph(@Param(value = "dashboardId") Long dashboardId); - - List search(@Param(value = "query") GraphQuery query); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogAnalyseGraphTypeMapper.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogAnalyseGraphTypeMapper.java deleted file mode 100644 index 46c5bc841..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogAnalyseGraphTypeMapper.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.xiaomi.mone.log.manager.model.pojo.MilogAnalyseGraphTypeDO; -import org.apache.ibatis.annotations.Mapper; - -/** - *

- * Mapper - *

- * - * @author wanghaoyang - * @since 2022-08-18 - */ -@Mapper -public interface MilogAnalyseGraphTypeMapper extends BaseMapper { - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogEsClusterMapper.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogEsClusterMapper.java deleted file mode 100644 index 0e838db11..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogEsClusterMapper.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.xiaomi.mone.log.manager.model.pojo.MilogEsClusterDO; -import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; - -import java.util.List; - -/** - *

- * Mapper - *

- * - * @author wanghaoyang - * @since 2021-09-18 - */ -@Mapper -public interface MilogEsClusterMapper extends BaseMapper { - - /** - * obtain tag Obtain the tag corresponding to the ES client - * - * @param tag - * @return - */ - List selectByTag(@Param("tag") String tag); - - /** - * Find all - * - * @return - */ - List selectAll(); - - /** - * Obtain the ES client corresponding to the region - * - * @param region - * @return - */ - MilogEsClusterDO selectByRegion(@Param("region") String region); - - /** - * Obtain the ES client corresponding to the area - * - * @param area - * @return - */ - List selectByArea(@Param("area") String area, @Param("label") String label); - - List selectByAlias(@Param("alias") String alias); - - /** - * Obtain the ES cluster information corresponding to tags and areas - * - * @param tag - * @param area - * @return - */ - MilogEsClusterDO selectByTagAndArea(@Param("tag") String tag, @Param("area") String area); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogEsIndexMapper.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogEsIndexMapper.java deleted file mode 100644 index 6c35d9095..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogEsIndexMapper.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.xiaomi.mone.log.manager.model.pojo.MilogEsIndexDO; -import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; - -import java.util.List; - -/** - *

- * Mapper - *

- * - * @author wanghaoyang - * @since 2021-09-18 - */ -@Mapper -public interface MilogEsIndexMapper extends BaseMapper { - - /** - * Query all ES indexes under region - * @param region - * @return - */ - List selectRegionIndexList(@Param(value = "region") String region); - - /** - * Query all ES indexes under region... - * @param area - * @return - */ - List selectAreaIndexList(@Param(value = "area") String area); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogLogCountMapper.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogLogCountMapper.java deleted file mode 100644 index 84b7abe78..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogLogCountMapper.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.xiaomi.mone.log.manager.model.pojo.LogCountDO; -import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; - -import java.util.List; -import java.util.Map; - -/** - *

- * Mapper - *

- * - * @author wanghaoyang - * @since 2022-01-07 - */ -@Mapper -public interface MilogLogCountMapper extends BaseMapper { - /** - * Bulk insertion - * @param logCountList - * @return - */ - Long batchInsert(@Param("logCountList") List logCountList); - - /** - * Top 10 statistical log storage - * @return - */ - List> collectTopCount(@Param("fromDay") String fromDay, @Param("toDay") String toDay); - - /** - * Statistics on log storage trends - */ - List> collectTrend(@Param("fromDay") String fromDay, @Param("toDay") String toDay, @Param("tailId") Long tailId); - - /** - * Statistics on the storage trend of SPACE logs - */ - List> collectSpaceTrend(@Param("fromDay") String fromDay, @Param("toDay") String toDay); - - - /** - * Whether this day has been counted - * @param day - * @return - */ - Long isLogtailCountDone(@Param("day") String day); - - /** - * Deletes data older than the specified date - * @param day - */ - void deleteBeforeDay(@Param("day") String day); - - /** - * Delete statistics for this day - * @param day - */ - void deleteThisDay(@Param("day") String day); - - /** - * Space log statistics top 10 - * @param fromDay - * @param toDay - * @return - */ - List> collectSpaceCount(@Param("fromDay") String fromDay, @Param("toDay") String toDay); - - /** - * App log statistics - */ - List> collectAppLog(@Param("day") String day, @Param("threshold") Long threshold); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogLogNumAlertMapper.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogLogNumAlertMapper.java deleted file mode 100644 index f3d232936..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogLogNumAlertMapper.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogNumAlertDO; -import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; - -import java.util.List; - -@Mapper -public interface MilogLogNumAlertMapper extends BaseMapper { - void insertBatch(@Param(value = "doList") List doList); - - Long isSend(@Param(value = "appId") Long appId, @Param(value = "day") String day); - - void deleteThisDay(@Param(value = "day") String day); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogLogProcessMapper.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogLogProcessMapper.java deleted file mode 100644 index e6234f2ec..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogLogProcessMapper.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.mapper; - - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogProcessDOMybatis; -import org.apache.ibatis.annotations.Mapper; - -@Mapper -public interface MilogLogProcessMapper extends BaseMapper { - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogLogSearchSaveMapper.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogLogSearchSaveMapper.java deleted file mode 100644 index b170ff971..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogLogSearchSaveMapper.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.xiaomi.mone.log.manager.model.dto.SearchSaveDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogSearchSaveDO; -import org.apache.ibatis.annotations.Param; - -import java.util.List; - -/** - *

- * Mapper - *

- * - * @author wanghaoyang - * @since 2022-03-29 - */ -public interface MilogLogSearchSaveMapper extends BaseMapper { - - List selectByStoreId(@Param(value = "storeId") Long storeId, @Param(value = "startIndex") Integer startIndex, @Param(value = "pageSize") Integer pageSize); - - Long countByStoreId(@Param(value = "storeId") Long storeId); - - Long countByStoreAndName(@Param(value = "name") String name, @Param(value = "creator") String creator); - - int removeById(@Param(value = "id") Long id); - - List selectByCreator(@Param(value = "creator") String creator, @Param(value = "sort") Integer sort); - - Integer getMaxOrder(@Param(value = "creator") String creator, @Param(value = "sort") Integer sort); - - Integer isMyFavouriteStore(@Param(value = "creator") String creator, @Param(value = "storeId") Long storeId); - - Integer isMyFavouriteTail(@Param(value = "creator") String creator, @Param(value = "tailId") Long tailId); - - List getAll(); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogLogTemplateDetailMapper.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogLogTemplateDetailMapper.java deleted file mode 100644 index e5258b2d5..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogLogTemplateDetailMapper.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTemplateDetailDO; -import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; - -/** - *

- * milog log template detail Mapper - *

- * - * @author wanghaoyang - * @since 2021-09-09 - */ - -@Mapper -public interface MilogLogTemplateDetailMapper extends BaseMapper { - - /** - * Query the details of the template - * @param templateId - * @return - */ - MilogLogTemplateDetailDO getByTemplateId(@Param("templateId") long templateId); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogLogTemplateMapper.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogLogTemplateMapper.java deleted file mode 100644 index 4911e39ce..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogLogTemplateMapper.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTemplateDO; -import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; - -import java.util.List; - -/** - *

- * milog log templateMapper - *

- * - * @author wanghaoyang - * @since 2021-09-09 - */ -@Mapper -public interface MilogLogTemplateMapper extends BaseMapper { - - List selectSupportedTemplate(@Param(value = "area") String area); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogLogstailMapper.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogLogstailMapper.java deleted file mode 100644 index 7b8d0a11b..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/mapper/MilogLogstailMapper.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogstailDO; - -import java.util.List; -import java.util.Map; - -/** - *

- * Mapper - *

- * - * @author wanghaoyang - * @since 2022-01-10 - */ -public interface MilogLogstailMapper extends BaseMapper { - /** - * Obtain the tail used to count the number of log entries - * @return - */ - List> getAllTailForCount(); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/BaseCommon.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/BaseCommon.java deleted file mode 100644 index 85b181c10..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/BaseCommon.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model; - -import lombok.Data; -import org.nutz.dao.entity.annotation.ColDefine; -import org.nutz.dao.entity.annotation.ColType; -import org.nutz.dao.entity.annotation.Column; -import org.nutz.dao.entity.annotation.Comment; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/17 17:01 - */ -@Data -public class BaseCommon { - - @Column(value = "ctime") - @ColDefine(customType = "bigint") - @Comment("create time") - private Long ctime; - - - @Column(value = "utime") - @ColDefine(customType = "bigint") - @Comment("update time") - private Long utime; - - - @Column(value = "creator") - @ColDefine(type = ColType.VARCHAR, width = 50) - @Comment("creator") - private String creator; - - - @Column(value = "updater") - @ColDefine(type = ColType.VARCHAR, width = 50) - @Comment("updater") - private String updater; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/MilogSpaceParam.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/MilogSpaceParam.java deleted file mode 100644 index ce4a108b4..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/MilogSpaceParam.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.List; - -/** - * @author milog - */ -@Data -@Builder -@AllArgsConstructor -@NoArgsConstructor -public class MilogSpaceParam { - private Long id; - private Long tenantId; - private String spaceName; - private String description; - private String permDeptId; - /** - * Administrator, the first person is the creator, and the others are administrators - */ - private List admins; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/PageVo.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/PageVo.java deleted file mode 100644 index 993086be2..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/PageVo.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model; - -import lombok.Data; - -import java.io.Serializable; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/9/19 15:45 - */ -@Data -public class PageVo implements Serializable { - private Integer pageNum = 1; - private Integer pageSize = 20; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/Pair.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/Pair.java deleted file mode 100644 index 1f9345aa7..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/Pair.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model; - -import java.io.Serializable; -import java.util.Objects; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/9/19 14:55 - */ -public class Pair implements Serializable { - private static final long serialVersionUID = 1L; - - protected K key; - protected V value; - - /** - * 构建{@code Pair}对象 - * - * @param 键类型 - * @param 值类型 - * @param key 键 - * @param value 值 - * @return {@code Pair} - * @since 5.4.3 - */ - public static Pair of(K key, V value) { - return new Pair<>(key, value); - } - - /** - * 构造 - * - * @param key 键 - * @param value 值 - */ - public Pair(K key, V value) { - this.key = key; - this.value = value; - } - - /** - * 获取键 - * - * @return 键 - */ - public K getKey() { - return this.key; - } - - /** - * 获取值 - * - * @return 值 - */ - public V getValue() { - return this.value; - } - - @Override - public String toString() { - return "Pair [key=" + key + ", value=" + value + "]"; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o instanceof Pair) { - Pair pair = (Pair) o; - return Objects.equals(getKey(), pair.getKey()) && - Objects.equals(getValue(), pair.getValue()); - } - return false; - } - - @Override - public int hashCode() { - //copy from 1.8 HashMap.Node - return Objects.hashCode(key) ^ Objects.hashCode(value); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/StatisticsQuery.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/StatisticsQuery.java deleted file mode 100644 index 9240778e2..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/StatisticsQuery.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model; - -import lombok.Data; - -import java.io.Serializable; - -@Data -public class StatisticsQuery implements Serializable { - - private Long spaceId; - private Long logstoreId; - private String tail; - private Long startTime; - private Long endTime; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/AccessMilogParam.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/AccessMilogParam.java deleted file mode 100644 index 991efa31a..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/AccessMilogParam.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import lombok.Data; - -import java.time.Instant; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/4/19 14:16 - */ -@Data -public class AccessMilogParam { - - private String spaceName; - private String storeName; - private Long appId; - private String appName; - private String appCreator; - private Long appCreatTime; - private Long funcId; - private String funcName; - private String logPath; - private Integer appType; - private String appTypeText; - private String envName; - private Long envId; - - private String machineRoom = "cn"; - - public static void main(String[] args) { - System.out.println(Instant.now().toEpochMilli()); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/AppTopicParam.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/AppTopicParam.java deleted file mode 100644 index 3fc4e74ab..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/AppTopicParam.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import lombok.Data; - -import java.io.Serializable; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/27 11:22 - */ -@Data -public class AppTopicParam implements Serializable { - - private Long appId; - - private String appName; - - private Integer pageSize=10; - - private Integer page=1; - -// private String type; -// -// private String ak; -// -// private String sk; -// -// private String topic; -// -// private String tag; -// -// private Integer partitionCnt; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/AutoAccessLogParam.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/AutoAccessLogParam.java deleted file mode 100644 index 38c145347..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/AutoAccessLogParam.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import lombok.Data; - -import java.util.List; - -/** - * - * @description - * @version 1.0 - * @author wtt - * @date 2024/7/10 10:54 - * - */ -@Data -public class AutoAccessLogParam { - - private Long appId; - private String appName; - private Long envId; - private String envName; - private List ips; - private String logPath; - - private String tailName; - - private Long spaceId; - private Long storeId; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/BatchQueryParam.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/BatchQueryParam.java deleted file mode 100644 index a088d558d..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/BatchQueryParam.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import lombok.Data; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/19 11:32 - */ -@Data -public class BatchQueryParam { - private List ids; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/CalcuAggrParam.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/CalcuAggrParam.java deleted file mode 100644 index 75bb0a033..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/CalcuAggrParam.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@AllArgsConstructor -@NoArgsConstructor -public class CalcuAggrParam { - - private Integer graphType; - - private String graphParam; - - private String bead; - - private Long startTime; - - private Long endTime; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/EsStatisticParam.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/EsStatisticParam.java deleted file mode 100644 index d1b678fc8..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/EsStatisticParam.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import lombok.Data; - -import java.util.Map; - -@Data -public class EsStatisticParam { - private String logStoreName; - private Long start; - private Long end; - private Map params; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/IpPartitionBalance.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/IpPartitionBalance.java deleted file mode 100644 index c2313d705..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/IpPartitionBalance.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import com.xiaomi.mone.log.manager.model.Pair; -import lombok.Data; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/10/12 10:39 - */ -@Data -public class IpPartitionBalance { - - private List> spaceList; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/LogTailParam.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/LogTailParam.java deleted file mode 100644 index 3b1186d8b..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/LogTailParam.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import com.xiaomi.mone.log.manager.model.dto.MotorRoomDTO; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.List; - -@Data -@NoArgsConstructor -@AllArgsConstructor -@Builder -public class LogTailParam { - private Long id; - private Long spaceId; - private Long storeId; - private Long milogAppId; - private Long appId; - private String appName; - private String deploySpace; - private Long envId; - private String envName; - private List ips; - private String tail; - private Integer parseType; - private String parseScript; - private String logPath; - private String valueList; - private String tailRate; - private Long ctime; - private Long utime; - private Long middlewareConfigId; - - private String logSplitExpress; - - /** - * Beginning of line rule - */ - private String firstLineReg; - /** - * Application type 0:mione application - **/ - private Integer appType; - /** - * Machine type 0.Container 1.Physical machine - */ - private Integer machineType; - /** - * The node information of the computer room where the mis application is located - */ - private List motorRooms; - - private String topicName; - private List middlewareConfig; - /** - * Deployment method 1-mione; - */ - private Integer deployWay; - - private Integer batchSendSize = 20; - - private Boolean collectionReady = true; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MachineParamParam.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MachineParamParam.java deleted file mode 100644 index 9fb12d575..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MachineParamParam.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import com.xiaomi.mone.log.manager.model.pojo.MiLogMachine; -import lombok.Data; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/21 17:05 - */ -@Data -public class MachineParamParam extends MiLogMachine { -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MachinePartitionBalance.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MachinePartitionBalance.java deleted file mode 100644 index 5c4fd0497..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MachinePartitionBalance.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import cn.hutool.core.lang.Pair; -import lombok.Data; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/9/19 14:49 - */ -@Data -public class MachinePartitionBalance { - - private String machineUnique; - private List> data; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MachineQueryParam.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MachineQueryParam.java deleted file mode 100644 index 33c7b1093..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MachineQueryParam.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import lombok.Data; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/21 17:24 - */ -@Data -public class MachineQueryParam { - private Long id; - private String ip; - private Integer type; - private Integer pageSize; - private Integer pageNum; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MiddlewareAddParam.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MiddlewareAddParam.java deleted file mode 100644 index 82e8ff9b1..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MiddlewareAddParam.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import lombok.Data; - -import java.io.Serializable; -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/22 11:57 - */ -@Data -public class MiddlewareAddParam implements Serializable { - /** - * com.xiaomi.mone.log.api.enums.MiddlewareEnum.code - */ - private Integer type; - - private String regionEn; - - private List types; - - private String alias; - - private String nameServer; - - private String serviceUrl; - - private String ak; - - private String sk; - - private String authorization; - - private String orgId; - - private String teamId; - - private Integer isDefault = 0; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MiddlewareQueryParam.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MiddlewareQueryParam.java deleted file mode 100644 index 8c3ff27b4..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MiddlewareQueryParam.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import lombok.Data; - -import java.io.Serializable; -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/22 11:57 - */ -@Data -public class MiddlewareQueryParam extends MilogPageParam implements Serializable { - /** - * com.xiaomi.mone.log.api.enums.MiddlewareEnum - */ - private List types; - - private String alias; - - private String regionEn; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MiddlewareUpdateParam.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MiddlewareUpdateParam.java deleted file mode 100644 index dcc6376da..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MiddlewareUpdateParam.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import lombok.Data; - -import java.io.Serializable; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/22 11:57 - */ -@Data -public class MiddlewareUpdateParam extends MiddlewareAddParam implements Serializable { - private Long id; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MilogAgentIpParam.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MilogAgentIpParam.java deleted file mode 100644 index e1e0b2976..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MilogAgentIpParam.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import lombok.Data; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/1 17:25 - */ -@Data -public class MilogAgentIpParam { - private List ips; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MilogDictionaryParam.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MilogDictionaryParam.java deleted file mode 100644 index 7079bbf75..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MilogDictionaryParam.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import lombok.Data; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/28 10:00 - */ -@Data -public class MilogDictionaryParam { - private List codes; - private Long middlewareId; - private String nameEn; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MilogLogstoreBo.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MilogLogstoreBo.java deleted file mode 100644 index c028ad852..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MilogLogstoreBo.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import lombok.Data; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/1/14 16:41 - */ -@Data -public class MilogLogstoreBo extends MilogLogStoreDO { - private String logTypeText; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MilogPageParam.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MilogPageParam.java deleted file mode 100644 index 10e0185df..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MilogPageParam.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import lombok.Data; - -import java.io.Serializable; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/22 11:59 - */ -@Data -public class MilogPageParam implements Serializable { - - private Integer pageSize = 20; - - private Integer page = 1; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MlogParseParam.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MlogParseParam.java deleted file mode 100644 index b096351c6..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/MlogParseParam.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import lombok.Data; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/1/5 14:36 - */ -@Data -public class MlogParseParam { - private Long storeId; - private String parseScript; - private String valueList; - private Integer parseType; - private String msg; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/RegionZoneBo.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/RegionZoneBo.java deleted file mode 100644 index 3279b4eca..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/RegionZoneBo.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import lombok.Data; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/10/22 17:37 - */ -@Data -public class RegionZoneBo { - private Integer code; - private String message; - private String userMessage; - private String level; - private List data; - - @Data - public static class InnerClass { - private Integer id; - private String zone_name_en; - private String zone_name_cn; - private String region_cn; - private String region_en; - private Boolean is_used; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/RocketMqStatisticParam.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/RocketMqStatisticParam.java deleted file mode 100644 index 80643e0a3..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/RocketMqStatisticParam.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import lombok.Data; - -import java.util.List; - -@Data -public class RocketMqStatisticParam { - private List topicList; - private List groupList; - /** - * Parameter optional value: (* or broker group name), if not null, return the sum of all broker groups - */ - private String broker; - private Long begin; - private Long end; - /** - * Parameter optional values: (avg, sum, min, max) default avg - */ - private String aggregator; - - private String client; - private String metirc; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/SpacePartitionBalance.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/SpacePartitionBalance.java deleted file mode 100644 index ed1632be7..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/SpacePartitionBalance.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import lombok.Data; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/9/19 14:49 - */ -@Data -public class SpacePartitionBalance { - private Long spaceId; - private String spaceName; - private String spaceIdentifiers; - private List machineUniques; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/StoreSpaceAuth.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/StoreSpaceAuth.java deleted file mode 100644 index abe3626ce..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/bo/StoreSpaceAuth.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.bo; - -import lombok.Data; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/7/14 16:17 - */ -@Data -public class StoreSpaceAuth { - private Long storeId; - private Long spaceId; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/cache/LogCellectProcessCache.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/cache/LogCellectProcessCache.java deleted file mode 100644 index 8c95e943b..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/cache/LogCellectProcessCache.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.cache; - -import lombok.Data; - -@Data -public class LogCellectProcessCache { - - private String tailId; - - private String tailName; - - private String path; - - private String ip; - - private String pattern; - - private Long appId; - - - private String appName; - - // Log file line number - private Long fileRowNumber; - - private Long pointer; - - private Long fileMaxPointer; - - // Collect progress - private String collectPercentage; - - // Collection time - private Long collectTime; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/DGRefConvert.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/DGRefConvert.java deleted file mode 100644 index 690ef20ff..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/DGRefConvert.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.convert; - -import com.xiaomi.mone.log.manager.model.pojo.MilogAnalyseDashboardGraphRefDO; -import com.xiaomi.mone.log.manager.model.vo.DGRefCmd; -import com.xiaomi.mone.log.manager.model.vo.DGRefUpdateCmd; -import org.mapstruct.Mapper; -import org.mapstruct.Mapping; -import org.mapstruct.Mappings; -import org.mapstruct.factory.Mappers; - -@Mapper -public interface DGRefConvert { - DGRefConvert INSTANCE = Mappers.getMapper(DGRefConvert.class); - - MilogAnalyseDashboardGraphRefDO toDo(DGRefCmd cmd); - - @Mappings({ - @Mapping(target = "privateName", source = "graphPrivateName") - }) - MilogAnalyseDashboardGraphRefDO toDo(DGRefUpdateCmd.DGRefDetailUpdateCmd cmd); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/DashboardConvert.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/DashboardConvert.java deleted file mode 100644 index 9ae9b7450..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/DashboardConvert.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.convert; - -import com.xiaomi.mone.log.manager.model.dto.DashboardDTO; -import com.xiaomi.mone.log.manager.model.dto.DashboardGraphDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogAnalyseDashboardDO; -import com.xiaomi.mone.log.manager.model.vo.CreateDashboardCmd; -import org.mapstruct.Mapper; -import org.mapstruct.Mapping; -import org.mapstruct.Mappings; -import org.mapstruct.factory.Mappers; - -import java.util.List; - -@Mapper -public interface DashboardConvert { - DashboardConvert INSTANCE = Mappers.getMapper(DashboardConvert.class); - - @Mappings({ - @Mapping(source = "dashboardDO.name", target = "dashboardName"), - @Mapping(source = "dashboardDO.id", target = "dashboardId") - }) - DashboardDTO fromDO(MilogAnalyseDashboardDO dashboardDO, List graphList); - - MilogAnalyseDashboardDO toDO(CreateDashboardCmd cmd); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/GraphConvert.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/GraphConvert.java deleted file mode 100644 index a70d3fe87..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/GraphConvert.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.convert; - -import com.xiaomi.mone.log.manager.model.pojo.MilogAnalyseGraphDO; -import com.xiaomi.mone.log.manager.model.vo.CreateGraphCmd; -import com.xiaomi.mone.log.manager.model.vo.UpdateGraphCmd; -import org.mapstruct.Mapper; -import org.mapstruct.factory.Mappers; - -@Mapper -public interface GraphConvert { - GraphConvert INSTANCE = Mappers.getMapper(GraphConvert.class); - - MilogAnalyseGraphDO toDO(CreateGraphCmd cmd); - - MilogAnalyseGraphDO toDO(UpdateGraphCmd cmd); - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/GraphTypeConvert.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/GraphTypeConvert.java deleted file mode 100644 index 9f7713b92..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/GraphTypeConvert.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.convert; - -import com.xiaomi.mone.log.manager.model.dto.GraphTypeDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogAnalyseGraphTypeDO; -import org.mapstruct.Mapper; -import org.mapstruct.Mapping; -import org.mapstruct.Mappings; -import org.mapstruct.factory.Mappers; - -import java.util.List; - -@Mapper -public interface GraphTypeConvert { - - GraphTypeConvert INSTANCE = Mappers.getMapper(GraphTypeConvert.class); - @Mappings({ - @Mapping(target = "typeCode", source = "type") - }) - GraphTypeDTO toDTO(MilogAnalyseGraphTypeDO DO); - - List toDTOList(List DOList); - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/LogProcessConvert.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/LogProcessConvert.java deleted file mode 100644 index 49e01d88c..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/LogProcessConvert.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.convert; - -import com.xiaomi.mone.log.api.model.vo.AgentLogProcessDTO; -import com.xiaomi.mone.log.api.model.vo.TailLogProcessDTO; -import com.xiaomi.mone.log.api.model.vo.UpdateLogProcessCmd; -import com.xiaomi.mone.log.manager.model.cache.LogCellectProcessCache; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogProcessDO; -import org.mapstruct.Mapper; -import org.mapstruct.Mapping; -import org.mapstruct.Mappings; -import org.mapstruct.factory.Mappers; - -import java.util.List; - -@Mapper -public interface LogProcessConvert { - LogProcessConvert INSTANCE = Mappers.getMapper(LogProcessConvert.class); - - AgentLogProcessDTO fromDO(MilogLogProcessDO milogLogProcessDO); - -// @Mappings({ -// @Mapping(source = "collectTime", target = "collectTime") -// }) - AgentLogProcessDTO collectProcessToDTO(UpdateLogProcessCmd.CollectDetail collectDetail); - - List collectProcessToDTOList(List collectDetailList); - - LogCellectProcessCache cmdToCache(UpdateLogProcessCmd.CollectDetail collectDetail); - - List cmdToCacheList(List collectDetailList); - - AgentLogProcessDTO cacheToAgentDTO(LogCellectProcessCache cache); - - List cacheToAgentDTOList(List cacheList); - - @Mappings({ - @Mapping(source = "ip", target = "ip"), - @Mapping(source = "tailName", target = "tailName"), - }) - TailLogProcessDTO cacheToTailDTO(LogCellectProcessCache cache, String ip, String tailName); - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/MilogCollecMetaConvert.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/MilogCollecMetaConvert.java deleted file mode 100644 index 84ca4b261..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/MilogCollecMetaConvert.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.convert; - -import org.mapstruct.Mapper; -import org.mapstruct.factory.Mappers; - -@Mapper -public interface MilogCollecMetaConvert { - MilogCollecMetaConvert INSTANCE = Mappers.getMapper(MilogCollecMetaConvert.class); - - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/MilogLogTemplateConvert.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/MilogLogTemplateConvert.java deleted file mode 100644 index 2df29f78d..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/MilogLogTemplateConvert.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.convert; - -import com.xiaomi.mone.log.manager.model.dto.LogTemplateDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTemplateDO; -import org.mapstruct.Mapper; -import org.mapstruct.Mapping; -import org.mapstruct.Mappings; -import org.mapstruct.factory.Mappers; - -import java.util.List; - -@Mapper -public interface MilogLogTemplateConvert { - MilogLogTemplateConvert INSTANCE = Mappers.getMapper(MilogLogTemplateConvert.class); - - @Mappings({ - @Mapping(target = "value", source = "id"), - @Mapping(target = "label", source = "templateName") - }) - LogTemplateDTO fromDO(MilogLogTemplateDO milogLogTemplateDO); - - List fromDOList(List milogLogTemplateDOList); - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/MilogLogstoreConvert.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/MilogLogstoreConvert.java deleted file mode 100644 index bf953a882..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/MilogLogstoreConvert.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.convert; - -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.vo.LogStoreParam; -import org.mapstruct.Mapper; -import org.mapstruct.Mapping; -import org.mapstruct.Mappings; -import org.mapstruct.factory.Mappers; - -@Mapper -public interface MilogLogstoreConvert { - MilogLogstoreConvert INSTANCE = Mappers.getMapper(MilogLogstoreConvert.class); - - @Mappings({ - @Mapping(target = "esClusterId", source = "esResourceId") - }) - MilogLogStoreDO fromCommand(LogStoreParam command); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/MilogLogtailConvert.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/MilogLogtailConvert.java deleted file mode 100644 index 093a5bb13..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/MilogLogtailConvert.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.convert; - -import com.xiaomi.mone.log.manager.model.bo.LogTailParam; -import com.xiaomi.mone.log.manager.model.dto.LogTailDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import org.mapstruct.Mapper; -import org.mapstruct.factory.Mappers; - -@Mapper -public interface MilogLogtailConvert { - MilogLogtailConvert INSTANCE = Mappers.getMapper(MilogLogtailConvert.class); - - LogTailDTO fromDO(MilogLogTailDo milogLogtailDo); - - MilogLogTailDo fromDo(LogTailParam logTailParam); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/MilogLongTemplateDetailConvert.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/MilogLongTemplateDetailConvert.java deleted file mode 100644 index 9b3f8a26e..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/MilogLongTemplateDetailConvert.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.convert; - -import com.xiaomi.mone.log.manager.model.dto.LogTemplateDetailDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTemplateDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTemplateDetailDO; -import org.mapstruct.Mapper; -import org.mapstruct.Mapping; -import org.mapstruct.Mappings; -import org.mapstruct.factory.Mappers; -@Mapper -public interface MilogLongTemplateDetailConvert { - MilogLongTemplateDetailConvert INSTANCE = Mappers.getMapper(MilogLongTemplateDetailConvert.class); - - @Mappings({ - @Mapping(target = "templateName", source = "templateDO.templateName"), - @Mapping(target = "propertiesKey", source = "detailDO.propertiesKey"), - @Mapping(target = "propertiesType", source = "detailDO.propertiesType") - }) - LogTemplateDetailDTO fromDO(MilogLogTemplateDO templateDO, MilogLogTemplateDetailDO detailDO); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/MilogSpaceConvert.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/MilogSpaceConvert.java deleted file mode 100644 index 07e8ed7e7..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/MilogSpaceConvert.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.convert; - -import com.xiaomi.mone.log.manager.model.dto.MilogSpaceDTO; -import com.xiaomi.mone.log.manager.model.page.PageInfo; -import com.xiaomi.mone.log.manager.model.pojo.MilogSpaceDO; -import com.xiaomi.mone.tpc.common.vo.NodeVo; -import com.xiaomi.mone.tpc.common.vo.PageDataVo; -import org.mapstruct.Mapper; -import org.mapstruct.Mapping; -import org.mapstruct.Mappings; -import org.mapstruct.factory.Mappers; - -import java.util.List; - -@Mapper -public interface MilogSpaceConvert { - MilogSpaceConvert INSTANCE = Mappers.getMapper(MilogSpaceConvert.class); - - MilogSpaceDTO fromDO(MilogSpaceDO milogSpace); - - List fromDOList(List milogSpace); - - PageInfo fromTpcPage(PageDataVo tpcRes); - - @Mappings({ - @Mapping(target = "id", source = "outId"), - @Mapping(target = "spaceName", source = "nodeName"), - @Mapping(target = "ctime", source = "createTime"), - @Mapping(target = "utime", source = "updateTime"), - @Mapping(target = "creator", source = "createrAcc"), - @Mapping(target = "description", source = "desc"), - @Mapping(target = "tpcNodeId", source = "id"), - }) - MilogSpaceDTO fromTpc(NodeVo tpcNode); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/MilogStreamConvert.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/MilogStreamConvert.java deleted file mode 100644 index 4adb5ee67..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/MilogStreamConvert.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.convert; - -import org.mapstruct.Mapper; -import org.mapstruct.factory.Mappers; - -@Mapper -public interface MilogStreamConvert { - MilogStreamConvert INSTANCE = Mappers.getMapper(MilogStreamConvert.class); - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/SearchSaveConvert.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/SearchSaveConvert.java deleted file mode 100644 index 11cdbe3ba..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/SearchSaveConvert.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.convert; - -import com.xiaomi.mone.log.manager.model.dto.SearchSaveDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogSearchSaveDO; -import com.xiaomi.mone.log.manager.model.vo.SearchSaveInsertCmd; -import org.mapstruct.Mapper; -import org.mapstruct.Mapping; -import org.mapstruct.factory.Mappers; - -import java.util.List; - -@Mapper -public interface SearchSaveConvert { - SearchSaveConvert INSTANCE = Mappers.getMapper(SearchSaveConvert.class); - - @Mapping(target = "id", resultType = String.class) - SearchSaveDTO fromDO(MilogLogSearchSaveDO obj); - - List fromDOList(List objList); - - MilogLogSearchSaveDO toDO(SearchSaveInsertCmd obj); - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/UserConvert.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/UserConvert.java deleted file mode 100644 index fa91bccb5..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/convert/UserConvert.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.convert; - -import com.xiaomi.mone.log.manager.user.MoneUser; -import com.xiaomi.mone.tpc.login.vo.AuthUserVo; -import org.mapstruct.Mapper; -import org.mapstruct.Mapping; -import org.mapstruct.Mappings; -import org.mapstruct.factory.Mappers; - -@Mapper -public interface UserConvert { - UserConvert INSTANCE = Mappers.getMapper(UserConvert.class); - - @Mappings({ - @Mapping(target = "user", source = "account"), - @Mapping(target = "displayName", source = "name"), - @Mapping(target = "uID", source = "casUid") - }) - MoneUser userAdapter(AuthUserVo userVo); - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/AppTypeTailDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/AppTypeTailDTO.java deleted file mode 100644 index 21463f275..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/AppTypeTailDTO.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/10/26 19:43 - */ -@Data -public class AppTypeTailDTO { - - private Integer appType; - - private String appTypName; - - private List tailAppList; - - @Data - public static class TailApp { - - private String nameEn; - - private String nameCn; - - private List tailInfos; - } - - @Data - public static class TailInfo { - private Long id; - private String tailName; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/DTTableGetParamDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/DTTableGetParamDTO.java deleted file mode 100644 index 21e3ab214..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/DTTableGetParamDTO.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -/** - * @author zhangjuan - * @Description Query the parameters for the matrix es domain name from the data factory - * @date 2022-06-17 - */ -@Data -public class DTTableGetParamDTO { - private String catalog; - private String dbName; - private String tableNameEn; - private String esToken; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/DTTableGetResponseDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/DTTableGetResponseDTO.java deleted file mode 100644 index 26dd85908..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/DTTableGetResponseDTO.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -/** - * @author zhangjuan - * @Description The matrix es domain name returned by the data factory - * @date 2022-06-17 - */ -@Data -public class DTTableGetResponseDTO { - private String esClusterDomain; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/DashboardDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/DashboardDTO.java deleted file mode 100644 index 2942ea56c..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/DashboardDTO.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -import java.io.Serializable; -import java.util.List; - -@Data -public class DashboardDTO implements Serializable { - - private String dashboardName; - - private String dashboardId; - - private List graphList; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/DashboardGraphDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/DashboardGraphDTO.java deleted file mode 100644 index b9e0c3365..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/DashboardGraphDTO.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -import java.io.Serializable; - -@Data -public class DashboardGraphDTO implements Serializable { - - private Long graphId; - - private String graphPrivateName; - - private String graphName; - - private Integer graphType; - - private String point; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/DictionaryDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/DictionaryDTO.java deleted file mode 100644 index 789939202..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/DictionaryDTO.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/26 15:30 - */ -@Data -@NoArgsConstructor -@AllArgsConstructor -public class DictionaryDTO { - - private T value; - private String label; - - private boolean showDeploymentType; - private boolean showEnvGroup; - private boolean showMachineType; - private boolean showMachineRegion; - private boolean showServiceIp; - private boolean showDeploymentSpace; - private boolean showMqConfig; - - private boolean showOrgInfo; - private boolean showBrokerName; - - private List children; - - public DictionaryDTO(T value, String label, List children) { - this.value = value; - this.label = label; - this.children = children; - } - - public static DictionaryDTO Of(Object label, String value) { - return new DictionaryDTO(label, value, null); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/DishboardDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/DishboardDTO.java deleted file mode 100644 index 289ce91a6..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/DishboardDTO.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -import java.util.List; - -@Data -public class DishboardDTO { - private String dishboardName; - private List graphList; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/DockerScaleBo.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/DockerScaleBo.java deleted file mode 100644 index f4722b261..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/DockerScaleBo.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/8/25 14:44 - */ -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class DockerScaleBo { - private Long projectId; - private Long envId; - private String mioneEnv; - private Integer type; - private List ips; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/EsInfoDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/EsInfoDTO.java deleted file mode 100644 index d47991c07..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/EsInfoDTO.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -@Data -public class EsInfoDTO { - private Long clusterId; - private String index; - - public EsInfoDTO(Long clusterId, String index) { - this.clusterId = clusterId; - this.index = index; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/EsSearchParamDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/EsSearchParamDTO.java deleted file mode 100644 index d9ad0da4c..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/EsSearchParamDTO.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -/** - * @author zhangjuan - * @Description Parameters that query matrix es data - * @date 2022-06-17 - */ -@Data -public class EsSearchParamDTO { - private String esDomain; - private String index; - private String catalog; - private String dbName; - private String esToken; - - public EsSearchParamDTO(String esDomain, String index, String catalog, String dbName, String esToken) { - this.index = index; - this.esDomain = esDomain; - this.catalog = catalog; - this.dbName = dbName; - this.esToken = esToken; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/EsStatisticResult.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/EsStatisticResult.java deleted file mode 100644 index 927baf681..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/EsStatisticResult.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; -import org.apache.commons.collections.CollectionUtils; - -import java.util.List; -import java.util.Objects; - -@Data -public class EsStatisticResult { - private List timestamps; - private List counts; - private String name; - private Long totalCounts; - private Object queryBuilder; - - public void calTotalCounts() { - if (CollectionUtils.isEmpty(counts)) { - this.totalCounts = 0L; - return; - } - this.totalCounts = counts.parallelStream() - .filter(Objects::nonNull) - .reduce(Long::sum).orElse(0L); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/EsStatisticsKeyWord.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/EsStatisticsKeyWord.java deleted file mode 100644 index 83686e034..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/EsStatisticsKeyWord.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/6/9 11:35 - */ -@Data -public class EsStatisticsKeyWord { - - private String key; - - private List statisticsRation; - @Data - public static class StatisticsRation { - private String value; - private String ration; - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/GraphDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/GraphDTO.java deleted file mode 100644 index bfd34f367..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/GraphDTO.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -import java.io.Serializable; - -@Data -public class GraphDTO implements Serializable { - - private String name; - - private String fieldName; - - private String graphType; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/GraphTypeDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/GraphTypeDTO.java deleted file mode 100644 index b893b9171..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/GraphTypeDTO.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -import java.io.Serializable; - -@Data -public class GraphTypeDTO implements Serializable { - - private String name; - - private String typeCode; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogAnalyseDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogAnalyseDTO.java deleted file mode 100644 index 65f1ceb77..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogAnalyseDTO.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -@Data -public class LogAnalyseDTO { - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogAnalyseDataDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogAnalyseDataDTO.java deleted file mode 100644 index 6fcd6b883..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogAnalyseDataDTO.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import java.io.Serializable; - -public abstract class LogAnalyseDataDTO implements Serializable { - -} - diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogAnalyseDataDateDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogAnalyseDataDateDTO.java deleted file mode 100644 index 412815503..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogAnalyseDataDateDTO.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.List; -import java.util.Set; - -@Data -@AllArgsConstructor -@NoArgsConstructor -public class LogAnalyseDataDateDTO extends LogAnalyseDataDTO { - -// data structure -// data: -// [ -// ['2022-10-22', 44, 55, 66, 2], -// ['2022-10-23', 6, 16, 23, 1], -// ['2022-10-24', 6, 16, 23, 1], -// ['2022-10-25', 6, 16, 23, 1] -// ] -// type: -// ['error', 'info', 'close', 'open'] - - private List> data; - - private Set type; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogAnalyseDataFieldDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogAnalyseDataFieldDTO.java deleted file mode 100644 index 7d8903a65..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogAnalyseDataFieldDTO.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@AllArgsConstructor -@NoArgsConstructor -public class LogAnalyseDataFieldDTO extends LogAnalyseDataDTO { - - private String data; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogDTO.java deleted file mode 100644 index beb9f0c47..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogDTO.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; -import org.elasticsearch.search.builder.SearchSourceBuilder; - -import java.io.Serializable; -import java.util.List; - -@Data -public class LogDTO implements Serializable { - private List logDataDTOList; - private Long total; - private Object[] thisSortValue; - private SearchSourceBuilder sourceBuilder; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogDataDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogDataDTO.java deleted file mode 100644 index e2c674219..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogDataDTO.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -@Data -public class LogDataDTO { - private Map logOfKV = new LinkedHashMap<>(); - private String logOfString; - private String fileName; - private String lineNumber; - private String ip; - private String timestamp; - private Map> highlight; - - public void setValue(String key, Object value) { - logOfKV.put(key, value); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogStorageData.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogStorageData.java deleted file mode 100644 index 00fcc29dc..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogStorageData.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/11/10 16:02 - */ -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class LogStorageData { - - private Long clusterId; - - private Long storeId; - - private String logStoreName; - - private String updateStoreName; - - private String keys; - - private Integer logType; - - private String columnTypes; - - private String updateKeys; - - private String updateColumnTypes; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogStoreDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogStoreDTO.java deleted file mode 100644 index 620450ed1..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogStoreDTO.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import lombok.Data; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/12/17 11:38 - */ -@Data -public class LogStoreDTO extends MilogLogStoreDO { - /** - * Log type name Chinese - */ - private String logTypeName; - /** - * The name of the computer room Chinese - */ - private String machineRoomName; - /** - * Whether to select a custom index - */ - private Boolean selectCustomIndex; - - private Long esResourceId ; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogTailDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogTailDTO.java deleted file mode 100644 index 19465a51d..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogTailDTO.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -import java.util.List; - -@Data -public class LogTailDTO { - private Long id; - private Long ctime; - private Long utime; - private Long spaceId; - private Long storeId; - private Long appId; - private Long milogAppId; - private String appName; - private Long envId; - private String envName; - private List ips; - private String tail; - private Integer parseType; - private String parseScript; - private String logPath; - private String logSplitExpress; - private String valueList; - private String tailRate; - /** - * Service deployment space - **/ - private String deploySpace; - - /** - * Row regex - **/ - private String firstLineReg; - - /** - * china - */ - private String source; - /** - * App type 0: Mione app - **/ - private Integer appType; - /** - * Machine Type 0. Container 1. Physical machine - */ - private Integer machineType; - /** - * The node information of the data center where the application resides - */ - private List motorRooms; - - private String topicName; - private List middlewareConfig; - - private Integer deployWay; - - private Integer projectSource; - - private Integer batchSendSize; - - private Boolean collectionReady; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogTemplateDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogTemplateDTO.java deleted file mode 100644 index 9beb140f1..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogTemplateDTO.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -@Data -public class LogTemplateDTO { - private Long value; - private String label; - private Integer type; - private String describe; - - private LogTemplateDetailDTO logTemplateDetailDTOList; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogTemplateDetailDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogTemplateDetailDTO.java deleted file mode 100644 index ac246ec41..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogTemplateDetailDTO.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -@Data -public class LogTemplateDetailDTO { - private String templateName; - private String propertiesKey; - private String propertiesType; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogtailCollectTopDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogtailCollectTopDTO.java deleted file mode 100644 index 164982c87..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogtailCollectTopDTO.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -@Data -public class LogtailCollectTopDTO { - private String tail; - private String number; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogtailCollectTrendDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogtailCollectTrendDTO.java deleted file mode 100644 index 058adf2ff..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/LogtailCollectTrendDTO.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -@Data -public class LogtailCollectTrendDTO { - private String day; - private String number; - private String showNumber; -} - - diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MapChildrenDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MapChildrenDTO.java deleted file mode 100644 index 0e63ce6e9..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MapChildrenDTO.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -@Data -public class MapChildrenDTO { - private K label; - private V children; - - public MapChildrenDTO() { - } - - public MapChildrenDTO(K label, V children) { - this.label = label; - this.children = children; - } - - public static MapChildrenDTO Of(Object label, Object value) { - return new MapChildrenDTO(label, value); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MapDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MapDTO.java deleted file mode 100644 index 9b3e3fbda..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MapDTO.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -@Data -public class MapDTO { - private K label; - private V value; - private String key; - - public MapDTO() { - } - - public MapDTO(K label, V value) { - this.label = label; - this.value = value; - } - - public MapDTO(K label, V value, String key) { - this.label = label; - this.value = value; - this.key = key; - } - - public static MapDTO Of(Object label, Object value) { - return new MapDTO(label, value); - } - - public static MapDTO Of(Object label, Object value, String key) { - return new MapDTO(label, value, key); - } -} \ No newline at end of file diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MenuDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MenuDTO.java deleted file mode 100644 index 7569a468a..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MenuDTO.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/9/22 10:36 - */ -@Data -public class MenuDTO { - private K key; - private V label; - private List> children; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MiddlewareConfigDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MiddlewareConfigDTO.java deleted file mode 100644 index 5546e782a..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MiddlewareConfigDTO.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import com.xiaomi.mone.log.manager.model.pojo.MilogMiddlewareConfig; -import lombok.Data; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/10/21 16:07 - */ -@Data -public class MiddlewareConfigDTO extends MilogMiddlewareConfig { - private List types; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MilogAgentDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MilogAgentDTO.java deleted file mode 100644 index 0ee85ad5b..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MilogAgentDTO.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import com.xiaomi.mone.log.api.model.vo.AgentLogProcessDTO; -import lombok.Data; - -import java.util.List; - -@Data -public class MilogAgentDTO { - private Long id; - private String ip; - private String container; - private Integer state; - private String version; - private Long updateTime; - private String computerRoomName; - private List processList; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MilogAppConfigTailDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MilogAppConfigTailDTO.java deleted file mode 100644 index ec5dc7c53..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MilogAppConfigTailDTO.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import com.xiaomi.mone.log.manager.model.pojo.MilogAppMiddlewareRel; -import lombok.Data; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/24 15:45 - */ -@Data -public class MilogAppConfigTailDTO { - - private Long id; - private String appId; - private String appName; - private String source; - private Integer type; - private Long ctime; - private String creator; - - private List configTailDTOList; - - @Data - public static class ConfigTailDTO { - private Long middlewareId; - private String type; - private String middlewareName; - private Long tailId; - private String tailName; - private String tailCreator; - private Long tailCreateTime; - private Long tailUpdateTime; - private String tailUpdater; - public MilogAppMiddlewareRel.Config mqConfig; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MilogAppEnvDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MilogAppEnvDTO.java deleted file mode 100644 index d2db8eed5..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MilogAppEnvDTO.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.io.Serializable; -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/8/19 11:19 - */ -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class MilogAppEnvDTO implements Serializable { - /** - * Environment name - */ - private String label; - /** - * Environment primary key - */ - private Long value; - /** - * IPS collection in the environment - */ - private List ips; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MilogAppOpenVo.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MilogAppOpenVo.java deleted file mode 100644 index 5638896e0..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MilogAppOpenVo.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Builder; -import lombok.Data; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/14 16:35 - */ -@Data -@Builder -public class MilogAppOpenVo { - private String label; - private Long value; - private String source; - - private Long platform; - private String appId; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MilogSpaceDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MilogSpaceDTO.java deleted file mode 100644 index 1c34e87ae..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MilogSpaceDTO.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -@Data -public class MilogSpaceDTO { - private Long id; - private Long ctime; - private Long utime; - private Long tenantId; - private String source; - private String tenantName; - private String spaceName; - private String creator; - private String description; - private String permDeptId; - private String createDeptId; - private Long tpcNodeId; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MilogSpaceTreeDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MilogSpaceTreeDTO.java deleted file mode 100644 index 6e927edb7..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MilogSpaceTreeDTO.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.List; - -@Data -@NoArgsConstructor -@AllArgsConstructor -public class MilogSpaceTreeDTO { - private String label; - private Long value; - private List> children; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MotorRoomDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MotorRoomDTO.java deleted file mode 100644 index ccf551b0b..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/MotorRoomDTO.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/10/19 15:59 - */ -@Data -public class MotorRoomDTO { - - private String nameEn; - - private List podDTOList; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/PermTreeDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/PermTreeDTO.java deleted file mode 100644 index 1b17a2bd0..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/PermTreeDTO.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -import java.util.List; - -@Data -public class PermTreeDTO { - private String id; - private String label; - public Integer deptLevel; - private List children; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/PodDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/PodDTO.java deleted file mode 100644 index 8336a96b4..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/PodDTO.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.*; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -@EqualsAndHashCode -public class PodDTO { - /** - * pod name - */ - private String podName; - /** - * pod IP - */ - private String podIP; - /** - * Physical machine name - */ - private String nodeName; - /** - * Physical IP - */ - private String nodeIP; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/ProjectInfo.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/ProjectInfo.java deleted file mode 100644 index bd4bb5bff..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/ProjectInfo.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -import java.io.Serializable; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/15 11:50 - */ -@Data -public class ProjectInfo implements Serializable { - private Long id; - - private String name; - - private String mioneEnv; - - private String desc; - - private long ctime; - - private long utime; - - private String domain; - /** - * Project Importance 1. Level 1 Project 2.2 Level Project Level 3.3 Project - */ - private Integer importLevel; - - private Long iamTreeId; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/Properties.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/Properties.java deleted file mode 100644 index 1f5244085..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/Properties.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -@Data -public class Properties { - private String broker; - private String group; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/RadarAppInfoDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/RadarAppInfoDTO.java deleted file mode 100644 index 8e866a83c..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/RadarAppInfoDTO.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import com.google.gson.annotations.SerializedName; -import lombok.Data; - -import java.util.List; - -/** - * @Auther: wtt - * @Date: 2022/3/24 10:36 - * @Description: - */ -@Data -public class RadarAppInfoDTO { - private Long id; - private String name; - private List members; - private String createTime; - private String updateTime; - private boolean joined; - - - @Data - public static class Member { - @SerializedName(value = "user_id") - private String userId; - @SerializedName(value = "user_name") - private String userName; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/RadarResponseDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/RadarResponseDTO.java deleted file mode 100644 index 075e186e1..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/RadarResponseDTO.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/10/13 15:33 - */ -@Data -public class RadarResponseDTO { - private String code; - private String message; - private RadarData data; - - @Data - public static class RadarData { - private Integer pageSize; - private Integer total; - private Integer page; - private Integer pageIndex; - private T list; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/RegionDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/RegionDTO.java deleted file mode 100644 index 2a2f3666d..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/RegionDTO.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Builder; -import lombok.Data; - -import java.util.List; - -@Data -@Builder -public class RegionDTO { - - /** - * English name - */ - private String regionNameEN; - - /** - * Chinese name - */ - private String regionNameCN; - - /** - * Zone collection - */ - private List zoneDTOList; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/RmqStatisticRulst.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/RmqStatisticRulst.java deleted file mode 100644 index 5812bf6c8..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/RmqStatisticRulst.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -import java.util.Map; - -@Data -public class RmqStatisticRulst { - private String cluster; - private String topic; - private Properties properties; - private Map timestamps; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/RocketMqResponseDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/RocketMqResponseDTO.java deleted file mode 100644 index adc337a9b..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/RocketMqResponseDTO.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -@Data -public class RocketMqResponseDTO { - private Integer code; - private String message; - private String requestId; - private String cost; - private T data; - - - @Data - public static class SubGroup { - private String orgId; - private String name; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/RocketMqStatisticDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/RocketMqStatisticDTO.java deleted file mode 100644 index deff4ef29..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/RocketMqStatisticDTO.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -import java.util.List; - -@Data -public class RocketMqStatisticDTO { - private String requestId; - private String cost; - private List data; -} \ No newline at end of file diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/SearchSaveDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/SearchSaveDTO.java deleted file mode 100644 index e78e1cada..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/SearchSaveDTO.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -@Data -public class SearchSaveDTO { - private String id; - private String name; - private Long spaceId; - private String spaceName; - private Long storeId; - private String storeName; - private String tailId; - private String tailName; - private String appName; - private String queryText; - private Integer isFixTime; - private Long startTime; - private Long endTime; - private Long sort; - private Integer orderNum; - private String common; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/SimpleAppEnvDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/SimpleAppEnvDTO.java deleted file mode 100644 index 33a90af35..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/SimpleAppEnvDTO.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -import java.io.Serializable; -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/10/18 14:18 - */ -@Data -public class SimpleAppEnvDTO implements Serializable { - private String nameEn; - private String nameCn; - private List podDTOList; - private List nodeIps; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/SpaceCollectTopDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/SpaceCollectTopDTO.java deleted file mode 100644 index 23e1e2433..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/SpaceCollectTopDTO.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -@Data -public class SpaceCollectTopDTO { - private String spaceName; - private String orgName; - private String number; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/SpaceCollectTrendDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/SpaceCollectTrendDTO.java deleted file mode 100644 index 067b0890f..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/SpaceCollectTrendDTO.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -@Data -public class SpaceCollectTrendDTO { - private String spaceName; - private String orgName; - private String day; - private String number; - private String showNumber; - -} - - diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/SpacePermTreeDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/SpacePermTreeDTO.java deleted file mode 100644 index 3d170081a..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/SpacePermTreeDTO.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - - -@Data -public class SpacePermTreeDTO { - private String checkId; - private String createDeptId; - private PermTreeDTO treeData; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/SpaceTreeFavouriteDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/SpaceTreeFavouriteDTO.java deleted file mode 100644 index 4d21046c5..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/SpaceTreeFavouriteDTO.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -import java.util.List; - -@Data -public class SpaceTreeFavouriteDTO { - private String label; - private Long value; - private List children; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/StatisticRulst.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/StatisticRulst.java deleted file mode 100644 index 02eb97a3f..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/StatisticRulst.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -import java.util.Map; - -@Data -public class StatisticRulst { - private String name; - private Map timestamps; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/StoreTreeDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/StoreTreeDTO.java deleted file mode 100644 index dc80b616e..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/StoreTreeDTO.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.AllArgsConstructor; -import lombok.Data; - -@Data -@AllArgsConstructor -public class StoreTreeDTO { - private Long value; - private String label; - private Integer isFavourite; - - public static StoreTreeDTO Of (Long value, String label, Integer isFavourite) { - return new StoreTreeDTO(value, label, isFavourite); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/TopicInfo.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/TopicInfo.java deleted file mode 100644 index e1efcfd5e..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/TopicInfo.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/14 17:54 - */ -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class TopicInfo { - private String orgId; - private String name; - private Integer queueTotalCount; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/UnAccessAppDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/UnAccessAppDTO.java deleted file mode 100644 index 90910cfc1..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/UnAccessAppDTO.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@AllArgsConstructor -public class UnAccessAppDTO { - private Long milogAppId; - private String appName; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/ValueDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/ValueDTO.java deleted file mode 100644 index e8d5fccf7..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/ValueDTO.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Data; - -@Data -public class ValueDTO { - private V value; - - public ValueDTO() { - } - - public ValueDTO(V value) { - this.value = value; - } - - public static ValueDTO Of(Object label, Object value) { - return new ValueDTO(value); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/ZoneDTO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/ZoneDTO.java deleted file mode 100644 index 188df1434..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/dto/ZoneDTO.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.dto; - -import lombok.Builder; -import lombok.Data; - -import java.util.List; - -@Data -@Builder -public class ZoneDTO { - - /** - * english name - */ - private String zoneNameEN; - - /** - * chinese name - */ - private String zoneNameCN; - - /** - * pod collect - */ - private List podDTOList; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/page/PageInfo.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/page/PageInfo.java deleted file mode 100644 index 7914dcc7c..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/page/PageInfo.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.page; - -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/22 11:48 - */ -@Data -@NoArgsConstructor -public class PageInfo implements Serializable { - private Integer page; - private Integer pageSize; - private Integer total; - private Integer totalPageCount; - private List list; - - public static PageInfo emptyPageInfo() { - return new PageInfo(0, 0, 0, Collections.emptyList()); - } - - - public PageInfo(int page, int pageSize, int total, List list) { - this.page = page; - this.pageSize = pageSize; - this.total = total; - initPage(page, pageSize, total, list, true); - } - - /** - * @param page // How many pages are current - * @param pageSize // Per page size - * @param total // total - * @param list // Data carriers - * @param hasPaged // Whether pagination has been made - */ - public PageInfo(int page, int pageSize, int total, List list, boolean hasPaged) { - this.page = page; - this.pageSize = pageSize; - this.total = total; - initPage(page, pageSize, total, list, hasPaged); - } - - private void initPage(int pageIndex, int pageSize, int totalItemCount, List data, boolean hasPaged) { - // Calculate the total number of pages - if (totalItemCount > 0 && pageSize > 0) { - this.totalPageCount = totalItemCount / pageSize + 1; - } - // Intercepts the current page data - if (null == data) { - this.list = new ArrayList(); - } else if (data.isEmpty() || hasPaged) { - this.list = data; - } else { - if (pageIndex == this.totalPageCount) { // Last page - this.list = data.subList((pageIndex - 1) * pageSize, data.size()); - } else if (pageIndex < this.totalPageCount) { // Not the last page - this.list = data.subList((pageIndex - 1) * pageSize, pageIndex * pageSize); - } else { // The current page exceeds the total number of pages - throw new IndexOutOfBoundsException("The current number of pages exceeds the total number of pages"); - } - } - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/ConfigPushData.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/ConfigPushData.java deleted file mode 100644 index 5b222a71b..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/ConfigPushData.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import lombok.Data; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/15 15:22 - */ -@Data -public class ConfigPushData { - - private Long id; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/LogCountDO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/LogCountDO.java deleted file mode 100644 index feb2d24fc..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/LogCountDO.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.io.Serializable; - -/** - *

- * - *

- * - * @author wanghaoyang - * @since 2022-01-07 - */ -@Data -@EqualsAndHashCode(callSuper = false) -@TableName("milog_log_count") -public class LogCountDO implements Serializable { - - private static final long serialVersionUID = 1L; - @TableId(value = "id", type = IdType.AUTO) - private Long id; - - /** - * tail id - */ - private Long tailId; - - /** - * es index name - */ - private String esIndex; - - /** - * The log data is generated on the day yyyy-MM-dd - */ - private String day; - - /** - * Number of logs - */ - private Long number; - - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/LogStreamBalanceConfig.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/LogStreamBalanceConfig.java deleted file mode 100644 index 3bf50168d..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/LogStreamBalanceConfig.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import com.xiaomi.mone.log.manager.model.BaseCommon; -import lombok.Data; -import org.nutz.dao.entity.annotation.*; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/10/16 15:22 - */ -@Table("milog_stream_balance") -@Comment("日志消费均衡器") -@Data -public class LogStreamBalanceConfig extends BaseCommon { - - @Id - @Comment("主键Id") - @ColDefine(customType = "bigint") - private Long id; - - @Column(value = "machine_room") - @ColDefine(type = ColType.VARCHAR) - @Comment("区域") - private String machineRoom; - - @Column(value = "stream_server_name") - @ColDefine(type = ColType.VARCHAR) - @Comment("stream启动的服务名") - private String streamServerName; - - @Column(value = "stream_balance_data_id") - @ColDefine(type = ColType.VARCHAR) - @Comment("nacos中的分配stream的dataId") - private String streamBalanceDataId; - - @Column(value = "old_stream_key") - @ColDefine(type = ColType.VARCHAR) - @Comment("old-stream的key") - private String oldStreamKey; - - @Column(value = "status") - @ColDefine(type = ColType.INT, width = 10) - @Comment("0.开启 1.关闭") - private Integer status; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MiLogMachine.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MiLogMachine.java deleted file mode 100644 index 370dc2408..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MiLogMachine.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import lombok.Data; -import org.nutz.dao.entity.annotation.*; - -/** - * @author wtt - * @version 1.0 - * @description Machine information - * @date 2021/7/16 11:26 - */ -@Table("milog_machine") -@Comment("Machine information in milog") -@Data -public class MiLogMachine { - @Id - @Comment("Primary key Id") - @ColDefine(customType = "bigint") - private Long id; - - @Column(value = "ctime") - @ColDefine(customType = "bigint") - @Comment("Creation time") - private Long ctime; - - @Column(value = "utime") - @ColDefine(customType = "bigint") - @Comment("Update time") - private Long utime; - - @Column(value = "type") - @ColDefine(type = ColType.INT) - @Comment("Machine resolution type: 1: agent, 2.stream") - private Integer type; - - @Column(value = "ip") - @ColDefine(type = ColType.TEXT) - @Comment("machine ip") - private String ip; - - @Column(value = "creator") - @ColDefine(type = ColType.VARCHAR, width = 128) - @Comment("Creator") - private String creator; - - @Column(value = "description") - @ColDefine(type = ColType.VARCHAR, width = 1024) - @Comment("Remarks description") - private String description; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogAgentDO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogAgentDO.java deleted file mode 100644 index 317ef2743..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogAgentDO.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import lombok.Data; -import org.nutz.dao.entity.annotation.*; - -@Table("milog_log_agent") -@Comment("milog Agent") -@Data -public class MilogAgentDO { - @Id - @Comment("Primary key Id") - @ColDefine(customType = "bigint") - private Long id; - - @Column(value = "ip") - @ColDefine(type = ColType.VARCHAR, width = 128) - @Comment("agent ip") - private String ip; - - @Column(value = "computer_room_id") - @ColDefine(customType = "bigint") - @Comment("The ID of the data center") - private Long computerRoomId; - - @Column(value = "container") - @ColDefine(type = ColType.VARCHAR, width = 128) - @Comment("container") - private String container; - - @Column(value = "state") - @ColDefine(type = ColType.INT) - @Comment("Agent status") - private Integer state; - - @Column(value = "version") - @ColDefine(type = ColType.VARCHAR, width = 128) - @Comment("Agent version") - private String version; - - @Column(value = "updateTime") - @ColDefine(customType = "bigint") - @Comment("Version update time") - private Long updateTime; - - @Column(value = "ctime") - @ColDefine(customType = "bigint") - @Comment("Creation time") - private Long ctime; - - @Column(value = "utime") - @ColDefine(customType = "bigint") - @Comment("Update time") - private Long utime; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogAnalyseDashboardDO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogAnalyseDashboardDO.java deleted file mode 100644 index aa3046c94..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogAnalyseDashboardDO.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.io.Serializable; - -/** - *

- * - *

- * - * @author wanghaoyang - * @since 2022-08-18 - */ -@Data -@EqualsAndHashCode(callSuper = false) -@TableName("milog_analyse_dashboard") -public class MilogAnalyseDashboardDO implements Serializable { - - private static final long serialVersionUID = 1L; - - @TableId(value = "id", type = IdType.AUTO) - private Long id; - - private String name; - - private Long storeId; - - private Long spaceId; - - private String creator; - - private String updater; - - private Long createTime; - - private Long updateTime; - - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogAnalyseDashboardGraphRefDO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogAnalyseDashboardGraphRefDO.java deleted file mode 100644 index 9270c4238..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogAnalyseDashboardGraphRefDO.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.io.Serializable; - -/** - *

- * - *

- * - * @author wanghaoyang - * @since 2022-08-18 - */ -@Data -@EqualsAndHashCode(callSuper = false) -@TableName("milog_analyse_dashboard_graph_ref") -public class MilogAnalyseDashboardGraphRefDO implements Serializable { - - private static final long serialVersionUID = 1L; - - @TableId(value = "id", type = IdType.AUTO) - private Long id; - - private Long dashboardId; - - private Long graphId; - - private String point; - - private String privateName; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogAnalyseGraphDO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogAnalyseGraphDO.java deleted file mode 100644 index 3f6505ccb..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogAnalyseGraphDO.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.io.Serializable; - -/** - *

- * - *

- * - * @author wanghaoyang - * @since 2022-08-18 - */ -@Data -@EqualsAndHashCode(callSuper = false) -@TableName("milog_analyse_graph") -public class MilogAnalyseGraphDO implements Serializable { - - private static final long serialVersionUID = 1L; - - @TableId(value = "id", type = IdType.AUTO) - private Long id; - - private String name; - - private String fieldName; - - private Integer graphType; - - private String graphParam; - - private Long spaceId; - - private Long storeId; - - private String creator; - - private String updater; - - private Long createTime; - - private Long updateTime; - - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogAnalyseGraphTypeDO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogAnalyseGraphTypeDO.java deleted file mode 100644 index 5d9c9faf7..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogAnalyseGraphTypeDO.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.io.Serializable; - -/** - *

- * - *

- * - * @author wanghaoyang - * @since 2022-08-18 - */ -@Data -@EqualsAndHashCode(callSuper = false) -@TableName("milog_analyse_graph_type") -public class MilogAnalyseGraphTypeDO implements Serializable { - - private static final long serialVersionUID = 1L; - - @TableId(value = "id", type = IdType.AUTO) - private Long id; - - private String name; - - private Integer type; - - private String calculate; - - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogAppMiddlewareRel.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogAppMiddlewareRel.java deleted file mode 100644 index 44e142977..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogAppMiddlewareRel.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import com.xiaomi.mone.log.manager.model.BaseCommon; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.nutz.dao.entity.annotation.*; -import org.nutz.json.JsonField; - -import java.io.Serializable; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/17 16:55 - */ -@Table("milog_app_middleware_rel") -@Comment("The application and middleware configuration association table") -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class MilogAppMiddlewareRel extends BaseCommon implements Serializable { - - @Id - @Comment("Primary key Id") - @ColDefine(customType = "bigint") - private Long id; - - @Column(value = "milog_app_id") - @ColDefine(customType = "bigint") - @Comment("milog app talble Primary key Id") - private Long milogAppId; - - @Column(value = "middleware_id") - @ColDefine(customType = "bigint") - @Comment("The middleware configuration table ID") - private Long middlewareId; - - @Column(value = "tail_id") - @ColDefine(customType = "bigint") - @Comment("Collect the trail ID of the log path") - private Long tailId; - - @Column(value = "config") - @ColDefine(type = ColType.MYSQL_JSON) - @Comment("Configuration information, in JSON format") - @JsonField - private Config config; - - @Data - public static class Config implements Serializable { - - private String topic; - - private String consumerGroup; - - private String tag; - - private Integer partitionCnt; - - /** - * es consumption group, which can be extended to other groups for other analysis scenarios - */ - private String esConsumerGroup; - - private Integer batchSendSize; - - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogAppTopicRelDO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogAppTopicRelDO.java deleted file mode 100644 index 6f8b80746..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogAppTopicRelDO.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import com.xiaomi.mone.log.manager.model.BaseCommon; -import lombok.*; -import org.nutz.dao.entity.annotation.*; - -import java.util.LinkedHashMap; -import java.util.List; - -/** - * @author shanwb - * @date 2021-06-28 - */ -@Table("milog_app_topic_rel") -@Comment("The application association table with MQ") -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -@EqualsAndHashCode -public class MilogAppTopicRelDO extends BaseCommon { - @Id - @Comment("Primary key Id") - @ColDefine(customType = "bigint") - private Long id; - - @Column(value = "tenant_id") - @ColDefine(customType = "bigint") - @Comment("Tenant ID") - private Long tenantId; - - @Column(value = "iam_tree_id") - @ColDefine(customType = "bigint") - @Comment("IAM tree ID (unique to the mione app)") - private Long iamTreeId; - - @Column(value = "app_id") - @ColDefine(customType = "bigint") - @Comment("app id") - private Long appId; - - @Column(value = "app_name") - @ColDefine(type = ColType.VARCHAR, width = 256) - @Comment("app name") - private String appName; - - @Column(value = "tree_ids") - @ColDefine(type = ColType.MYSQL_JSON) - @Comment("The IDS of the mounted tree of the project") - private List treeIds; - - @Column(value = "node_ips") - @ColDefine(type = ColType.MYSQL_JSON) - @Comment("The IPS of the physical machine where the application resides") - private LinkedHashMap> nodeIPs; - - @Column(value = "operator") - @ColDefine(type = ColType.VARCHAR, width = 128) - @Comment("operator") - private String operator; - - @Column(value = "source") - @ColDefine(type = ColType.VARCHAR, width = 64) - @Comment("Service source") - private String source; - - @Column(value = "type") - @ColDefine(type = ColType.VARCHAR, width = 10) - @Comment("0.mione project") - private Integer type; - - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogComputerRoomDO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogComputerRoomDO.java deleted file mode 100644 index eda238c94..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogComputerRoomDO.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import lombok.Data; -import org.nutz.dao.entity.annotation.*; - -@Table("milog_log_computer_room") -@Comment("machine Room") -@Data -public class MilogComputerRoomDO { - @Id - @Comment("Primary key Id") - @ColDefine(customType = "bigint") - private Long id; - - @Column(value = "room_name") - @ColDefine(type = ColType.VARCHAR, width = 128) - @Comment("machine name") - private String roomName; - - @Column(value = "room_type") - @ColDefine(type = ColType.INT) - @Comment("machine room type") - private Integer roomType; - - @Column(value = "ctime") - @ColDefine(customType = "bigint") - @Comment("create tine") - private Long ctime; - - @Column(value = "utime") - @ColDefine(customType = "bigint") - @Comment("update time") - private Long utime; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogEsClusterDO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogEsClusterDO.java deleted file mode 100644 index 20b2f9b8c..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogEsClusterDO.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableField; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import com.baomidou.mybatisplus.extension.handlers.FastjsonTypeHandler; -import com.xiaomi.mone.log.api.enums.MiddlewareEnum; -import com.xiaomi.mone.log.api.model.bo.MiLogResource; -import com.xiaomi.mone.log.api.model.vo.EsIndexVo; -import com.xiaomi.mone.log.api.model.vo.ResourceInfo; -import com.xiaomi.mone.log.manager.model.BaseCommon; -import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionServiceFactory; -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.nutz.dao.entity.annotation.ColDefine; -import org.nutz.dao.entity.annotation.ColType; -import org.nutz.dao.entity.annotation.Column; -import org.nutz.dao.entity.annotation.Comment; - -import java.io.Serializable; -import java.util.List; - -/** - *

- * - *

- * - * @author wanghaoyang - * @since 2021-09-18 - */ -@Data -@EqualsAndHashCode(callSuper = false) -@TableName(value = "milog_es_cluster", autoResultMap = true) -public class MilogEsClusterDO extends BaseCommon implements Serializable { - - private static final long serialVersionUID = 1L; - - @TableId(value = "id", type = IdType.AUTO) - private Long id; - - @Column(value = "log_storage_type") - @ColDefine(type = ColType.VARCHAR, width = 50) - @Comment("Log storage type") - private String logStorageType; - - /** - * Cluster type - */ - private String tag; - - /** - * Cluster name - */ - private String name; - - /** - * Cluster name - */ - private String clusterName; - - /** - * room - */ - private String region; - - /** - * area - */ - private String area; - - /** - * ES address - */ - private String addr; - - /** - * ES username - */ - private String user; - - /** - * ES password - */ - private String pwd; - - private String token; - - private String dtCatalog; - - private String dtDatabase; - - private String conWay; - - private Integer isDefault; - - @TableField(typeHandler = FastjsonTypeHandler.class) - private List labels; - - public static MilogMiddlewareConfig miLogEsResourceToConfig(MilogEsClusterDO resource) { - MilogMiddlewareConfig milogMiddlewareConfig = new MilogMiddlewareConfig(); - milogMiddlewareConfig.setId(resource.getId()); - milogMiddlewareConfig.setAlias(resource.getName()); - milogMiddlewareConfig.setClusterName(resource.clusterName); - milogMiddlewareConfig.setType(MiddlewareEnum.ELASTICSEARCH.getCode()); - milogMiddlewareConfig.setStorageType(resource.logStorageType); - milogMiddlewareConfig.setRegionEn(resource.getArea()); - milogMiddlewareConfig.setServiceUrl(resource.getAddr()); - milogMiddlewareConfig.setAk(resource.getUser()); - milogMiddlewareConfig.setSk(resource.getPwd()); - milogMiddlewareConfig.setLabels(resource.getLabels()); - milogMiddlewareConfig.setCreator(resource.getCreator()); - milogMiddlewareConfig.setUpdater(resource.getUpdater()); - milogMiddlewareConfig.setUtime(resource.getUtime()); - milogMiddlewareConfig.setCtime(resource.getCtime()); - milogMiddlewareConfig.setIsDefault(resource.getIsDefault()); - return milogMiddlewareConfig; - } - - - public ResourceInfo configToResourceVO(List multipleEsIndex) { - return ResourceInfo.builder().id(this.id) - .alias(this.name) - .storageType(this.logStorageType) - .clusterName(this.clusterName) - .regionEn(this.area) - .regionCn(CommonExtensionServiceFactory.getCommonExtensionService().getMachineRoomName(this.area)) - .serviceUrl(this.addr) - .ak(this.user) - .sk(this.pwd) - .labels(this.labels) - .esToken(this.token) - .conWay(this.conWay) - .catalog(this.dtCatalog) - .database(this.dtDatabase) - .multipleEsIndex(multipleEsIndex) - .ctime(this.getCtime()) - .utime(this.getUtime()) - .creator(this.getCreator()) - .updater(this.getUpdater()).build(); - } - - public static MilogEsClusterDO miLogEsResourceToConfig(MiLogResource resource) { - MilogEsClusterDO clusterDO = new MilogEsClusterDO(); - clusterDO.setName(resource.getAlias()); - clusterDO.setClusterName(resource.getClusterName()); - clusterDO.setArea(resource.getRegionEn()); - clusterDO.setAddr(resource.getServiceUrl()); - clusterDO.setUser(resource.getAk()); - clusterDO.setPwd(resource.getSk()); - clusterDO.setLabels(resource.getLabels()); - clusterDO.setConWay(resource.getConWay()); - clusterDO.setToken(resource.getEsToken()); - clusterDO.setDtCatalog(resource.getCatalog()); - clusterDO.setDtDatabase(resource.getDatabase()); - clusterDO.setLogStorageType(resource.getStorageType()); - return clusterDO; - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogEsIndexDO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogEsIndexDO.java deleted file mode 100644 index b2e781b81..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogEsIndexDO.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import com.xiaomi.mone.log.api.model.vo.EsIndexVo; -import com.xiaomi.mone.log.manager.domain.ClusterIndexVO; -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.nutz.dao.entity.annotation.Column; - -import java.io.Serializable; -import java.util.List; -import java.util.stream.Collectors; - -/** - *

- * - *

- * - * @author wanghaoyang - * @since 2021-09-18 - */ -@Data -@EqualsAndHashCode(callSuper = false) -@TableName("milog_es_index") -public class MilogEsIndexDO implements Serializable { - - private static final long serialVersionUID = -4651856748263697198L; - /** - * primary key - */ - @TableId(value = "id", type = IdType.AUTO) - @Column(value = "id") - private Long id; - - /** - * The ID of the cluster to which it belongs - */ - @Column(value = "cluster_id") - private Long clusterId; - - /** - * Log type - */ - @Column(value = "log_type") - private Integer logType; - - /** - * es index name - */ - @Column(value = "index_name") - private String indexName; - - - public static List essIndexVoToIndexDO(Long clusterId, EsIndexVo esIndexVo) { - - return esIndexVo.getEsIndexList().stream().map(esIndex -> { - MilogEsIndexDO milogEsIndexDO = new MilogEsIndexDO(); - milogEsIndexDO.setClusterId(clusterId); - milogEsIndexDO.setLogType(esIndexVo.getLogTypeCode()); - milogEsIndexDO.setIndexName(esIndex); - return milogEsIndexDO; - }).collect(Collectors.toList()); - } - - public ClusterIndexVO toClusterIndexVO() { - return ClusterIndexVO.builder().clusterId(this.getClusterId()).indexName(this.getIndexName()).build(); - } - - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogNumAlertDO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogNumAlertDO.java deleted file mode 100644 index 3b0e1deae..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogNumAlertDO.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import lombok.Data; - -@Data -@TableName("milog_log_num_alert") -public class MilogLogNumAlertDO { - private static final long serialVersionUID = 1L; - @TableId(value = "id", type = IdType.AUTO) - private Long id; - - private String day; - - private Long number; - - private Long appId; - - private String appName; - - private String alertUser; - - private Long ctime; - - public MilogLogNumAlertDO(String day, Long number, Long appId, String appName, String alertUser, Long ctime) { - this.day = day; - this.number = number; - this.appId = appId; - this.appName = appName; - this.alertUser = alertUser; - this.ctime = ctime; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogProcessDO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogProcessDO.java deleted file mode 100644 index 71021b250..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogProcessDO.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import lombok.Data; -import org.nutz.dao.entity.annotation.*; - -@Table("milog_log_process") -@Comment("Log collection progress") -@Data -public class MilogLogProcessDO { - @Id - @Comment("Primary key Id") - @ColDefine(customType = "bigint") - private Long id; - - @Column(value = "tailId") - @ColDefine(customType = "bigint") - @Comment("tailId") - private Long tailId; - - @Column(value = "agent_id") - @ColDefine(customType = "bigint") - @Comment("agentId") - private Long agentId; - - @Column(value = "ip") - @ColDefine(type = ColType.VARCHAR, width = 256) - @Comment("ip") - private String ip; - - @Column(value = "file_row_number") - @ColDefine(type = ColType.INT) - @Comment("Log file line number") - private Integer fileRowNumber; - - @Column(value = "pointer") - @ColDefine(type = ColType.INT) - private Long pointer; - - @Column(value = "collect_time") - @ColDefine(customType = "bigint") - @Comment("Log collection time") - private Long collectTime; - - @Column(value = "ctime") - @ColDefine(customType = "bigint") - @Comment("Creation time") - private Long ctime; - - @Column(value = "utime") - @ColDefine(customType = "bigint") - @Comment("Update time") - private Long utime; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogProcessDOMybatis.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogProcessDOMybatis.java deleted file mode 100644 index 730536bd2..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogProcessDOMybatis.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import lombok.Data; -import lombok.experimental.Accessors; - -@Data -@Accessors(chain=true) -@TableName("milog_log_process") -public class MilogLogProcessDOMybatis { - @TableId(type= IdType.AUTO) - private Long id; - private Long agentId; - private String ip; - private Integer fileRowNumber; - private Long pointer; - private Long collectTime; - private Long ctime; - private Long utime; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogSearchSaveDO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogSearchSaveDO.java deleted file mode 100644 index bedd9f245..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogSearchSaveDO.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import com.baomidou.mybatisplus.annotation.TableName; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.io.Serializable; - -/** - *

- * - *

- * - * @author wanghaoyang - * @since 2022-03-29 - */ -@Data -@EqualsAndHashCode(callSuper = false) -@TableName("milog_log_search_save") -public class MilogLogSearchSaveDO implements Serializable { - - private static final long serialVersionUID = 1L; - - private Long id; - - private String name; - - private Long spaceId; - - private Long storeId; - - private Long tailId; - - private String queryText; - - /** - * 1 - the time parameter is saved; 0 - Not saved - */ - private Integer isFixTime; - - private Long startTime; - - private Long endTime; - - /** - * remark - */ - private String common; - - private Integer sort; - - private Integer orderNum; - - private String creator; - - private String updater; - - private Long createTime; - - private Long updateTime; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogStoreDO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogStoreDO.java deleted file mode 100644 index 2904a19b7..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogStoreDO.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import com.xiaomi.mone.log.manager.model.BaseCommon; -import lombok.Data; -import org.nutz.dao.entity.annotation.*; - -/** - * @author shanwb - * @date 2021-06-28 - */ - -@Table("milog_logstore") -@Comment("Milog log storage") -@Data -public class MilogLogStoreDO extends BaseCommon { - @Id - @Comment("Primary key Id") - @ColDefine(customType = "bigint") - private Long id; - - @Column(value = "space_id") - @ColDefine(customType = "bigint") - @Comment("spaceId") - private Long spaceId; - - @Column(value = "mq_resource_id") - @ColDefine(customType = "bigint") - @Comment("The primary key ID of the mq resource") - private Long mqResourceId; - - - @Column(value = "logstoreName") - @ColDefine(type = ColType.VARCHAR, width = 256) - @Comment("Log storage name") - private String logstoreName; - - - @Column(value = "store_period") - @ColDefine(type = ColType.INT) - @Comment("Storage period: 1-3-5-7") - private Integer storePeriod; - - - @Column(value = "shard_cnt") - @ColDefine(type = ColType.INT) - @Comment("Number of storage shards") - private Integer shardCnt; - - @Column(value = "key_list") - @ColDefine(type = ColType.VARCHAR, width = 1024) - @Comment("A list of keys, multiple separated by commas") - private String keyList; - - @Column(value = "column_type_list") - @ColDefine(type = ColType.VARCHAR, width = 128) - @Comment("Field types, multiple separated by commas") - private String columnTypeList; - - @Column(value = "log_type") - @ColDefine(type = ColType.INT) - @Comment("1:app,2:ngx..") - private Integer logType; - - @Column(value = "is_matrix_app") - @ColDefine(type = ColType.BOOLEAN) - @Default("false") - private Boolean usePlatformResource; - - @Column(value = "es_index") - @ColDefine(type = ColType.VARCHAR, width = 256) - @Comment("es index:milog_logstoreName") - private String esIndex; - - @Column(value = "es_cluster_id") - @ColDefine(customType = "bigint") - @Comment("logstore corresponds to the ES cluster ID") - private Long esClusterId; - - @Column(value = "machine_room") - @ColDefine(type = ColType.VARCHAR, width = 50) - @Comment("Computer room information") - private String machineRoom; - - public MilogLogStoreDO() { - } - - public MilogLogStoreDO(Long spaceId, String logstoreName, Integer storePeriod, Integer shardCnt, String keyList, Integer logType, Boolean usePlatformResource) { - this.logstoreName = logstoreName; - this.spaceId = spaceId; - this.storePeriod = storePeriod; - this.shardCnt = shardCnt; - this.keyList = keyList; - this.logType = logType; - this.usePlatformResource = usePlatformResource; - } - - public boolean isPlatformResourceStore() { - return this.usePlatformResource == null ? false : this.usePlatformResource; - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogTailDo.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogTailDo.java deleted file mode 100644 index 2fe885853..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogTailDo.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import com.alibaba.fastjson.annotation.JSONField; -import com.xiaomi.mone.log.api.model.meta.FilterDefine; -import com.xiaomi.mone.log.manager.model.BaseCommon; -import com.xiaomi.mone.log.manager.model.dto.MotorRoomDTO; -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.nutz.dao.entity.annotation.*; -import org.nutz.json.JsonField; - -import java.util.List; - -/** - * @author shanwb - * @date 2021-06-28 - */ - -@Table("milog_logstail") -@Comment("Milog log collection") -@Data -@EqualsAndHashCode -public class MilogLogTailDo extends BaseCommon { - @Id - @Comment("Primary key Id") - @ColDefine(customType = "bigint") - private Long id; - - @Column(value = "tail") - @ColDefine(type = ColType.VARCHAR) - @Comment("App aliases") - private String tail; - - @Column(value = "space_id") - @ColDefine(customType = "bigint") - @Comment("spaceId") - private Long spaceId; - - @Column(value = "store_id") - @ColDefine(customType = "bigint") - @Comment("storeId") - private Long storeId; - - @Column(value = "milog_app_id") - @ColDefine(type = ColType.INT) - @Comment("Application table ID") - private Long milogAppId; - - @Column(value = "app_id") - @ColDefine(type = ColType.INT) - @Comment("App ID") - private Long appId; - - @Column(value = "app_name") - @ColDefine(type = ColType.VARCHAR) - @Comment("App name") - private String appName; - - @Column(value = "app_type") - @ColDefine(type = ColType.VARCHAR, width = 10) - @Comment("0.mione project") - private Integer appType; - - @Column(value = "env_id") - @ColDefine(type = ColType.INT) - @Comment("environment Id") - private Long envId; - - @Column(value = "env_name") - @ColDefine(type = ColType.VARCHAR) - @Comment("environment name") - private String envName; - - @Column(value = "machine_type") - @ColDefine(type = ColType.INT) - @Comment("Machine Type 0. Container 1. Physical machine") - private Integer machineType; - - @Column(value = "ips") - @ColDefine(type = ColType.MYSQL_JSON) - @Comment("tail Collect IP list (appId + envId corresponding to IP list)") - @JsonField - private List ips; - - @Column(value = "motor_rooms") - @ColDefine(type = ColType.MYSQL_JSON) - @Comment("Apply the information of the computer room") - @JSONField - private List motorRooms; - - @Column(value = "parse_type") - @ColDefine(type = ColType.INT) - @Comment("Log parsing type: 1: service application log, 2. delimiter, 3: single line, 4: multiple line, 5: custom") - private Integer parseType; - - @Column(value = "parse_script") - @ColDefine(type = ColType.TEXT) - @Comment("For delimiter, the field specifies the delimiter; For customization, this field specifies the log read script") - private String parseScript; - - @Column(value = "log_path") - @ColDefine(type = ColType.VARCHAR, width = 1024) - @Comment("Comma split, multiple log file paths,e.g.:/home/work/log/xxx/server.log") - private String logPath; - - @Column(value = "log_split_express") - @ColDefine(type = ColType.VARCHAR, width = 1024) - @Comment("Log slicing expression") - private String logSplitExpress; - - @Column(value = "value_list") - @ColDefine(type = ColType.VARCHAR, width = 1024) - @Comment("A list of values, multiple separated by commas") - private String valueList; - - @Column(value = "filter") - @ColDefine(type = ColType.MYSQL_JSON) - @Comment("filterDefine") - @JSONField - private List filter; - - @Column(value = "deploy_way") - @ColDefine(type = ColType.INT) - @Comment("Deployment method：1-mione; 2-miline;") - private Integer deployWay; - - @Column(value = "deploy_space") - @ColDefine(type = ColType.VARCHAR, width = 1024) - @Comment("The deployment space corresponding to the service") - private String deploySpace; - - @Column(value = "first_line_reg") - @ColDefine(type = ColType.VARCHAR, width = 1024) - @Comment("A regular match expression at the beginning of a row") - private String firstLineReg; - - @Column(value = "collection_ready") - @ColDefine(type = ColType.VARCHAR, width = 1024) - @Comment("start to ready coll") - private Boolean collectionReady; - - public MilogLogTailDo() { - } - - public MilogLogTailDo(Long spaceId, Long storeId, Long appId, Integer parseType, String parseScript, String logPath, String valueList) { - this.spaceId = spaceId; - this.storeId = storeId; - this.appId = appId; - this.parseType = parseType; - this.parseScript = parseScript; - this.logPath = logPath; - this.valueList = valueList; - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogTemplateDO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogTemplateDO.java deleted file mode 100644 index 4b582e242..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogTemplateDO.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.io.Serializable; - -/** - *

- * milog Log templates - *

- * - * @author wanghaoyang - * @since 2021-09-09 - */ -@Data -@EqualsAndHashCode(callSuper = false) -@TableName("milog_log_template") -public class MilogLogTemplateDO implements Serializable { - - /** - * Primary key Id - */ - @TableId(value = "id", type = IdType.AUTO) - private Long id; - - /** - * create time - */ - private Long ctime; - - /** - * update time - */ - private Long utime; - - /** - * Log template name - */ - private String templateName; - - /** - * Log template type 0 - Custom log; 1-app; 2-nginx ... - */ - private Integer type; - - private String supportArea; - - /** - * sort - */ - private Integer orderCol; - /** - * Whether self-sustaining consumption, 1. Support - */ - private Integer supportedConsume; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogTemplateDetailDO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogTemplateDetailDO.java deleted file mode 100644 index a83245bc3..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogTemplateDetailDO.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.io.Serializable; - -/** - *

- * milogLog templates are verbose - *

- * - * @author wanghaoyang - * @since 2021-09-09 - */ -@Data -@EqualsAndHashCode(callSuper = false) -@TableName("milog_log_template_detail") -public class MilogLogTemplateDetailDO implements Serializable { - - private static final long serialVersionUID = 1L; - - /** - * Primary key Id - */ - @TableId(value = "id", type = IdType.AUTO) - private Long id; - - /** - * create time - */ - private Long ctime; - - /** - * update time - */ - private Long utime; - - /** - * Log template ID - */ - private String templateId; - - /** - * Log template property name - */ - private String propertiesKey; - - /** - * Log template property type - */ - private String propertiesType; - - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogstailDO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogstailDO.java deleted file mode 100644 index c1aabbb21..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogLogstailDO.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.io.Serializable; - -/** - *

- * - *

- * - * @author wanghaoyang - * @since 2022-01-10 - */ -@Data -@EqualsAndHashCode(callSuper = false) -@TableName("milog_logstail") -public class MilogLogstailDO implements Serializable { - - private static final long serialVersionUID = 1L; - - @TableId(value = "id", type = IdType.AUTO) - private Long id; - - /** - * Creation time - */ - private Long ctime; - - /** - * Update time - */ - private Long utime; - - /** - * creator - */ - private String creator; - - /** - * updater - */ - private String updater; - - /** - * spaceId - */ - private Long spaceId; - - /** - * storeId - */ - private Long storeId; - - /** - * Milog table primary key - */ - private Long milogAppId; - - /** - * App ID - */ - private Long appId; - - /** - * App name - */ - private String appName; - - /** - * 0.mione project - */ - private Integer appType; - - /** - * machine type 0. Container 1. Physical machine - */ - private Integer machineType; - - /** - * environment Id - */ - private Integer envId; - - /** - * environment name - */ - private String envName; - - /** - * Log parsing type: 1: service application log, 2. delimiter, 3: single line, 4: multiple line, 5: custom - */ - private Integer parseType; - - /** - * For delimiter, the field specifies the delimiter; For customization, this field specifies the log read script - */ - private String parseScript; - - /** - * Comma split, multiple log file paths,e.g.:/home/work/log/xxx/server.log - */ - private String logPath; - - /** - * A list of values, multiple separated by commas - */ - private String valueList; - - /** - * ip list - */ - private String ips; - - /** - * Apply the information of the computer room - */ - private String motorRooms; - - /** - * App aliases - */ - private String tail; - - /** - * Filter configuration - */ - private String filter; - - /** - * applies the index configuration - */ - private String enEsIndex; - - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogMiddlewareConfig.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogMiddlewareConfig.java deleted file mode 100644 index 921475826..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogMiddlewareConfig.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import com.xiaomi.mone.log.api.model.bo.MiLogResource; -import com.xiaomi.mone.log.api.model.vo.ResourceInfo; -import com.xiaomi.mone.log.manager.model.BaseCommon; -import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionServiceFactory; -import lombok.Data; -import org.nutz.dao.entity.annotation.*; - -import java.io.Serializable; -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description Middleware configuration - * @date 2021/9/17 15:51 - */ -@Table("milog_middleware_config") -@Comment("Milog configures middleware configuration") -@Data -public class MilogMiddlewareConfig extends BaseCommon implements Serializable { - - @Id - @Comment("Primary key Id") - @ColDefine(customType = "bigint") - private Long id; - - @Column(value = "type") - @ColDefine(customType = "smallint") - @Comment("type MiddlewareEnum.code") - public Integer type; - - @Column(value = "region_en") - @ColDefine(type = ColType.VARCHAR, width = 40) - @Comment("type MachineRegionEnum.en") - private String regionEn; - - @Column(value = "alias") - @ColDefine(type = ColType.VARCHAR, width = 128) - @Comment("alias") - public String alias; - - @Column(value = "name_server") - @ColDefine(type = ColType.VARCHAR, width = 128) - @Comment("nameServer address") - private String nameServer; - - @Column(value = "service_url") - @ColDefine(type = ColType.VARCHAR, width = 128) - @Comment("nameServer address") - private String serviceUrl; - - @Column(value = "ak") - @ColDefine(type = ColType.VARCHAR, width = 200) - @Comment("ak") - private String ak; - - @Column(value = "sk") - @ColDefine(type = ColType.VARCHAR, width = 200) - @Comment("sk") - private String sk; - - @Column(value = "broker_name") - @ColDefine(type = ColType.VARCHAR, width = 200) - @Comment("broker_name") - private String brokerName; - - @Column(value = "authorization") - @ColDefine(type = ColType.VARCHAR, width = 1024) - @Comment("Authorization Information") - private String authorization; - - @Column(value = "org_id") - @ColDefine(type = ColType.INT, width = 20) - @Comment("team Id") - private String orgId; - - @Column(value = "team_id") - @ColDefine(type = ColType.INT, width = 20) - @Comment("user group ID") - private String teamId; - - @Column(value = "is_default") - @ColDefine(type = ColType.INT, width = 20) - @Comment("Whether to default this configuration when mq is not selected (1.Yes 0.No)") - private Integer isDefault; - - @Column(value = "labels") - @ColDefine(type = ColType.MYSQL_JSON) - @Comment("A list of tags") - private List labels; - - @Column(value = "token") - @ColDefine(type = ColType.VARCHAR, width = 255) - @Comment("token") - private String token; - - @Column(value = "dt_catalog") - @ColDefine(type = ColType.VARCHAR, width = 255) - @Comment("catalog") - private String dtCatalog; - - @Column(value = "dt_database") - @ColDefine(type = ColType.VARCHAR, width = 255) - @Comment("database") - private String dtDatabase; - - private String clusterName; - - private String storageType; - - - public ResourceInfo configToResourceVO() { - return ResourceInfo.builder().id(this.id) - .alias(this.alias) - .regionEn(this.regionEn) - .regionCn(CommonExtensionServiceFactory.getCommonExtensionService().getMachineRoomName(this.regionEn)) - .clusterName(this.nameServer) - .serviceUrl(this.serviceUrl) - .brokerName(this.brokerName) - .storageType(this.storageType) - .ak(this.ak) - .sk(this.sk) - .orgId(this.orgId) - .teamId(this.teamId) - .labels(this.labels) - .mqType(this.type) - .ctime(this.getCtime()) - .utime(this.getUtime()) - .creator(this.getCreator()) - .updater(this.getUpdater()).build(); - - } - - public static MilogMiddlewareConfig miLogMqResourceToConfig(MiLogResource resource) { - MilogMiddlewareConfig milogMiddlewareConfig = new MilogMiddlewareConfig(); - milogMiddlewareConfig.setAlias(resource.getAlias()); - milogMiddlewareConfig.setType(null == resource.getMqType() ? resource.getResourceCode() : resource.getMqType()); - milogMiddlewareConfig.setRegionEn(resource.getRegionEn()); - milogMiddlewareConfig.setNameServer(resource.getClusterName()); - milogMiddlewareConfig.setServiceUrl(resource.getServiceUrl()); - milogMiddlewareConfig.setAk(resource.getAk()); - milogMiddlewareConfig.setSk(resource.getSk()); - milogMiddlewareConfig.setOrgId(resource.getOrgId()); - milogMiddlewareConfig.setTeamId(resource.getTeamId()); - milogMiddlewareConfig.setBrokerName(resource.getBrokerName()); - milogMiddlewareConfig.setLabels(resource.getLabels()); - milogMiddlewareConfig.setIsDefault(resource.getIsDefault()); - milogMiddlewareConfig.setStorageType(resource.getStorageType()); - return milogMiddlewareConfig; - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogRegionAvailableZoneDO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogRegionAvailableZoneDO.java deleted file mode 100644 index 2c07b7eef..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogRegionAvailableZoneDO.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import com.xiaomi.mone.log.manager.model.BaseCommon; -import lombok.Data; -import org.nutz.dao.entity.annotation.*; - -/** - * @author zhangping17 - * @date 2021-10-15 Region, AZ entity - */ -@Table("milog_region_zone") -@Comment("milog Regional Availability Zones") -@Data -public class MilogRegionAvailableZoneDO extends BaseCommon { - @Id - @Comment("Primary key Id") - @ColDefine(customType = "bigint") - private Long id; - - @Column(value = "region_name_en") - @ColDefine(type = ColType.VARCHAR) - @Comment("The English name of region") - private String regionNameEN; - - @Column(value = "region_name_cn") - @ColDefine(type = ColType.VARCHAR) - @Comment("Region Chinese name") - private String regionNameCN; - - @Column(value = "zone_name_en") - @ColDefine(type = ColType.VARCHAR) - @Comment("The English name of zone") - private String zoneNameEN; - - @Column(value = "zone_name_cn") - @ColDefine(type = ColType.VARCHAR) - @Comment("Zone Chinese name") - private String zoneNameCN; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogSpaceDO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogSpaceDO.java deleted file mode 100644 index 6cbdd4488..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogSpaceDO.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import com.xiaomi.mone.log.manager.model.BaseCommon; -import lombok.Data; -import org.nutz.dao.entity.annotation.*; - -/** - * @author shanwb - * @date 2021-06-28 - */ - -@Table("milog_space") -@Comment("Milog project space table") -@Data -public class MilogSpaceDO extends BaseCommon { - @Id - @Comment("Primary key Id") - @ColDefine(customType = "bigint") - private Long id; - - - @Column(value = "tenant_id") - @ColDefine(customType = "bigint") - @Comment("Tenant ID") - private Long tenantId; - - - @Column(value = "space_name") - @ColDefine(type = ColType.VARCHAR, width = 256) - @Comment("Project space name") - private String spaceName; - - - @Column(value = "source") - @ColDefine(type = ColType.VARCHAR, width = 128) - @Comment("source") - private String source; - - @Column(value = "perm_dept_id") - @ColDefine(type = ColType.VARCHAR, width = 2000) - @Comment("You can view the third-level department ID of this space") - private String permDeptId; - - - @Column(value = "create_dept_id") - @ColDefine(type = ColType.VARCHAR, width = 50) - @Comment("The created three-level department ID") - private String createDeptId; - - - @Column(value = "description") - @ColDefine(type = ColType.VARCHAR, width = 1024) - @Comment("Remarks description") - private String description; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogStoreSpaceAuth.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogStoreSpaceAuth.java deleted file mode 100644 index de6f32143..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/pojo/MilogStoreSpaceAuth.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.pojo; - -import com.xiaomi.mone.log.manager.model.BaseCommon; -import lombok.Data; -import org.nutz.dao.entity.annotation.*; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/7/14 16:01 - */ -@Table("milog_store_space_auth") -@Comment("milog Store authorization form, store can bind additional space") -@Data -public class MilogStoreSpaceAuth extends BaseCommon { - @Id - @Comment("Primary key Id") - @ColDefine(customType = "bigint") - private Long id; - - @Column(value = "store_id") - @ColDefine(customType = "bigint") - @Comment("Store primary key") - private Long storeId; - - @Column(value = "space_id") - @ColDefine(customType = "bigint") - @Comment("Space primary key") - private Long spaceId; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/AccessMiLogVo.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/AccessMiLogVo.java deleted file mode 100644 index add0e28e9..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/AccessMiLogVo.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Builder; -import lombok.Data; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/4/20 15:06 - */ -@Data -@Builder -public class AccessMiLogVo { - private Long spaceId; - private Long storeId; - private Long tailId; - private String tailName; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/AgentListQuery.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/AgentListQuery.java deleted file mode 100644 index 6310bdabc..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/AgentListQuery.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - -@Data -public class AgentListQuery { - private String ip; - private String isDeployed; - private Integer page; - private Integer pageSize; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/CreateDashboardCmd.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/CreateDashboardCmd.java deleted file mode 100644 index 1a0ce84c8..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/CreateDashboardCmd.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - -import java.io.Serializable; - -@Data -public class CreateDashboardCmd implements Serializable { - - private String name; - - private Long spaceId; - - private Long storeId; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/CreateGraphCmd.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/CreateGraphCmd.java deleted file mode 100644 index 25fb3e8ef..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/CreateGraphCmd.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - -import java.io.Serializable; - -@Data -public class CreateGraphCmd implements Serializable { - - private String name; - - private String fieldName; - - private Long spaceId; - - private Long storeId; - - private Integer graphType; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/CreateIndexTemplatePropertyCommand.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/CreateIndexTemplatePropertyCommand.java deleted file mode 100644 index a9aaecc1a..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/CreateIndexTemplatePropertyCommand.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - - -@Data -public class CreateIndexTemplatePropertyCommand { - private String name; - private String type; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/CreateOrUpdateLogStoreCmd.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/CreateOrUpdateLogStoreCmd.java deleted file mode 100644 index 7dccda0d4..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/CreateOrUpdateLogStoreCmd.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class CreateOrUpdateLogStoreCmd { - private Long id; - private Long spaceId; - private String logstoreName; - private Integer storePeriod; - private Integer shardCnt; - private String keyList; - private String columnTypeList; - /** - * 1. Service application logs - */ - private Integer logType; - private String esIndex; - private String machineRoom; - /** - * mq resource ID - */ - private Long mqResourceId; - /** - * es resource ID - */ - private Long esResourceId; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/DGRefCmd.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/DGRefCmd.java deleted file mode 100644 index 3676c4d60..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/DGRefCmd.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - -import java.io.Serializable; - -@Data -public class DGRefCmd implements Serializable { - private Long dashboardId; - - private Long graphId; - - private String point; - - private String privateName; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/DGRefDelCmd.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/DGRefDelCmd.java deleted file mode 100644 index bd4f01e3a..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/DGRefDelCmd.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - -import java.io.Serializable; - -@Data -public class DGRefDelCmd implements Serializable { - - private Long dashboardId; - - private Long graphId; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/DGRefUpdateCmd.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/DGRefUpdateCmd.java deleted file mode 100644 index 7efee96e5..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/DGRefUpdateCmd.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - -import java.io.Serializable; -import java.util.List; - -@Data -public class DGRefUpdateCmd implements Serializable { - - private Long dashboardId; - - private List graphList; - - @Data - public class DGRefDetailUpdateCmd implements Serializable { - - private Long graphId; - - private String point; - - private String graphPrivateName ; - - } - -} - - diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/GraphQuery.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/GraphQuery.java deleted file mode 100644 index 2532807fb..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/GraphQuery.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - -import java.io.Serializable; -@Data -public class GraphQuery implements Serializable { - - private Long spaceId; - - private Long storeId; - - private String graphName; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/KeywordPageParam.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/KeywordPageParam.java deleted file mode 100644 index a65f0ab28..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/KeywordPageParam.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import com.xiaomi.mone.log.manager.model.PageVo; -import lombok.Data; - -/** - * - * @description - * @version 1.0 - * @author wtt - * @date 2024/6/18 9:49 - * - */ -@Data -public class KeywordPageParam extends PageVo { - private String keyword; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogAgentListBo.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogAgentListBo.java deleted file mode 100644 index 97352b511..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogAgentListBo.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/11/15 17:08 - */ -@Data -public class LogAgentListBo { - - private String podIP; - private String podName; - private String agentIP; - private String agentName; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogAnalyseDataPreQuery.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogAnalyseDataPreQuery.java deleted file mode 100644 index 0d73be7b8..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogAnalyseDataPreQuery.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - -import java.io.Serializable; - -@Data -public class LogAnalyseDataPreQuery implements Serializable { - - private Long storeId; - - private String fieldName; - - private Integer typeCode; - - private String graphParam; - - private Long startTime; - - private Long endTime; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogAnalyseDataQuery.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogAnalyseDataQuery.java deleted file mode 100644 index 02f549bae..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogAnalyseDataQuery.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - -import java.io.Serializable; - -@Data -public class LogAnalyseDataQuery implements Serializable { - - private Long graphId; - - private Long startTime; - - private Long endTime; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogAnalyseQuery.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogAnalyseQuery.java deleted file mode 100644 index 1cfe9134c..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogAnalyseQuery.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - -import java.io.Serializable; - -@Data -public class LogAnalyseQuery implements Serializable { - - private Long spaceId; - - private Long storeId; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogContectQuery.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogContectQuery.java deleted file mode 100644 index a2e1a391e..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogContectQuery.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; -import lombok.Data; - -import java.io.Serializable; - -@Data -public class LogContectQuery implements Serializable { - - private String logstore; - private String tail; - private Integer pageSize; - private Object[] beginSortValue; - private String logip; - private Integer sort; //1 positive order 2 reverse order -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogContextQuery.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogContextQuery.java deleted file mode 100644 index 93b0f927d..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogContextQuery.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - -import java.io.Serializable; -@Data -public class LogContextQuery implements Serializable { - private String logstore; - private String ip; - private String fileName; - private Long lineNumber; - private String timestamp; - private Integer pageSize; - private Integer type; //0-around;1-after;2-before -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogPathTopicVo.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogPathTopicVo.java deleted file mode 100644 index dd9a7ef8a..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogPathTopicVo.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import com.xiaomi.mone.log.api.model.meta.MQConfig; -import lombok.Builder; -import lombok.Data; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/8/17 15:42 - */ -@Data -@Builder -public class LogPathTopicVo { - /** - * Log path - */ - private String logPath; - /** - * tailId - */ - private Long tailId; - - private String source; - /** - * Parse the script - */ - private String parseScript; - /** - * Log format - */ - private String valueList; - /** - * MQ-related configuration - */ - private MQConfig mqConfig; - /** - * Service alias - */ - private String serveAlias; - - private String regionEn; - - private String regionCn; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogQuery.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogQuery.java deleted file mode 100644 index cbdf8dfec..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogQuery.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - -import java.io.Serializable; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; - -@Data -public class LogQuery implements Serializable { - private String logstore; - private Long storeId; - private String tail; - private List tailIds; - private Long startTime; - private Long endTime; - private Integer pageSize; - private Integer page; - private Object[] beginSortValue; - private String fullTextSearch; - private String sortKey = "timestamp"; - private Boolean asc = false; - private Long[] appIds; - - public LogQuery() { - } - - public LogQuery(String logstore, Long storeId, String tail, Long startTime, Long endTime, String fullTextSearch, String sortKey) { - this.logstore = logstore; - this.storeId = storeId; - this.tail = tail; - this.startTime = startTime; - this.endTime = endTime; - this.fullTextSearch = fullTextSearch; - this.sortKey = sortKey; - } - - public LogQuery(String logstore, String tail, Long startTime, Long endTime, String fullTextSearch, String sortKey, Long[] appIds) { - this.logstore = logstore; - this.tail = tail; - this.startTime = startTime; - this.endTime = endTime; - this.fullTextSearch = fullTextSearch; - this.sortKey = sortKey; - this.appIds = appIds; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - LogQuery logQuery = (LogQuery) o; - return Objects.equals(logstore, logQuery.logstore) && Objects.equals(tail, logQuery.tail) && Objects.equals(startTime, logQuery.startTime) && Objects.equals(endTime, logQuery.endTime) && Objects.equals(pageSize, logQuery.pageSize) && Arrays.equals(beginSortValue, logQuery.beginSortValue) && Objects.equals(fullTextSearch, logQuery.fullTextSearch) && Objects.equals(sortKey, logQuery.sortKey) && Objects.equals(asc, logQuery.asc) && Arrays.equals(appIds, logQuery.appIds); - } - - @Override - public int hashCode() { - int result = Objects.hash(logstore, tail, startTime, endTime, pageSize, fullTextSearch, sortKey, asc); - result = 31 * result + Arrays.hashCode(beginSortValue); - result = 31 * result + Arrays.hashCode(appIds); - return result; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogStoreCommand.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogStoreCommand.java deleted file mode 100644 index 73268b0a6..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogStoreCommand.java +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; \ No newline at end of file diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogStoreParam.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogStoreParam.java deleted file mode 100644 index 2f64f0673..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogStoreParam.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * @author milog - */ -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class LogStoreParam { - private Long id; - private Long spaceId; - private String logstoreName; - private Integer storePeriod; - private Integer shardCnt; - private String keyList; - private String columnTypeList; - /** - * 1. Service application logs - */ - private Integer logType; - private Boolean usePlatformResource; - private String esIndex; - private String machineRoom; - /** - * mq resource ID - */ - private Long mqResourceId; - /** - * es resource ID - */ - private Long esResourceId; - - private String storageType; - - private Boolean nameSameStatus = Boolean.FALSE; - - public boolean isPlatformResourceStore() { - return this.usePlatformResource == null ? false : this.usePlatformResource; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogTailSendLokiVo.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogTailSendLokiVo.java deleted file mode 100644 index 8b70289c9..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/LogTailSendLokiVo.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import com.xiaomi.mone.log.model.LogtailConfig; -import lombok.Data; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/1/5 10:47 - */ -@Data -public class LogTailSendLokiVo { - private String spaceName; - private Long spaceId; - private String storeName; - private Long storeId; - private String tailName; - private Long tailId; - private String keyList; - - private LogtailConfig config; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/MachinePartitionParam.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/MachinePartitionParam.java deleted file mode 100644 index 4a0b04156..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/MachinePartitionParam.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import com.xiaomi.mone.log.manager.model.PageVo; -import lombok.Data; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/9/19 15:44 - */ -@Data -public class MachinePartitionParam extends PageVo { - private String machineRoom; - private String spaceName; - private String uniqueKey; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/MilogUserVo.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/MilogUserVo.java deleted file mode 100644 index c9e0dc9ae..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/MilogUserVo.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/9 16:44 - */ -@Data -public class MilogUserVo { - /** - * Username, such as: zhangsan - */ - private String user; - - /** - * Username, such as: Zhang San - */ - private String name; - - /** - * User display name, such as: sa zhang Zhangsan - */ - private String displayName; - - /** - * Department name - */ - private String departmentName; - - /** - * mailbox - */ - private String email; - - /** - * miID - */ - private String miID; - - /** - * miID - */ - private String uID; - - /** - * avatar - */ - private String avatar; - - /** - * - */ - private String zone; - /** - * Last level department Id - */ - private String deptId; - - private String company; - - private boolean isAdmin; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/QuickQueryVO.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/QuickQueryVO.java deleted file mode 100644 index 1090ecb86..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/QuickQueryVO.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.io.Serializable; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/4/28 11:16 - */ -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class QuickQueryVO implements Serializable { - private Long spaceId; - private String spaceName; - private Long storeId; - private String storeName; - private Long tailId; - private String tailName; - private Long envId; - private String envName; - private Integer isFavourite; - private String deploySpace; - - private Boolean collectionReady; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/RegionTraceLogQuery.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/RegionTraceLogQuery.java deleted file mode 100644 index c7608f704..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/RegionTraceLogQuery.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - -@Data -public class RegionTraceLogQuery { - public String traceId; - public String region; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/RocketMQStatisCommand.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/RocketMQStatisCommand.java deleted file mode 100644 index 4fa21f89d..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/RocketMQStatisCommand.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - -@Data -public class RocketMQStatisCommand { - private String topic; - private Long begin; - private Long end; - private String aggreator; - private String metric; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/SearchSaveInsertCmd.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/SearchSaveInsertCmd.java deleted file mode 100644 index 3ca5cfef3..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/SearchSaveInsertCmd.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - -@Data -public class SearchSaveInsertCmd { - private Long spaceId; - private Long storeId; - /** - * Multiple to, split - */ - private Long tailId; - private String name; - private String queryText; - private Long startTime; - private Long endTime; - private Integer isFixTime; - private String common; - private Integer sort; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/SearchSaveUpdateCmd.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/SearchSaveUpdateCmd.java deleted file mode 100644 index b15e57c8d..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/SearchSaveUpdateCmd.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - -@Data -public class SearchSaveUpdateCmd { - private Long id; - private String name; - private String queryText; - private Long startTime; - private Long endTime; - private Integer isFixTime; - private String common; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/SpaceIpParam.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/SpaceIpParam.java deleted file mode 100644 index 8841e2b96..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/SpaceIpParam.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import com.xiaomi.mone.log.manager.model.PageVo; -import lombok.Data; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/10/16 14:14 - */ -@Data -public class SpaceIpParam extends PageVo { - private String machineRoom; - private String uniqueKey; - private Long spaceId; - private String spaceName; - - private List spaceIds; - private List uniqueKeys; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/StreamListQuery.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/StreamListQuery.java deleted file mode 100644 index ab0550481..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/StreamListQuery.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - -@Data -public class StreamListQuery { - private Integer page; - private Integer pageSize; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/TraceAppLogUrlQuery.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/TraceAppLogUrlQuery.java deleted file mode 100644 index 82866e239..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/TraceAppLogUrlQuery.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - -@Data -public class TraceAppLogUrlQuery { - private String traceId; - private Long appId; - private Long timestamp; - private Long envId; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/UpdateGraphCmd.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/UpdateGraphCmd.java deleted file mode 100644 index 16f0213c2..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/UpdateGraphCmd.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - -import java.io.Serializable; -@Data -public class UpdateGraphCmd implements Serializable { - - private Long id; - - private String name; - - private String fieldName; - - private Integer graphType; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/UpdateIndexTemplateCommand.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/UpdateIndexTemplateCommand.java deleted file mode 100644 index 800b80931..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/model/vo/UpdateIndexTemplateCommand.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.model.vo; - -import lombok.Data; - -import java.util.List; -@Data -public class UpdateIndexTemplateCommand { - private String indexTemplateName; - private Integer indexShards; - private Integer indexReplicas; - private String lifecycle; - - private List propertyList; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/BaseService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/BaseService.java deleted file mode 100644 index 90f1bd558..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/BaseService.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import com.xiaomi.mone.log.api.enums.OperateEnum; -import com.xiaomi.mone.log.manager.common.context.MoneUserContext; -import com.xiaomi.mone.log.manager.model.BaseCommon; -import com.xiaomi.mone.log.manager.model.MilogSpaceParam; -import com.xiaomi.mone.log.manager.model.pojo.MilogSpaceDO; - -import static com.xiaomi.mone.log.common.Constant.DEFAULT_OPERATOR; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/12/20 15:56 - */ -public class BaseService { - - public void wrapMilogSpace(MilogSpaceDO ms, MilogSpaceParam param) { - ms.setSpaceName(param.getSpaceName()); - ms.setTenantId(param.getTenantId()); - ms.setSource(MoneUserContext.getCurrentUser().getZone()); - ms.setDescription(param.getDescription()); - } - - public void wrapMilogSpace(MilogSpaceDO ms, MilogSpaceParam param, String source) { - ms.setSpaceName(param.getSpaceName()); - ms.setTenantId(param.getTenantId()); - ms.setSource(source); - ms.setDescription(param.getDescription()); - } - - public void wrapBaseCommon(BaseCommon common, OperateEnum operateEnum) { - String operator = MoneUserContext.getCurrentUser() != null ? MoneUserContext.getCurrentUser().getUser() : DEFAULT_OPERATOR; - if (operateEnum == OperateEnum.ADD_OPERATE) { - common.setCtime(System.currentTimeMillis()); - common.setCreator(operator); - } - common.setUtime(System.currentTimeMillis()); - common.setUpdater(operator); - } - - public void wrapBaseCommon(BaseCommon common, OperateEnum operateEnum, String appCreator) { - if (operateEnum == OperateEnum.ADD_OPERATE) { - common.setCtime(System.currentTimeMillis()); - common.setCreator(appCreator); - } - common.setUtime(System.currentTimeMillis()); - common.setUpdater(appCreator); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/CommonRocketMqService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/CommonRocketMqService.java deleted file mode 100644 index 6abcb8a61..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/CommonRocketMqService.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.gson.Gson; -import com.xiaomi.mone.log.common.Config; -import com.xiaomi.mone.log.manager.model.bo.RocketMqStatisticParam; -import org.apache.commons.lang3.StringUtils; - -import java.util.*; -import java.util.stream.Collectors; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/6/30 11:09 - */ -public interface CommonRocketMqService { - - String authorization = Config.ins().get("rocketmq_authorization", ""); - - String clientVerify = Config.ins().get("rocketmq_x-ssl-client-verify", ""); - - String clientDn = Config.ins().get("rocketmq_x-ssl-client-dn", ""); - - String rocketmqAddress = Config.ins().get("rocketmq_address", ""); - - String rocketmqAk = Config.ins().get("rocketmq_ak", ""); - - String rocketmqSk = Config.ins().get("rocketmq_sk", ""); - - String rocketmqNamesrvAddr = Config.ins().get("rocketmq_namesrv_addr", ""); - - String rocketmqOrgId = Config.ins().get("rocketmq_org_id", ""); - - String userGroupIds = Config.ins().get("rocketmq_user_group_ids", "CI102316,CI102322"); - - List brokers = Arrays.stream(Config.ins().get("rocketmq_brokers", "").split(",")).collect(Collectors.toList()); - Integer queueEachBroker = 1; - Integer queueTotalCount = 1; - - String METHOD_DELETE = "DELETE"; - String MQ_AUTHORIZATION = "authorization"; - String MQ_X_SSL_CLIENT_VERIFY = "x-ssl-client-verify"; - String MQ_X_SSL_CLIENT_DN = "x-ssl-client-dn"; - String MQ_ORG_ID = "orgId"; - String MQ_NAME = "name"; - String MQ_BROKERS = "brokers"; - String MQ_QUEUEEACH_BROKER = "queueEachBroker"; - String MQ_QUEUE_TOTAL_COUNT = "queueTotalCount"; - String MQ_CONTENT_TYPE = "Content-Type"; - - /** - * Encapsulate mq header Chen map - * - * @return - */ - default Map getSendMqHeader(String userAuthorization) { - Map headMap = Maps.newHashMap(); - if (StringUtils.isEmpty(userAuthorization)) { - headMap.put(MQ_AUTHORIZATION, authorization); - } else { - headMap.put(MQ_AUTHORIZATION, userAuthorization); - } - headMap.put(MQ_X_SSL_CLIENT_VERIFY, clientVerify); - headMap.put(MQ_X_SSL_CLIENT_DN, clientDn); - headMap.put(MQ_CONTENT_TYPE, "application/json"); - return headMap; - } - - /** - * Encapsulate the header of mq into a list - * - * @return - */ - default List getSendMqHeader2List(String authorization) { - List lists = Lists.newArrayList(); - getSendMqHeader(authorization).entrySet().stream().forEach(map -> { - lists.add(map.getKey()); - lists.add(map.getValue()); - }); - return lists; - } - - /** - * Create some public parameters of topic in mq - * - * @param topicName - * @return - */ - default String createTopicBodyParams(String topicName) { - Map paramsMap = Maps.newHashMap(); - paramsMap.put(MQ_ORG_ID, rocketmqOrgId); - paramsMap.put(MQ_NAME, topicName); - paramsMap.put(MQ_BROKERS, brokers); - paramsMap.put(MQ_QUEUEEACH_BROKER, queueEachBroker); - paramsMap.put(MQ_QUEUE_TOTAL_COUNT, queueTotalCount); - return new Gson().toJson(paramsMap); - } - - /** - * Parameters for creating consumer Group - * - * @param name - * @return - */ - default String createConsumerGroupParams(String name, String orgId) { - Map paramsMap = Maps.newHashMap(); - if (StringUtils.isNotEmpty(orgId)) { - paramsMap.put(MQ_ORG_ID, orgId); - } else { - paramsMap.put(MQ_ORG_ID, rocketmqOrgId); - } - paramsMap.put(MQ_NAME, name); - return new Gson().toJson(paramsMap); - } - - default String updateTopicAuthParams(String topicName) { - Map paramsMap = Maps.newHashMap(); - paramsMap.put(MQ_NAME, topicName); - List> mapList = Lists.newArrayList(); - Arrays.stream(userGroupIds.split(",")).forEach(groupId -> { - Map orgMap = new TreeMap<>(); - orgMap.put("teamId", groupId); - orgMap.put("value", "PUB|SUB"); - mapList.add(orgMap); - }); - paramsMap.put("teamPerms", mapList); - System.out.println(new Gson().toJson(paramsMap)); - return new Gson().toJson(paramsMap); - } - - default String createTopicStatisticBodyParams(String topic, String group, String broker, String aggregator, Long begin, Long end) { - RocketMqStatisticParam rmqsc = new RocketMqStatisticParam(); - rmqsc.setTopicList(new ArrayList() {{ - add(topic); - }}); - rmqsc.setBroker("*"); - rmqsc.setAggregator(aggregator); - rmqsc.setBegin(begin); - rmqsc.setEnd(end); - return new Gson().toJson(rmqsc); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/EsDataBaseService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/EsDataBaseService.java deleted file mode 100644 index c54658be2..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/EsDataBaseService.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import org.elasticsearch.common.text.Text; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.fetch.subphase.highlight.HighlightField; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public interface EsDataBaseService { - - default Map> getHightlinghtMap(SearchHit hit) { - Map> highlinghtMap = new HashMap<>(); - Map highlightFields = hit.getHighlightFields(); - if (highlightFields != null && !highlightFields.isEmpty()) { - for (Map.Entry highlightFieldEntry : highlightFields.entrySet()) { - List highlightMessage = Arrays.stream(highlightFieldEntry.getValue().getFragments()) - .map(Text::toString) - .collect(Collectors.toList()); - highlinghtMap.put(highlightFieldEntry.getKey(), highlightMessage); - } - } - return highlinghtMap; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/EsDataService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/EsDataService.java deleted file mode 100644 index 7065a8827..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/EsDataService.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import com.xiaomi.mone.log.api.model.dto.TraceLogDTO; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.dto.EsStatisticResult; -import com.xiaomi.mone.log.manager.model.dto.LogDTO; -import com.xiaomi.mone.log.manager.model.vo.LogQuery; -import com.xiaomi.mone.log.manager.model.vo.RegionTraceLogQuery; - -import java.io.IOException; -import java.util.Map; - -public interface EsDataService { - - /** - * Read the data in the ES index - * - * @param logQuery - * @return - */ - Result logQuery(LogQuery logQuery); - - /** - * Insert data - * - * @param indexName - * @param data - * @return - */ - void insertDoc(String indexName, Map data) throws IOException; - - Result EsStatistic(LogQuery param) throws Exception; - - /** - * Obtain trace logs in the data center - * - * @param regionTraceLogQuery - * @return - */ - Result queryRegionTraceLog(RegionTraceLogQuery regionTraceLogQuery) throws IOException; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/EsIndexOperateService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/EsIndexOperateService.java deleted file mode 100644 index cef0c06d0..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/EsIndexOperateService.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2024/1/19 17:10 - */ -public interface EsIndexOperateService { - void createIndexPre(); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/EsIndexTemplateService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/EsIndexTemplateService.java deleted file mode 100644 index 36dd34356..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/EsIndexTemplateService.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import com.xiaomi.mone.log.manager.model.vo.UpdateIndexTemplateCommand; - -import java.io.IOException; - -public interface EsIndexTemplateService { - - /** - * Update index template - * - * @param updateIndexTemplateCommand - * @return - */ - public boolean updateIndexTemplate(UpdateIndexTemplateCommand updateIndexTemplateCommand) throws IOException; - - /** - * Create index - * - * @param indexName - * @return - * @throws IOException - */ - boolean createIndex(String indexName) throws IOException; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/HeraAppEnvService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/HeraAppEnvService.java deleted file mode 100644 index ca6c4292c..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/HeraAppEnvService.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import org.apache.ozhera.app.api.model.HeraAppEnvData; -import org.apache.ozhera.app.api.model.HeraSimpleEnv; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/11/12 11:47 - */ -public interface HeraAppEnvService { - - List querySimpleEnvAppBaseInfoId(Integer appBaseId); - - List queryEnvById(Long id, Long heraAppId, Long envId); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/HeraAppService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/HeraAppService.java deleted file mode 100644 index 860461a17..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/HeraAppService.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import org.apache.ozhera.app.api.model.HeraAppBaseInfoModel; -import org.apache.ozhera.app.api.model.HeraAppBaseInfoParticipant; -import org.apache.ozhera.app.api.model.HeraAppBaseQuery; -import org.apache.ozhera.app.api.response.AppBaseInfo; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/11/11 17:00 - */ -public interface HeraAppService { - - List queryAppInfoWithLog(String appName, Integer type); - - List queryAllExistsApp(); - - AppBaseInfo queryById(Long id); - - List queryByIds(List ids); - - AppBaseInfo queryByAppId(Long appId, Integer type); - - AppBaseInfo queryByAppIdPlatFormType(String bindId, Integer platformTypeCode); - - AppBaseInfo queryByIamTreeId(Long iamTreeId, String bingId, Integer platformType); - - Long countByParticipant(HeraAppBaseQuery query); - - List queryByParticipant(HeraAppBaseQuery query); - - Long count(HeraAppBaseInfoModel baseInfo); - - List query(HeraAppBaseInfoModel baseInfo, Integer pageCount, Integer pageNum); - - HeraAppBaseInfoModel getById(Integer id); - - int delById(Integer id); - - Long getAppCount(); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/HeralogHomePageService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/HeralogHomePageService.java deleted file mode 100644 index 5fd344edf..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/HeralogHomePageService.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.xiaomi.mone.log.manager.service; - -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.dto.MilogSpaceTreeDTO; -import com.xiaomi.mone.log.manager.model.dto.UnAccessAppDTO; -import com.xiaomi.mone.log.manager.model.dto.ValueDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; - -import java.util.List; -import java.util.Map; - - -public interface HeralogHomePageService { - - /** - * Get all apps info - * - * @return - */ - Result> milogAccess(); - - /** - * Package and return all application details. - * - * @return - */ - Result> unAccessAppList(); - - /** - * Get all logStoreInfo by spaceId and return in a tree structure. - * - * @param spaceId - * @return - */ - Result> getMilogSpaceTree(Long spaceId); - - /** - * Query the store that originally belonged to the space, and query the authorized store - * - * @param spaceId - * @return - */ - List getMilogLogStoreDOS(Long spaceId); - - /** - * Get log pattern from properties - * - * @return - */ - Result>> getMiloglogAccessPattern(); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/IMilogEsClusterService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/IMilogEsClusterService.java deleted file mode 100644 index 31523b8c5..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/IMilogEsClusterService.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import com.baomidou.mybatisplus.extension.service.IService; -import com.xiaomi.mone.log.manager.model.pojo.MilogEsClusterDO; - -/** - * - * @author wanghaoyang - * @since 2021-09-18 - */ -public interface IMilogEsClusterService extends IService { - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/IMilogEsIndexService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/IMilogEsIndexService.java deleted file mode 100644 index 0e3375ff5..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/IMilogEsIndexService.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import com.baomidou.mybatisplus.extension.service.IService; -import com.xiaomi.mone.log.manager.model.pojo.MilogEsIndexDO; - -/** - * - * @author wanghaoyang - * @since 2021-09-18 - */ -public interface IMilogEsIndexService extends IService { - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/IMilogLogSearchSaveService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/IMilogLogSearchSaveService.java deleted file mode 100644 index 0b93c4e2c..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/IMilogLogSearchSaveService.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -/** - * - * @author wanghaoyang - * @since 2022-03-29 - */ -public interface IMilogLogSearchSaveService { - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogAnalyseService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogAnalyseService.java deleted file mode 100644 index ba572c109..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogAnalyseService.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.dto.DashboardDTO; -import com.xiaomi.mone.log.manager.model.dto.GraphDTO; -import com.xiaomi.mone.log.manager.model.dto.GraphTypeDTO; -import com.xiaomi.mone.log.manager.model.dto.LogAnalyseDataDTO; -import com.xiaomi.mone.log.manager.model.vo.*; - -import java.io.IOException; -import java.util.List; - - public interface LogAnalyseService { - - Result getDashboardGraph(LogAnalyseQuery logAnalyseQuery); - - Result createGraph(CreateGraphCmd cmd); - - Result updateGraph(UpdateGraphCmd cmd); - - Result deleteGraph(Long graphId); - - Result> searchGraph(GraphQuery query); - - Result ref(DGRefCmd cmd); - - Result delRef(DGRefDelCmd cmd); - - Result updateRef(DGRefUpdateCmd cmd); - - Result data(LogAnalyseDataQuery query) throws IOException; - - Result dataPre(LogAnalyseDataPreQuery query) throws IOException; - - Result createDashboard(CreateDashboardCmd cmd); - - Result> type(); - - Result> supportKye(Long storeId); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogCountService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogCountService.java deleted file mode 100644 index 1d27e31fd..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogCountService.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.dto.LogtailCollectTopDTO; -import com.xiaomi.mone.log.manager.model.dto.LogtailCollectTrendDTO; -import com.xiaomi.mone.log.manager.model.dto.SpaceCollectTopDTO; -import com.xiaomi.mone.log.manager.model.dto.SpaceCollectTrendDTO; - -import java.io.IOException; -import java.util.List; - -public interface LogCountService { - - Result> collectTop(); - - Result> collectTrend(Long tailId); - - void collectTopCount(); - - void collectTrendCount(Long tailId); - - void collectLogCount(String thisDay) throws IOException; - - boolean isLogtailCountDone(String day); - - void deleteHistoryLogCount(); - - void collectLogDelete(String day); - - void collectTrendRefresh(); - - void showLogCountCache(); - - Result> collectSpaceTop(); - - void collectSpaceTopCount(); - - void collectSpaceTrend(); - - Result> spaceCollectTrend(Long spaceId); - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogQueryService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogQueryService.java deleted file mode 100644 index e07724b04..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogQueryService.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import com.xiaomi.mone.log.api.model.dto.TraceLogDTO; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.dto.EsStatisticResult; -import com.xiaomi.mone.log.manager.model.dto.LogDTO; -import com.xiaomi.mone.log.manager.model.vo.LogContextQuery; -import com.xiaomi.mone.log.manager.model.vo.LogQuery; -import com.xiaomi.mone.log.manager.model.vo.RegionTraceLogQuery; - -import java.io.IOException; - -public interface LogQueryService { - - /** - * Read the data in the ES index - * - * @param logQuery - * @return - */ - Result logQuery(LogQuery logQuery) throws Exception; - - Result EsStatistic(LogQuery param) throws Exception; - - /** - * Obtain trace logs in the data center - * - * @param regionTraceLogQuery - * @return - */ - Result queryRegionTraceLog(RegionTraceLogQuery regionTraceLogQuery) throws IOException; - - /** - * Obtain trace logs in the data center... - * - * @param logContextQuery - * @return - */ - Result getDocContext(LogContextQuery logContextQuery); - - /** - * Log export - * - * @param logQuery - * @throws Exception - */ - void logExport(LogQuery logQuery) throws Exception; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogSearchSaveService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogSearchSaveService.java deleted file mode 100644 index aa06f934b..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogSearchSaveService.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -/** - * - * @author wanghaoyang - * @since 2022-03-29 - */ -public interface LogSearchSaveService { - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogSpaceService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogSpaceService.java deleted file mode 100644 index 0f609da4c..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogSpaceService.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.MilogSpaceParam; -import com.xiaomi.mone.log.manager.model.dto.MapDTO; -import com.xiaomi.mone.log.manager.model.dto.MilogSpaceDTO; -import com.xiaomi.mone.log.manager.model.page.PageInfo; -import com.xiaomi.mone.log.manager.model.pojo.MilogSpaceDO; - -import java.util.List; - -public interface LogSpaceService { - - /** - * new - * - * @param cmd - * @return - */ - Result newMilogSpace(MilogSpaceParam cmd); - - /** - * getById - * - * @param id - * @return - */ - Result getMilogSpaceById(Long id); - - /** - * Paging query - * - * @param spaceName - * @param page - * @param pagesize - * @return - */ - Result> getMilogSpaceByPage(String spaceName, Long tenantId, Integer page, Integer pagesize); - - - Result>> getMilogSpaces(Long tenantId); - - /** - * update - * - * @param cmd - * @return - */ - Result updateMilogSpace(MilogSpaceParam cmd); - - Result deleteMilogSpace(Long id); - - Result setSpacePermission(Long spaceId, String permDeptIds); - - MilogSpaceDO buildMiLogSpace(MilogSpaceParam cmd, String appCreator); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogStoreService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogStoreService.java deleted file mode 100644 index cbf57db96..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogStoreService.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.dto.LogStoreDTO; -import com.xiaomi.mone.log.manager.model.dto.MapDTO; -import com.xiaomi.mone.log.manager.model.dto.MenuDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.vo.LogStoreParam; - -import java.util.List; -import java.util.Map; - -public interface LogStoreService { - - Result newLogStore(LogStoreParam cmd); - - MilogLogStoreDO buildLogStoreEsInfo(LogStoreParam cmd, String creator); - - Result getLogStoreById(Long id); - - Result>> getLogStoreBySpaceId(Long spaceId); - - Result> getLogStoreByPage(String logstoreName, Long spaceId, int page, int pagesize); - - Result> getAllLogStore(); - - Result> getLogStoreByIds(List ids); - - Result updateLogStore(LogStoreParam cmd); - - Result deleteLogStore(Long id); - - Result>> getStoreIps(Long storeId); - - Result>> queryDeptExIndexList(String regionCode, Integer logTypeCode); - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogTailService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogTailService.java deleted file mode 100644 index 146be468a..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogTailService.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import org.apache.ozhera.app.api.response.AppBaseInfo; -import org.apache.ozhera.app.model.vo.HeraEnvIpVo; -import com.xiaomi.mone.log.api.enums.LogStructureEnum; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.bo.LogTailParam; -import com.xiaomi.mone.log.manager.model.bo.MlogParseParam; -import com.xiaomi.mone.log.manager.model.dto.*; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.mone.log.manager.model.vo.QuickQueryVO; - -import java.util.List; -import java.util.Map; - -public interface LogTailService { - - Result newMilogLogTail(LogTailParam param); - - void sengMessageNewTail(LogTailParam param, MilogLogTailDo milogLogtailDo, MilogLogStoreDO milogLogStore); - - MilogLogTailDo buildLogTailDo(LogTailParam param, MilogLogStoreDO milogLogStore, AppBaseInfo appBaseInfo, String creator); - - void sengMessageToAgent(Long milogAppId, MilogLogTailDo logtailDo); - - void sengMessageToStream(MilogLogTailDo mt, Integer type); - - void handleNaocsConfigByMotorRoom(MilogLogTailDo mt, String motorRoomEn, Integer type, Integer projectType); - - /** - * Delete a part of a configuration in the configuration center - * - * @param spaceId - * @param id - * @param motorRoomEn - * @param logStructureEnum - * @return - */ - boolean deleteConfigRemote(Long spaceId, Long id, String motorRoomEn, LogStructureEnum logStructureEnum); - - Result getMilogLogtailById(Long id); - - Result> getMilogLogBypage(Long storeId, int page, int pagesize); - - Result> getLogTailCountByStoreId(Long storeId); - - Result> getMilogLogtailByIds(List ids); - - Result updateMilogLogTail(LogTailParam param); - - Result deleteLogTail(Long id); - - void sendMessageOnDelete(MilogLogTailDo mt, MilogLogStoreDO logStoreDO); - - Result> getAppInfoByName(String appName, Integer type); - - /** - * If the application is milog-agent, get a list of all machines through an additional interface - * - * @param milogAppId - * @param deployWay - * @return - */ - Result> getEnInfosByAppId(Long milogAppId, Integer deployWay, String machineRoom); - - Result> getTailNamesBystoreId(String tail, Integer appType, Long id); - - Result>> tailRatelimit(); - - /*** - * miline Dynamic expansion and contraction - * @param projectInfo - */ - void dockerScaleDynamic(DockerScaleBo projectInfo); - - LogTailDTO milogLogtailDO2DTO(MilogLogTailDo milogLogtailDo); - - Result> queryAppByStoreId(Long storeId); - - Result> queryAppTailByStoreId(Long storeId); - - Result> queryLogStoreByRegionEn(String nameEn); - - Result> getTailByStoreId(Long storeId); - - Result parseScriptTest(MlogParseParam mlogParseParam); - - Result> quickQueryByApp(Long milogAppId); - - void machineIpChange(HeraEnvIpVo heraEnvIpVo); - - Result queryAppStore(Long appId, Integer platFormCode); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogTemplateService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogTemplateService.java deleted file mode 100644 index 7459a554c..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/LogTemplateService.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import com.baomidou.mybatisplus.extension.service.IService; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.dto.LogTemplateDTO; -import com.xiaomi.mone.log.manager.model.dto.LogTemplateDetailDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTemplateDO; - -import java.io.IOException; -import java.util.List; - -public interface LogTemplateService extends IService { - /** - * Log template list - * - * @return - */ - Result> getLogTemplateList(String area); - - /** - * Get log template - * - * @param logTemplateId - * @return - * @throws IOException - */ - Result getLogTemplateById(long logTemplateId); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MiLogMetaManageService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MiLogMetaManageService.java deleted file mode 100644 index c6f701ec9..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MiLogMetaManageService.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import com.xiaomi.mone.log.api.model.meta.LogCollectMeta; - -/** - * @author shanwb - * @date 2021-07-08 - */ -public interface MiLogMetaManageService { - - /** - * Get the log collection metadata information of the specified agent - * - * @param agentId - * @param agentIp - * @return - */ - LogCollectMeta queryLogCollectMeta(String agentId, String agentIp); - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MilogAppMiddlewareRelService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MilogAppMiddlewareRelService.java deleted file mode 100644 index 0f63a3cd9..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MilogAppMiddlewareRelService.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/23 11:14 - */ -public interface MilogAppMiddlewareRelService { - void bindingTailConfigRel(Long tailId, Long milogAppId, Long configId, String topicName); - - void defaultBindingAppTailConfigRel(Long id, Long milogAppId, Long middleWareId, String topicName, Integer batchSendSize); - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MilogConfigNacosService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MilogConfigNacosService.java deleted file mode 100644 index 9b1a0e1c6..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MilogConfigNacosService.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -/** - * @author wtt - * @version 1.0 - * @description Configure the intermediate configuration of synchronized nacos - * @date 2021/7/19 16:10 - */ -public interface MilogConfigNacosService { - - String STREAM_PREFIX = "stream"; - - String SPACE_PREFIX = "space"; - - void publishStreamConfig(Long spaceId, Integer type, Integer projectType, String motorRoomEn); - - void publishNameSpaceConfig(String motorRoomEn, Long spaceId, Long storeId, Long tailId, Integer type, String changeType); - - void removeStreamConfig(String motorRoomEn, Long id); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MilogDictionaryService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MilogDictionaryService.java deleted file mode 100644 index c2c1d0f96..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MilogDictionaryService.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.bo.MilogDictionaryParam; -import com.xiaomi.mone.log.manager.model.dto.DictionaryDTO; - -import java.util.List; -import java.util.Map; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/26 15:34 - */ -public interface MilogDictionaryService { - - Result>>> queryDictionaryList(MilogDictionaryParam codes); - - Result downLoadFile(); - - Result fixLogTailMilogAppId(String appName); - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MilogMachineService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MilogMachineService.java deleted file mode 100644 index 7bfa87f44..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MilogMachineService.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.bo.MachineParamParam; -import com.xiaomi.mone.log.manager.model.bo.MachineQueryParam; - -import java.util.Map; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/16 11:32 - */ -public interface MilogMachineService { - - Result addMachineInfo(MachineParamParam param); - - Result deleteMachineInfo(Long id); - - Result> queryMachineByPage(MachineQueryParam param); - - Result queryMachineInfoById(Long id); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MilogMiddlewareConfigService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MilogMiddlewareConfigService.java deleted file mode 100644 index f9cc3fa61..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MilogMiddlewareConfigService.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import com.xiaomi.mone.log.api.model.bo.MiLogResource; -import com.xiaomi.mone.log.api.model.bo.ResourcePage; -import com.xiaomi.mone.log.api.model.vo.ResourceInfo; -import com.xiaomi.mone.log.api.model.vo.ResourceUserSimple; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.bo.MiddlewareAddParam; -import com.xiaomi.mone.log.manager.model.bo.MiddlewareQueryParam; -import com.xiaomi.mone.log.manager.model.bo.MiddlewareUpdateParam; -import com.xiaomi.mone.log.manager.model.page.PageInfo; -import com.xiaomi.mone.log.manager.model.pojo.MilogMiddlewareConfig; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/22 11:41 - */ -public interface MilogMiddlewareConfigService { - PageInfo queryMiddlewareConfigPage(MiddlewareQueryParam middlewareQueryParam); - - Result addMiddlewareConfig(MiddlewareAddParam middlewareQueryParam); - - Result updateMiddlewareConfig(MiddlewareUpdateParam middlewareAddParam); - - Result deleteMiddlewareConfig(Long id); - - List queryMiddlewareConfigList(); - - Result queryMiddlewareConfigById(Long id); - - PageInfo queryResourceWithTab(ResourcePage resourcePage); - - Result resourceOperate(MiLogResource miLogResource); - - String synchronousResourceLabel(Long id); - - ResourceUserSimple userResourceList(String regionCode, Integer logTypeCode); - - ResourceInfo resourceDetail(Integer resourceCode, Long id); - - MilogMiddlewareConfig queryMiddlewareConfigDefault(String regionCode); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MilogStoreSpaceAuthService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MilogStoreSpaceAuthService.java deleted file mode 100644 index a4eb1b707..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MilogStoreSpaceAuthService.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import com.xiaomi.mone.log.manager.model.bo.StoreSpaceAuth; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/7/14 16:12 - */ -public interface MilogStoreSpaceAuthService { - String storeSpaceAuth(StoreSpaceAuth storeSpaceAuth); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MilogStreamService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MilogStreamService.java deleted file mode 100644 index ec2f628d1..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MilogStreamService.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import com.xiaomi.mone.log.common.Result; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/27 17:25 - */ -public interface MilogStreamService { - Result configIssueStream(String ip); - - void executeSql(String sql); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MqConfigService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MqConfigService.java deleted file mode 100644 index 2dd6db3f1..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/MqConfigService.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import cn.hutool.core.util.ReUtil; -import cn.hutool.core.util.StrUtil; -import com.xiaomi.mone.log.manager.model.dto.DictionaryDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogAppMiddlewareRel; -import com.xiaomi.mone.log.utils.PinYin4jUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import static com.xiaomi.mone.log.common.Constant.COMMON_MQ_PREFIX; -import static com.xiaomi.mone.log.common.Constant.COMMON_MQ_SUFFIX; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/23 11:31 - */ -public interface MqConfigService { - - default List generateCommonTagTopicName(String orgId) { - return IntStream.range(0, COMMON_MQ_SUFFIX.size()).mapToObj(value -> { - String suffix = COMMON_MQ_SUFFIX.get(value); - if (StringUtils.isNotBlank(orgId)) { - return String.format("%s_%s_%s", COMMON_MQ_PREFIX, orgId, suffix); - } - return String.format("%s_%s", COMMON_MQ_PREFIX, suffix); - }).collect(Collectors.toList()); - } - - default String generateSimpleTopicName(Long appId, String appName, String source, Long tailId) { - if (StringUtils.isNotEmpty(appName)) { - // Chinese characters are converted to pinyin - appName = PinYin4jUtils.getAllPinyin(appName); - } - // Handle special characters - List collect = ReUtil.RE_KEYS.stream() - .map(character -> character.toString()).collect(Collectors.toList()); - String topicName = String.format("%s_%s_%s_%s", appId, appName, tailId, source); - topicName = StrUtil.removeAny(topicName, collect.toArray(new String[0])); - return topicName; - } - - default String generateSimpleTopicName(Long id, String name) { - if (StringUtils.isNotEmpty(name)) { - // Chinese characters are converted to pinyin - name = PinYin4jUtils.getAllPinyin(name); - } - // Handle special characters - List collect = ReUtil.RE_KEYS.stream().map(Object::toString).toList(); - String topicName = String.format("%s_%s", id, name); - topicName = StrUtil.removeAny(topicName, collect.toArray(new String[0])); - return topicName; - } - - MilogAppMiddlewareRel.Config generateConfig(String ak, String sk, String nameServer, String serviceUrl, - String authorization, String orgId, String teamId, Long exceedId, - String name, String source, Long id); - - List queryExistsTopic(String ak, String sk, String nameServer, String serviceUrl, - String authorization, String orgId, String teamId); - - /** - * Create several public topics and enable tag filtering - * - * @return - */ - List createCommonTagTopic(String ak, String sk, String nameServer, String serviceUrl, - String authorization, String orgId, String teamId); - - - boolean CreateGroup(String ak, String sk, String nameServer); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/SpaceAuthService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/SpaceAuthService.java deleted file mode 100644 index 778546cff..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/SpaceAuthService.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import com.xiaomi.mone.log.manager.model.MilogSpaceParam; -import com.xiaomi.mone.log.manager.model.pojo.MilogSpaceDO; -import com.xiaomi.mone.tpc.common.vo.NodeVo; -import com.xiaomi.mone.tpc.common.vo.PageDataVo; -import com.xiaomi.youpin.infra.rpc.Result; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/4/10 15:00 - */ -public interface SpaceAuthService { - - /** - * save permission - * - * @param spaceDO - * @param account - * @return - */ - Result saveSpacePerm(MilogSpaceDO spaceDO, String account); - - /** - * get current user have auth space - * - * @param spaceName - * @param page - * @param pageSize - * @return - */ - Result> getUserPermSpace(String spaceName, Integer page, Integer pageSize); - - /** - * delete space - * - * @param spaceId - * @param account - * @param userType - * @return - */ - Result deleteSpaceTpc(Long spaceId, String account, Integer userType); - - /** - * update space - * - * @param param - * @param account - * @return - */ - Result updateSpaceTpc(MilogSpaceParam param, String account); - - /** - * add member - * - * @param spaceId - * @param userAccount - * @param userType - * @param memberCode - */ - void addSpaceMember(Long spaceId, String userAccount, Integer userType, Integer memberCode); - - /** - * query access to user node list - * @param spaceName - * @param userName - * @return - */ - Result> queryUserListNode(String spaceName, String userName); - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/StatisticsService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/StatisticsService.java deleted file mode 100644 index e9f867469..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/StatisticsService.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.xiaomi.mone.log.manager.service; - -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.StatisticsQuery; -import com.xiaomi.mone.log.manager.model.dto.EsStatisticsKeyWord; -import com.xiaomi.mone.log.manager.model.vo.LogQuery; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - - -public interface StatisticsService { - - - /** - * Hourly data volume statistics for a single tail - * - * @param statisticsQuery - * @return - * @throws IOException - */ - Result> queryTailStatisticsByHour(StatisticsQuery statisticsQuery) throws IOException; - - - /** - * Top 5 in the data volume of all tails in a single store on the day - * - * @param statisticsQuery - * @return - * @throws IOException - */ - Result> queryStoreTopTailStatisticsByDay(StatisticsQuery statisticsQuery) throws IOException; - - - /** - * Top 5 in the volume of data of all stores in a single space - * - * @param statisticsQuery - * @return - * @throws IOException - */ - Result> querySpaceTopStoreByDay(StatisticsQuery statisticsQuery) throws IOException; - - /** - * Query and calculate the statistical percentages of log keywords in Elasticsearch. - * - * @param logQuery - * @return - */ - Result> queryEsStatisticsRation(LogQuery logQuery); - - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/StreamPartitionService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/StreamPartitionService.java deleted file mode 100644 index f9c7ea5e0..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/StreamPartitionService.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service; - -import cn.hutool.core.util.PageUtil; -import com.xiaomi.mone.log.manager.model.PageVo; -import com.xiaomi.mone.log.manager.model.Pair; -import com.xiaomi.mone.log.manager.model.bo.SpacePartitionBalance; -import com.xiaomi.mone.log.manager.model.page.PageInfo; -import com.xiaomi.mone.log.manager.model.vo.MachinePartitionParam; -import com.xiaomi.mone.log.manager.model.vo.SpaceIpParam; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/9/19 15:05 - */ -public interface StreamPartitionService { - PageInfo querySpacePartitionBalance(MachinePartitionParam partitionParam); - - PageInfo> queryIpPartitionBalance(MachinePartitionParam partitionParam); - - Boolean addSpaceToIp(SpaceIpParam param); - - Boolean delSpaceToIp(SpaceIpParam param); - - PageInfo> queryStreamList(MachinePartitionParam partitionParam); - - boolean streamReBalance(); - - String queryStreamHostname(String ip); - - default PageInfo buildPageInfo(PageVo pageVo, List dataList, List pageList) { - PageInfo pageInfo = new PageInfo<>(); - pageInfo.setPage(pageVo.getPageNum()); - pageInfo.setPageSize(pageVo.getPageSize()); - pageInfo.setTotal(dataList.size()); - pageInfo.setTotalPageCount(PageUtil.totalPage(dataList.size(), pageVo.getPageSize())); - pageInfo.setList(pageList); - return pageInfo; - } - - List> findUnIncludedSpaceList(SpaceIpParam param); - - List> queryAllUniqueKeyList(SpaceIpParam param); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/bind/ConfigLogTypeProcessor.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/bind/ConfigLogTypeProcessor.java deleted file mode 100644 index 34e965800..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/bind/ConfigLogTypeProcessor.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.bind; - -import com.xiaomi.mone.log.common.Config; - -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -/** - * @author wtt - * @version 1.0 - * @description Determine how to read the configuration file - * @date 2022/12/23 14:02 - */ -//@Processor(isDefault = true, order = 100) -public class ConfigLogTypeProcessor implements LogTypeProcessor { - - private final Config config; - - public ConfigLogTypeProcessor(Config config) { - this.config = config; - } - - @Override - public boolean supportedConsume(Integer type) { - String notConsume = config.get("log_type_mq_not_consume", ""); - List logTypesNotConsume = Arrays.stream(notConsume.split(",")) - .map(Integer::valueOf).collect(Collectors.toList()); - return !logTypesNotConsume.contains(type); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/bind/DataSourceLogTypeProcessor.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/bind/DataSourceLogTypeProcessor.java deleted file mode 100644 index 0453ee00d..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/bind/DataSourceLogTypeProcessor.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.bind; - -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.xiaomi.mone.log.manager.common.exception.MilogManageException; -import com.xiaomi.mone.log.manager.mapper.MilogLogTemplateMapper; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTemplateDO; - -import java.util.Objects; - -/** - * @author wtt - * @version 1.0 - * @description Judgment of how to read the database - * @date 2022/12/23 14:03 - */ -@Processor(isDefault = true, order = 1000) -public class DataSourceLogTypeProcessor implements LogTypeProcessor { - - private static final Integer EXIST_STATUS = 1; - - private final MilogLogTemplateMapper milogLogTemplateMapper; - - public DataSourceLogTypeProcessor(MilogLogTemplateMapper milogLogTemplateMapper) { - this.milogLogTemplateMapper = milogLogTemplateMapper; - } - - @Override - public boolean supportedConsume(Integer logTypeCode) { - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("type", logTypeCode); - MilogLogTemplateDO templateDO = milogLogTemplateMapper.selectOne(queryWrapper); - if (null == templateDO) { - throw new MilogManageException("log template not exist,logtypeType:" + logTypeCode); - } - return Objects.equals(EXIST_STATUS, templateDO.getSupportedConsume()); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/bind/LogTypeProcessor.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/bind/LogTypeProcessor.java deleted file mode 100644 index 85876f47d..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/bind/LogTypeProcessor.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.bind; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/12/23 14:00 - */ -public interface LogTypeProcessor { - /** - * Whether consumption is supported - * - * @param logTypeCode - * @return - */ - boolean supportedConsume(Integer logTypeCode); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/bind/LogTypeProcessorFactory.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/bind/LogTypeProcessorFactory.java deleted file mode 100644 index a013fd63d..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/bind/LogTypeProcessorFactory.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.bind; - -import cn.hutool.core.util.ClassUtil; -import com.xiaomi.mone.log.common.Config; -import com.xiaomi.mone.log.manager.common.exception.MilogManageException; -import com.xiaomi.mone.log.manager.mapper.MilogLogTemplateMapper; -import com.xiaomi.youpin.docean.anno.Component; -import lombok.Setter; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.util.Comparator; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/12/23 14:19 - */ -@Component -public class LogTypeProcessorFactory { - - private static final ConcurrentHashMap SPI_LOG_MAP = new ConcurrentHashMap<>(); - private Config config = Config.ins(); - @Setter - private MilogLogTemplateMapper milogLogTemplateMapper; - - public LogTypeProcessor getLogTypeProcessor() { - if (SPI_LOG_MAP.isEmpty()) { - Class logTypeProcessor = getTypeProcessor(LogTypeProcessor.class); - LogTypeProcessor processor = createLogTypeProcessor(logTypeProcessor, config, milogLogTemplateMapper); - SPI_LOG_MAP.putIfAbsent(logTypeProcessor.getClass().getName(), processor); - } - if (SPI_LOG_MAP.isEmpty()) { - throw new MilogManageException("LogTypeProcessor not exist"); - } - return SPI_LOG_MAP.values().stream().findAny().get(); - } - - private LogTypeProcessor createLogTypeProcessor(Class logTypeProcessor, Config config, MilogLogTemplateMapper milogLogTemplateMapper) { - if (Objects.equals(logTypeProcessor.getName(), ConfigLogTypeProcessor.class.getName())) { - return new ConfigLogTypeProcessor(config); - } - if (Objects.equals(logTypeProcessor.getName(), DataSourceLogTypeProcessor.class.getName())) { - if (null == milogLogTemplateMapper) { - throw new MilogManageException("logTemplateMapper is null"); - } - return new DataSourceLogTypeProcessor(milogLogTemplateMapper); - } - throw new MilogManageException("Unknown LogTypeProcessor class: " + logTypeProcessor.getName()); - } - - @Nullable - private static Class getTypeProcessor(Class typeProcessorClass) { - List> classList = getClasses(typeProcessorClass).stream() - .filter(processorClass -> { - Processor processor = processorClass.getAnnotation(Processor.class); - return null != processor && processor.isDefault(); - }) - .sorted(Comparator.comparingInt(o -> o.getAnnotation(Processor.class).order())) - .collect(Collectors.toList()); - return classList.stream().findFirst().get(); - } - - @NotNull - private static List> getClasses(Class LogTypeProcessorClass) { - Set> classSet = ClassUtil.scanPackage("com.xiaomi.mone.log.manager"); - List> classes = classSet.parallelStream() - .filter(LogTypeProcessorClass::isAssignableFrom) - .filter(clazz -> !Objects.equals(clazz, LogTypeProcessorClass)) - .collect(Collectors.toList()); - return classes; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/bind/Processor.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/bind/Processor.java deleted file mode 100644 index 938b11d9a..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/bind/Processor.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.bind; - -import java.lang.annotation.*; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/12/23 15:10 - */ -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.TYPE}) -@Documented -public @interface Processor { - /** - * Alias name of Processor class - * - * @return - */ - String value() default ""; - - /** - * Whether is the default Processor - */ - boolean isDefault() default false; - - /** - * Order priority of Processor class - */ - int order() default 0; - - int ORDER_HIGHEST = Integer.MIN_VALUE; - - int ORDER_LOWEST = Integer.MAX_VALUE; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/bind/package-info.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/bind/package-info.java deleted file mode 100644 index 00f7624d5..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/bind/package-info.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * @description - * @version 1.0 - * @author wtt - * @date 2022/12/23 13:56 - */ -package com.xiaomi.mone.log.manager.service.bind; -/** - * Use the annotation mechanism to select whether to write the configuration metadata information to NACOS - * (the user determines whether this type of log topic needs to be consumed) - */ \ No newline at end of file diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/consumer/MioneRocketMqConsumer.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/consumer/MioneRocketMqConsumer.java deleted file mode 100644 index 43b4db68d..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/consumer/MioneRocketMqConsumer.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.consumer; - -import com.google.gson.Gson; -import org.apache.ozhera.app.model.vo.HeraEnvIpVo; -import com.xiaomi.mone.log.manager.service.impl.LogTailServiceImpl; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.plugin.config.anno.Value; -import lombok.extern.slf4j.Slf4j; -import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; -import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus; -import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.common.message.MessageExt; - -import javax.annotation.Resource; - -/** - * @author wtt - * @version 1.0 - * @description MQ consumption IP changes - * @date 2021/7/14 20:15 - */ -@Slf4j -@Service -public class MioneRocketMqConsumer extends RocketMqConsumer { - - @Value("$rocketmq_consumer_topic") - private String consumeTopic; - - @Resource - private DefaultMQPushConsumer consumer; - - @Resource - private LogTailServiceImpl logTailService; - - public void init() { - log.info("consumer mq service init"); - try { - consumer.subscribe(consumeTopic, ""); - } catch (MQClientException e) { - log.error("Subscription IP address changed Mq consumption abnormality", e); - } - consumer.registerMessageListener((MessageListenerOrderly) (list, consumeOrderlyContext) -> { - list.forEach(this::ipChangeConsumeMessage); - return ConsumeOrderlyStatus.SUCCESS; - }); - - try { - consumer.start(); - } catch (Exception e) { - log.error("The Rocket Mq client starts unexpectedly when the subscription creates a project", e); - } - } - - private void ipChangeConsumeMessage(MessageExt message) { - try { - byte[] body = message.getBody(); - HeraEnvIpVo heraEnvIpVo = new Gson().fromJson(new String(body), HeraEnvIpVo.class); - log.info("【Dynamic expansion】The message data consumed by Rocket Mq is converted into objects: {}", heraEnvIpVo.toString()); - logTailService.machineIpChange(heraEnvIpVo); - log.info("【Dynamic Expansion】The news consumption of Rocket Mq consumption ends"); - } catch (Throwable ex) { - log.error("【Dynamic Scaling】Rocket Mq consumption of messages is abnormal:" + ex.getMessage(), ex); - } - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/consumer/RocketMqConsumer.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/consumer/RocketMqConsumer.java deleted file mode 100644 index f22a67c8c..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/consumer/RocketMqConsumer.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.consumer; - -import org.apache.rocketmq.acl.common.AclClientRPCHook; -import org.apache.rocketmq.acl.common.SessionCredentials; -import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; -import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; -import org.apache.rocketmq.remoting.RPCHook; - -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -import static org.apache.rocketmq.common.consumer.ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/3/4 14:36 - */ -public class RocketMqConsumer { - /** - * init Mq consumer - * @param ak - * @param sk - * @param consumerGroup - * @param address - * @return - */ - public DefaultMQPushConsumer initDefaultMQPushConsumer(String ak, String sk, String consumerGroup, String address) { - DefaultMQPushConsumer defaultMQPushConsumer; - if (!ak.equals("") && !sk.equals("")) { - SessionCredentials credentials = new SessionCredentials(ak, sk); - RPCHook rpcHook = new AclClientRPCHook(credentials); - defaultMQPushConsumer = new DefaultMQPushConsumer(consumerGroup, rpcHook, new AllocateMessageQueueAveragely()); - } else { - defaultMQPushConsumer = new DefaultMQPushConsumer(consumerGroup); - } - defaultMQPushConsumer.setNamesrvAddr(address); - defaultMQPushConsumer.setConsumeFromWhere(CONSUME_FROM_LAST_OFFSET); - return defaultMQPushConsumer; - } - - /** - * cult multi tags - * @param consumeTag - * @return - */ - public String getSplitTagString(String consumeTag) { - List projectTags = Arrays.stream(consumeTag.split(",")).collect(Collectors.toList()); - return projectTags.stream().collect(Collectors.joining("||")); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/env/HeraEnvIpService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/env/HeraEnvIpService.java deleted file mode 100644 index 234cde855..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/env/HeraEnvIpService.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.env; - -import com.xiaomi.mone.log.api.model.meta.LogPattern; -import com.xiaomi.mone.log.manager.model.vo.LogAgentListBo; - -import java.util.List; -import java.util.Map; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/11/15 19:25 - */ -public interface HeraEnvIpService { - /** - * Query all pod information in the node - * - * @param nodeIp - * @return - */ - List queryInfoByNodeIp(String nodeIp); - - Map> queryAgentIpByPodIps(List podIps); - - /** - * Query the node IP based on the pode IP - * - * @param ips - * @return - */ - List queryActualIps(List ips, String agentIp, String logPath); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/env/HeraEnvIpServiceFactory.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/env/HeraEnvIpServiceFactory.java deleted file mode 100644 index 19bfe8425..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/env/HeraEnvIpServiceFactory.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.env; - -import com.xiaomi.mone.log.api.enums.ProjectTypeEnum; -import com.xiaomi.youpin.docean.anno.Service; - -import javax.annotation.Resource; -import java.util.Objects; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/11/15 19:28 - */ -@Service -public class HeraEnvIpServiceFactory { - - @Resource - private MoneHeraEnvIpService moneHeraEnvIpService; - - public HeraEnvIpService getHeraEnvIpServiceByAppType(Integer code) { - - if (Objects.equals(ProjectTypeEnum.MIONE_TYPE.getCode(), code)) { - return moneHeraEnvIpService; - } - return moneHeraEnvIpService; - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/env/MoneHeraEnvIpService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/env/MoneHeraEnvIpService.java deleted file mode 100644 index 651ba42be..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/env/MoneHeraEnvIpService.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.env; - -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.api.model.meta.LogPattern; -import com.xiaomi.mone.log.manager.model.vo.LogAgentListBo; -import com.xiaomi.youpin.docean.anno.Service; - -import java.util.List; -import java.util.Map; - -import static com.xiaomi.mone.log.manager.common.utils.ManagerUtil.getPhysicsDirectory; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/11/16 15:41 - */ -@Service -public class MoneHeraEnvIpService implements HeraEnvIpService { - - @Override - public List queryInfoByNodeIp(String nodeIp) { - return Lists.newArrayList(); - } - - @Override - public Map> queryAgentIpByPodIps(List podIps) { - return null; - } - - @Override - public List queryActualIps(List ips, String agentIp, String logPath) { - return Lists.newArrayList(LogPattern.IPRel.builder().key(getPhysicsDirectory(logPath)).ip(agentIp).build()); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/agent/MilogAgentService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/agent/MilogAgentService.java deleted file mode 100644 index cf1f17beb..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/agent/MilogAgentService.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.extension.agent; - -import com.xiaomi.mone.log.api.model.meta.LogCollectMeta; -import com.xiaomi.mone.log.api.model.vo.AgentLogProcessDTO; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.model.bo.MilogAgentIpParam; - -import java.util.List; - -public interface MilogAgentService { - - String DEFAULT_AGENT_EXTENSION_SERVICE_KEY = "defaultAgentExtensionService"; - - String LOG_PATH_PREFIX = "/home/work/log"; - - Result> process(String ip); - - Result configIssueAgent(String agentId, String agentIp, String agentMachine); - - void publishIncrementConfig(Long tailId, Long milogAppId, List ips); - - void publishIncrementDel(Long tailId, Long milogAppId, List ips); - - void delLogCollDirectoryByIp(Long tailId, String directory, List ips); - - Result agentOfflineBatch(MilogAgentIpParam agentIpParam); - - LogCollectMeta getLogCollectMetaFromManager(String ip); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/agent/MilogAgentServiceFactory.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/agent/MilogAgentServiceFactory.java deleted file mode 100644 index 9ca68855a..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/agent/MilogAgentServiceFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.extension.agent; - -import com.xiaomi.mone.log.common.Config; -import com.xiaomi.youpin.docean.Ioc; -import lombok.extern.slf4j.Slf4j; - -import static com.xiaomi.mone.log.manager.service.extension.agent.MilogAgentService.DEFAULT_AGENT_EXTENSION_SERVICE_KEY; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/4/10 21:01 - */ -@Slf4j -public class MilogAgentServiceFactory { - - private static String factualServiceName; - - public static MilogAgentService getAgentExtensionService() { - factualServiceName = Config.ins().get("agent.extension.service", DEFAULT_AGENT_EXTENSION_SERVICE_KEY); - log.debug("StoreExtensionServiceFactory factualServiceName:{}", factualServiceName); - return Ioc.ins().getBean(factualServiceName); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/agent/MilogAgentServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/agent/MilogAgentServiceImpl.java deleted file mode 100644 index 7ba37c0da..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/agent/MilogAgentServiceImpl.java +++ /dev/null @@ -1,395 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.extension.agent; - -import com.google.common.collect.Lists; -import com.google.gson.Gson; -import com.xiaomi.data.push.context.AgentContext; -import com.xiaomi.data.push.rpc.netty.AgentChannel; -import org.apache.ozhera.app.api.response.AppBaseInfo; -import com.xiaomi.mone.log.api.enums.LogTypeEnum; -import com.xiaomi.mone.log.api.enums.MQSourceEnum; -import com.xiaomi.mone.log.api.enums.OperateEnum; -import com.xiaomi.mone.log.api.model.meta.*; -import com.xiaomi.mone.log.api.model.vo.AgentLogProcessDTO; -import com.xiaomi.mone.log.api.service.PublishConfigService; -import com.xiaomi.mone.log.common.Constant; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.common.Utils; -import com.xiaomi.mone.log.manager.dao.MilogAppMiddlewareRelDao; -import com.xiaomi.mone.log.manager.dao.MilogLogTailDao; -import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; -import com.xiaomi.mone.log.manager.dao.MilogMiddlewareConfigDao; -import com.xiaomi.mone.log.manager.domain.LogProcess; -import com.xiaomi.mone.log.manager.model.bo.MilogAgentIpParam; -import com.xiaomi.mone.log.manager.model.pojo.MilogAppMiddlewareRel; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.mone.log.manager.model.pojo.MilogMiddlewareConfig; -import com.xiaomi.mone.log.manager.service.env.HeraEnvIpService; -import com.xiaomi.mone.log.manager.service.env.HeraEnvIpServiceFactory; -import com.xiaomi.mone.log.manager.service.impl.HeraAppServiceImpl; -import com.xiaomi.mone.log.manager.service.impl.LogTailServiceImpl; -import com.xiaomi.mone.log.manager.service.path.LogPathMapping; -import com.xiaomi.mone.log.manager.service.path.LogPathMappingFactory; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.common.NamedThreadFactory; -import com.xiaomi.youpin.docean.plugin.dubbo.anno.Reference; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; - -import javax.annotation.Resource; -import java.util.*; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.SYMBOL_COLON; -import static com.xiaomi.mone.log.manager.service.extension.agent.MilogAgentService.DEFAULT_AGENT_EXTENSION_SERVICE_KEY; - -@Service(name = DEFAULT_AGENT_EXTENSION_SERVICE_KEY) -@Slf4j -public class MilogAgentServiceImpl implements MilogAgentService { - - @Resource - private LogPathMappingFactory logPathMappingFactory; - - @Resource - private HeraEnvIpServiceFactory heraEnvIpServiceFactory; - - @Resource - private MilogLogTailDao milogLogtailDao; - - @Resource - private MilogLogstoreDao logstoreDao; - - private Gson gson = Constant.GSON; - - @Resource - private LogProcess logProcess; - - @Resource - private MilogAppMiddlewareRelDao milogAppMiddlewareRelDao; - @Resource - private MilogMiddlewareConfigDao milogMiddlewareConfigDao; - - @Resource - private HeraAppServiceImpl heraAppService; - @Resource - private LogTailServiceImpl logTailService; - - @Reference(interfaceClass = PublishConfigService.class, group = "$dubbo.env.group", check = false, timeout = 14000) - private PublishConfigService publishConfigService; - - private static final ThreadPoolExecutor THREAD_POOL_EXECUTOR; - - - private static final AtomicInteger COUNT_INCR = new AtomicInteger(0); - - static { - THREAD_POOL_EXECUTOR = new ThreadPoolExecutor(6, 20, - 1, TimeUnit.MINUTES, new ArrayBlockingQueue<>(200), - new NamedThreadFactory("coll-base-data-start", true), - new ThreadPoolExecutor.DiscardOldestPolicy()); - THREAD_POOL_EXECUTOR.allowCoreThreadTimeOut(true); - } - - @Override - public Result> process(String ip) { - List dtoList = logProcess.getAgentLogProcess(ip); - return Result.success(dtoList); - } - - @Override - public Result configIssueAgent(String agentId, String agentIp, String agentMachine) { - if (StringUtils.isEmpty(agentIp)) { - return Result.failParam("The agent IP cannot be empty"); - } - // 1.Query the full configuration of the physical machine - LogCollectMeta logCollectMeta = queryMilogAgentConfig(agentId, agentIp, agentMachine); - log.info("{},this ip config data:{}", agentIp, gson.toJson(logCollectMeta)); - String k8sNodeIP = queryNodeIpByPodIp(agentIp); - if (StringUtils.isNotEmpty(k8sNodeIP)) { - log.info("query k8s ip succeed,ip:{},k8sNodeIP:{}", agentIp, k8sNodeIP); - agentIp = k8sNodeIP; - } - List ipAddress = Lists.newArrayList(); - AgentContext.ins().map.entrySet().forEach(agentChannelEntry -> { - String key = agentChannelEntry.getKey(); - ipAddress.add(StringUtils.substringBefore(key, SYMBOL_COLON)); - } - ); - log.info("agent ip list:{}", gson.toJson(ipAddress)); - //2.Delivery configuration - sengConfigToAgent(agentIp, logCollectMeta); - return Result.success("success"); - } - - /** - * Deliver the configuration to the specified IP address - * - * @param logCollectMeta - * @param agentIp - */ - public void sengConfigToAgent(final String agentIp, LogCollectMeta logCollectMeta) { - if (CollectionUtils.isEmpty(logCollectMeta.getAppLogMetaList()) || logCollectMeta.getAppLogMetaList() - .stream().allMatch(appLogMeta -> CollectionUtils.isEmpty(appLogMeta.getLogPatternList()))) { - return; - } - // Placed in the thread pool for execution - THREAD_POOL_EXECUTOR.execute(() -> { - publishConfigService.sengConfigToAgent(agentIp, logCollectMeta); - }); - } - - /** - * Send the configuration of the delta - * 1.Find all the physical IP addresses where the app is deployed - * 2.This new configuration is pushed to these physical machines or containers - * - * @param milogAppId - * @param ips - */ - @Override - public void publishIncrementConfig(Long tailId, Long milogAppId, List ips) { - log.info("push agent params,milogAppId:{},ips:{}", milogAppId, ips); - if (CollectionUtils.isEmpty(ips)) { - return; - } - printMangerInfo(); - AppBaseInfo appBaseInfo = heraAppService.queryById(milogAppId); - ips.forEach(ip -> { - AppLogMeta appLogMeta = assembleSingleConfig(milogAppId, queryLogPattern(milogAppId, ip, appBaseInfo.getPlatformType())); - LogCollectMeta logCollectMeta = new LogCollectMeta(); - logCollectMeta.setAgentIp(ip); - logCollectMeta.setAppLogMetaList(Arrays.asList(appLogMeta)); - AgentDefine agentDefine = new AgentDefine(); - agentDefine.setFilters(new ArrayList<>()); - logCollectMeta.setAgentDefine(agentDefine); - log.info("push agent config data,ip:{},{}", ip, gson.toJson(logCollectMeta)); - sengConfigToAgent(ip, logCollectMeta); - }); - } - - @NotNull - private Map getAgentChannelMap() { - Map logAgentMap = new HashMap<>(); - AgentContext.ins().map.forEach((k, v) -> logAgentMap.put(StringUtils.substringBefore(k, SYMBOL_COLON), v)); - return logAgentMap; - } - - private void printMangerInfo() { - List remoteAddress = publishConfigService.getAllAgentList(); - if (COUNT_INCR.getAndIncrement() % 200 == 0) { - log.info("The set of remote addresses for the connected agent machine is:{}", gson.toJson(remoteAddress)); - } - } - - public String queryNodeIpByPodIp(String ip) { - return ip; - } - - - @Override - public void publishIncrementDel(Long tailId, Long milogAppId, List ips) { - log.info("Delete the configuration synchronization to logAgent,tailId:{},milogAppId:{},ips:{}", tailId, milogAppId, gson.toJson(ips)); - AppLogMeta appLogMeta = new AppLogMeta(); - LogPattern logPattern = new LogPattern(); - assemblyAppInfo(milogAppId, appLogMeta); - logPattern.setLogtailId(tailId); - logPattern.setOperateEnum(OperateEnum.DELETE_OPERATE); - - appLogMeta.setLogPatternList(Arrays.asList(logPattern)); - ips.forEach(ip -> { - LogCollectMeta logCollectMeta = new LogCollectMeta(); - logCollectMeta.setAgentIp(ip); - logCollectMeta.setAgentMachine(""); - logCollectMeta.setAgentId(""); - logCollectMeta.setAppLogMetaList(Arrays.asList(appLogMeta)); - sengConfigToAgent(ip, logCollectMeta); - }); - } - - private void assemblyAppInfo(Long milogAppId, AppLogMeta appLogMeta) { - AppBaseInfo appBaseInfo = heraAppService.queryById(milogAppId); - appLogMeta.setAppId(milogAppId); - if (null != appBaseInfo) { - appLogMeta.setAppName(appBaseInfo.getAppName()); - } - } - - @Override - public void delLogCollDirectoryByIp(Long tailId, String directory, List ips) { - log.info("delLogCollDirectoryByIp logAgent,tailId:{},directory:{},ips:{}", tailId, directory, gson.toJson(ips)); - AppLogMeta appLogMeta = new AppLogMeta(); - LogPattern logPattern = new LogPattern(); - logPattern.setLogtailId(tailId); - logPattern.setOperateEnum(OperateEnum.DELETE_OPERATE); - appLogMeta.setLogPatternList(Arrays.asList(logPattern)); - LogCollectMeta logCollectMeta = new LogCollectMeta(); - for (String ip : ips) { - logCollectMeta.setAgentIp(ip); - logCollectMeta.setDelDirectory(directory); - logCollectMeta.setAppLogMetaList(Arrays.asList(appLogMeta)); - sengConfigToAgent(ip, logCollectMeta); - } - } - - @Override - public Result agentOfflineBatch(MilogAgentIpParam agentIpParam) { - if (null == agentIpParam || CollectionUtils.isEmpty(agentIpParam.getIps())) { - return Result.failParam("IP cannot be empty"); - } - return null; - } - - @Override - public LogCollectMeta getLogCollectMetaFromManager(String ip) { - return queryMilogAgentConfig("", ip, ""); - } - - /** - * Query the full configuration of the IP address of the physical machine - * - * @param agentId - * @param agentIp Physical IP - * @param agentMachine - * @return - */ - public LogCollectMeta queryMilogAgentConfig(String agentId, String agentIp, String agentMachine) { - LogCollectMeta logCollectMeta = buildLogCollectMeta(agentIp); - List appBaseInfos = Lists.newArrayList(); - List logTailDos = milogLogtailDao.queryByIp(agentIp); - if (CollectionUtils.isNotEmpty(logTailDos)) { - appBaseInfos = heraAppService.queryByIds( - logTailDos.stream().map(MilogLogTailDo::getMilogAppId) - .distinct() - .collect(Collectors.toList()) - ); - } - logCollectMeta.setAppLogMetaList(appBaseInfos.stream() - .map(appBaseInfo -> assembleSingleConfig(appBaseInfo.getId().longValue(), queryLogPattern(appBaseInfo.getId().longValue(), agentIp, appBaseInfo.getPlatformType()))) - .filter(appLogMeta -> CollectionUtils.isNotEmpty(appLogMeta.getLogPatternList())) - .collect(Collectors.toList())); - return logCollectMeta; - } - - private LogCollectMeta buildLogCollectMeta(String agentIp) { - LogCollectMeta logCollectMeta = new LogCollectMeta(); - logCollectMeta.setAgentIp(agentIp); - logCollectMeta.setAgentMachine(""); - logCollectMeta.setAgentId(""); - return logCollectMeta; - } - - /** - * Assemble incremental configurations - * - * @param milogAppId - * @param logPatternList - * @return - */ - private AppLogMeta assembleSingleConfig(Long milogAppId, List logPatternList) { - AppLogMeta appLogMeta = new AppLogMeta(); - assemblyAppInfo(milogAppId, appLogMeta); - appLogMeta.setLogPatternList(logPatternList); - return appLogMeta; - } - - private MQConfig decorateMQConfig(MilogLogTailDo milogLogtailDo) { - MQConfig mqConfig = new MQConfig(); - try { - Long mqResourceId = logstoreDao.queryById(milogLogtailDo.getStoreId()).getMqResourceId(); - - List milogAppMiddlewareRels = milogAppMiddlewareRelDao.queryByCondition(milogLogtailDo.getMilogAppId(), mqResourceId, milogLogtailDo.getId()); - - if (CollectionUtils.isEmpty(milogAppMiddlewareRels)) { - milogAppMiddlewareRels = milogAppMiddlewareRelDao.queryByCondition(milogLogtailDo.getMilogAppId(), null, milogLogtailDo.getId()); - } - - MilogAppMiddlewareRel milogAppMiddlewareRel = milogAppMiddlewareRels.get(milogAppMiddlewareRels.size() - 1); - - MilogMiddlewareConfig middlewareConfig = milogMiddlewareConfigDao.queryById(milogAppMiddlewareRel.getMiddlewareId()); - mqConfig.setClusterInfo(middlewareConfig.getNameServer()); - fillMqConfigData(mqConfig, MQSourceEnum.queryName(middlewareConfig.getType()), middlewareConfig, milogAppMiddlewareRel.getConfig()); - } catch (Exception e) { - log.error("The assembly MQ configuration information is abnormal,data:{}", gson.toJson(milogLogtailDo), e); - } - return mqConfig; - } - - private void fillMqConfigData(MQConfig mqConfig, String typeName, MilogMiddlewareConfig middlewareConfig, MilogAppMiddlewareRel.Config config) { - mqConfig.setType(typeName); - mqConfig.setAk(middlewareConfig.getAk()); - mqConfig.setProducerGroup(config.getConsumerGroup()); - mqConfig.setSk(middlewareConfig.getSk()); - mqConfig.setTopic(config.getTopic()); - mqConfig.setTag(config.getTag()); - mqConfig.setPartitionCnt(config.getPartitionCnt()); - mqConfig.setEsConsumerGroup(config.getEsConsumerGroup()); - mqConfig.setBatchSendSize(config.getBatchSendSize()); - } - - private List queryLogPattern(Long milogAppId, String agentIp, Integer type) { - List milogLogtailDos = milogLogtailDao.queryByAppIdAgentIp(milogAppId, agentIp); - if (CollectionUtils.isNotEmpty(milogLogtailDos)) { - return milogLogtailDos.stream().map(milogLogtailDo -> { - log.info("assemble data:{}", gson.toJson(milogAppId)); - LogPattern logPattern = generateLogPattern(milogLogtailDo); - logPattern.setIps(Lists.newArrayList(agentIp)); - logPattern.setIpDirectoryRel(Lists.newArrayList(LogPattern.IPRel.builder().ip(agentIp).build())); - LogPathMapping logPathMapping = logPathMappingFactory.queryLogPathMappingByAppType(type); - HeraEnvIpService heraEnvIpService = heraEnvIpServiceFactory.getHeraEnvIpServiceByAppType(type); - try { - logPattern.setLogPattern(logPathMapping.getLogPath(milogLogtailDo.getLogPath(), null)); - logPattern.setLogSplitExpress(logPathMapping.getLogPath(milogLogtailDo.getLogSplitExpress(), null)); - logPattern.setIpDirectoryRel(heraEnvIpService.queryActualIps(milogLogtailDo.getIps(), agentIp, milogLogtailDo.getLogPath())); - } catch (Exception e) { - log.error("assemble log path data error:", e); - } - //Set the MQ configuration - MQConfig mqConfig = decorateMQConfig(milogLogtailDo); - logPattern.setMQConfig(mqConfig); - return logPattern; - }).collect(Collectors.toList()); - } - return Lists.newArrayList(); - } - - private LogPattern generateLogPattern(MilogLogTailDo milogLogtailDo) { - LogPattern logPattern = new LogPattern(); - MilogLogStoreDO milogLogstoreDO = logstoreDao.queryById(milogLogtailDo.getStoreId()); - logPattern.setLogtailId(milogLogtailDo.getId()); - logPattern.setTailName(milogLogtailDo.getTail()); - logPattern.setLogPattern(milogLogtailDo.getLogPath()); - logPattern.setLogSplitExpress(milogLogtailDo.getLogSplitExpress()); - logPattern.setFilters(milogLogtailDo.getFilter()); - logPattern.setFirstLineReg(milogLogtailDo.getFirstLineReg()); - if (null != milogLogstoreDO && null != milogLogstoreDO.getLogType()) { - logPattern.setLogType(milogLogstoreDO.getLogType()); - if (LogTypeEnum.NGINX.getType().equals(milogLogstoreDO.getLogType())) { - logPattern.setLogPattern(milogLogtailDo.getLogPath()); - } - } - String tag = Utils.createTag(milogLogtailDo.getSpaceId(), milogLogtailDo.getStoreId(), milogLogtailDo.getId()); - logPattern.setPatternCode(tag); - return logPattern; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/CommonExtensionService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/CommonExtensionService.java deleted file mode 100644 index b658f1c33..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/CommonExtensionService.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.extension.common; - -import com.xiaomi.mone.log.manager.model.vo.LogQuery; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.TermQueryBuilder; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/4/27 16:13 - */ -public interface CommonExtensionService { - - String DEFAULT_COMMON_EXTENSION_SERVICE_KEY = "defaultCommonExtensionService"; - - - String getLogManagePrefix(); - - String getHeraLogStreamServerName(); - - String getMachineRoomName(String machineRoomEn); - - boolean middlewareEnumValid(Integer type); - - BoolQueryBuilder commonRangeQuery(LogQuery logQuery); - - String getSortedKey(LogQuery logQuery, String sortedKey); - - TermQueryBuilder multipleChooseBuilder(DefaultCommonExtensionService.QueryTypeEnum queryTypeEnum, Long storeId, String chooseVal); - - String queryDateHistogramField(Long storeId); - - String getSearchIndex(Long logStoreId, String esIndex); - - String getSpaceDataId(Long spaceId); - - List queryMachineRegions(); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/CommonExtensionServiceFactory.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/CommonExtensionServiceFactory.java deleted file mode 100644 index eec32b0f4..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/CommonExtensionServiceFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.extension.common; - -import com.xiaomi.mone.log.common.Config; -import com.xiaomi.youpin.docean.Ioc; -import lombok.extern.slf4j.Slf4j; - -import static com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionService.DEFAULT_COMMON_EXTENSION_SERVICE_KEY; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/4/27 16:16 - */ -@Slf4j -public class CommonExtensionServiceFactory { - - private static String factualServiceName; - - public static CommonExtensionService getCommonExtensionService() { - factualServiceName = Config.ins().get("common.extension.service", DEFAULT_COMMON_EXTENSION_SERVICE_KEY); - log.debug("DictionaryExtensionServiceFactory factualServiceName:{}", factualServiceName); - return Ioc.ins().getBean(factualServiceName); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/DefaultCommonExtensionService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/DefaultCommonExtensionService.java deleted file mode 100644 index 84a234d03..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/DefaultCommonExtensionService.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.extension.common; - -import com.xiaomi.mone.log.api.enums.MQSourceEnum; -import com.xiaomi.mone.log.api.enums.MachineRegionEnum; -import com.xiaomi.mone.log.manager.model.vo.LogQuery; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.TermQueryBuilder; - -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.*; -import static com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionService.DEFAULT_COMMON_EXTENSION_SERVICE_KEY; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/4/27 16:13 - */ -@Service(name = DEFAULT_COMMON_EXTENSION_SERVICE_KEY) -@Slf4j -public class DefaultCommonExtensionService implements CommonExtensionService { - - @Override - public String getLogManagePrefix() { - return LOG_MANAGE_PREFIX; - } - - @Override - public String getHeraLogStreamServerName() { - return DEFAULT_STREAM_SERVER_NAME; - } - - @Override - public String getMachineRoomName(String machineRoomEn) { - return MachineRegionEnum.queryCnByEn(machineRoomEn); - } - - @Override - public boolean middlewareEnumValid(Integer type) { - return Arrays.stream(MQSourceEnum.values()).map(MQSourceEnum::getCode).toList().contains(type); - } - - @Override - public BoolQueryBuilder commonRangeQuery(LogQuery logQuery) { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.filter(QueryBuilders.rangeQuery("timestamp").from(logQuery.getStartTime()).to(logQuery.getEndTime())); - boolQueryBuilder.filter(QueryBuilders.termQuery("storeId", logQuery.getStoreId())); - return boolQueryBuilder; - } - - @Override - public String getSortedKey(LogQuery logQuery, String sortedKey) { - return sortedKey; - } - - @Override - public TermQueryBuilder multipleChooseBuilder(DefaultCommonExtensionService.QueryTypeEnum queryTypeEnum, Long storeId, String chooseVal) { - if (QueryTypeEnum.ID == queryTypeEnum) { - return QueryBuilders.termQuery("tailId", chooseVal); - } - return QueryBuilders.termQuery("tail", chooseVal); - } - - @Override - public String queryDateHistogramField(Long storeId) { - return "timestamp"; - } - - @Override - public String getSearchIndex(Long logStoreId, String esIndexName) { - return esIndexName; - } - - @Override - public String getSpaceDataId(Long spaceId) { - return getLogManagePrefix() + NAMESPACE_CONFIG_DATA_ID; - } - - @Override - public List queryMachineRegions() { - return Arrays.stream(MachineRegionEnum.values()).map(MachineRegionEnum::getEn).collect(Collectors.toList()); - } - - @Getter - public static enum QueryTypeEnum { - ID, - TEXT - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/dictionary/DefaultDictionaryExtensionService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/dictionary/DefaultDictionaryExtensionService.java deleted file mode 100644 index c41d6fb48..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/dictionary/DefaultDictionaryExtensionService.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.extension.dictionary; - -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.api.enums.*; -import com.xiaomi.mone.log.manager.dao.MilogMiddlewareConfigDao; -import com.xiaomi.mone.log.manager.model.dto.DictionaryDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.mone.log.manager.model.pojo.MilogMiddlewareConfig; -import com.xiaomi.mone.log.manager.service.impl.KafkaMqConfigService; -import com.xiaomi.mone.log.manager.service.impl.RocketMqConfigService; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; - -import javax.annotation.Resource; -import java.util.Arrays; -import java.util.List; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.manager.service.extension.dictionary.DictionaryExtensionService.DEFAULT_DICTIONARY_EXTENSION_SERVICE_KEY; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/4/12 10:36 - */ -@Service(name = DEFAULT_DICTIONARY_EXTENSION_SERVICE_KEY) -@Slf4j -public class DefaultDictionaryExtensionService implements DictionaryExtensionService { - - @Resource - private MilogMiddlewareConfigDao milogMiddlewareConfigDao; - - @Resource - private RocketMqConfigService rocketMqConfigService; - - @Resource - private KafkaMqConfigService kafkaMqConfigService; - - @Override - public List> queryMiddlewareConfigDictionary(String monitorRoomEn) { - List milogMiddlewareConfigs = milogMiddlewareConfigDao.queryCurrentMontorRoomMQ(monitorRoomEn); - List> dictionaryDTOS = Lists.newArrayList(); - Arrays.stream(MQSourceEnum.values()) - .forEach(sourceEnum -> { - DictionaryDTO dictionaryDTO = new DictionaryDTO<>(); - dictionaryDTO.setValue(sourceEnum.getCode()); - dictionaryDTO.setLabel(sourceEnum.getName()); - if (CollectionUtils.isNotEmpty(milogMiddlewareConfigs)) { - dictionaryDTO.setChildren(milogMiddlewareConfigs.stream().filter(middlewareConfig -> sourceEnum.getCode().equals(middlewareConfig.getType())).map(middlewareConfig -> { - DictionaryDTO childDictionaryDTO = new DictionaryDTO<>(); - childDictionaryDTO.setValue(middlewareConfig.getId()); - childDictionaryDTO.setLabel(middlewareConfig.getAlias()); -// if (MQSourceEnum.ROCKETMQ.getCode().equals(middlewareConfig.getType())) { -// List existsTopic = rocketMqConfigService.queryExistsTopic(middlewareConfig.getAk(), middlewareConfig.getSk(), -// middlewareConfig.getNameServer(), middlewareConfig.getServiceUrl(), middlewareConfig.getAuthorization(), middlewareConfig.getOrgId(), middlewareConfig.getTeamId()); -// childDictionaryDTO.setChildren(existsTopic); -// } - return childDictionaryDTO; - }).collect(Collectors.toList())); - } - dictionaryDTOS.add(dictionaryDTO); - }); - return dictionaryDTOS; - } - - @Override - public List> queryResourceDictionary() { - return generateCommonDictionary(middlewareEnum -> Boolean.TRUE); - } - - @Override - public List> queryAppType() { - return Arrays.stream(ProjectTypeEnum.values()) - .map(projectTypeEnum -> { - DictionaryDTO dictionaryDTO = new DictionaryDTO<>(); - dictionaryDTO.setValue(projectTypeEnum.getCode()); - dictionaryDTO.setLabel(projectTypeEnum.getType()); - - dictionaryDTO.setShowDeploymentType(Boolean.TRUE); - dictionaryDTO.setShowEnvGroup(Boolean.TRUE); - dictionaryDTO.setShowServiceIp(Boolean.TRUE); - dictionaryDTO.setShowMqConfig(Boolean.TRUE); - - return dictionaryDTO; - }) - .collect(Collectors.toList()); - } - - @Override - public List querySpecialTails() { - return Lists.newArrayList(); - } - - @Override - public List> queryMachineRegion() { - List> dictionaryDTOS = Lists.newArrayList(); - for (MachineRegionEnum value : MachineRegionEnum.values()) { - DictionaryDTO dictionaryDTO = new DictionaryDTO(); - dictionaryDTO.setLabel(value.getCn()); - dictionaryDTO.setValue(value.getEn()); - dictionaryDTOS.add(dictionaryDTO); - } - return dictionaryDTOS; - } - - @Override - public List> queryDeployWay() { - List> dictionaryDTOS = Lists.newArrayList(); - for (DeployWayEnum value : DeployWayEnum.values()) { - DictionaryDTO dictionaryDTO = new DictionaryDTO(); - dictionaryDTO.setLabel(value.getName()); - dictionaryDTO.setValue(value.getCode()); - dictionaryDTOS.add(dictionaryDTO); - } - return dictionaryDTOS; - } - - @Override - public List> queryResourceTypeDictionary() { - return generateCommonDictionary(middlewareEnum -> ResourceEnum.MQ == middlewareEnum | - ResourceEnum.STORAGE == middlewareEnum); - } - - @Override - public List queryExistsTopic(String ak, String sk, String nameServer, String serviceUrl, String authorization, String orgId, String teamId) { - return Lists.newArrayList(); - } - - @Override - public List> queryMQDictionary() { - return Arrays.stream(MQSourceEnum.values()).map(mqSourceEnum -> { - DictionaryDTO dictionaryDTO = new DictionaryDTO<>(); - dictionaryDTO.setValue(mqSourceEnum.getCode()); - dictionaryDTO.setLabel(mqSourceEnum.getName()); - if (MQSourceEnum.ROCKETMQ == mqSourceEnum) { - dictionaryDTO.setShowBrokerName(Boolean.TRUE); - } - return dictionaryDTO; - }).collect(Collectors.toList()); - } - - private List> generateCommonDictionary(Predicate filter) { - List rDictionaryDTOS = Arrays.stream(MachineRegionEnum.values()) - .map(machineRegionEnum -> - DictionaryDTO.Of(machineRegionEnum.getEn(), machineRegionEnum.getCn())) - .collect(Collectors.toList()); - return Arrays.stream(ResourceEnum.values()) - .filter(filter) - .map(middlewareEnum -> { - DictionaryDTO dictionaryDTO = new DictionaryDTO<>(); - dictionaryDTO.setValue(middlewareEnum.getCode()); - dictionaryDTO.setLabel(middlewareEnum.getName()); - dictionaryDTO.setChildren(rDictionaryDTOS); - return dictionaryDTO; - }).collect(Collectors.toList()); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/dictionary/DictionaryExtensionService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/dictionary/DictionaryExtensionService.java deleted file mode 100644 index 53a296854..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/dictionary/DictionaryExtensionService.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.extension.dictionary; - -import com.xiaomi.mone.log.manager.model.dto.DictionaryDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/4/12 10:35 - */ -public interface DictionaryExtensionService { - - String DEFAULT_DICTIONARY_EXTENSION_SERVICE_KEY = "defaultDictionaryExtensionService"; - - List> queryMiddlewareConfigDictionary(String monitorRoomEn); - - List> queryResourceDictionary(); - - List> queryAppType(); - - List querySpecialTails(); - - List> queryMachineRegion(); - - List> queryDeployWay(); - - List> queryResourceTypeDictionary(); - - List queryExistsTopic(String ak, String sk, String nameServer, String serviceUrl, - String authorization, String orgId, String teamId); - - List> queryMQDictionary(); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/dictionary/DictionaryExtensionServiceFactory.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/dictionary/DictionaryExtensionServiceFactory.java deleted file mode 100644 index ccc719d48..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/dictionary/DictionaryExtensionServiceFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.extension.dictionary; - -import com.xiaomi.mone.log.common.Config; -import com.xiaomi.youpin.docean.Ioc; -import lombok.extern.slf4j.Slf4j; - -import static com.xiaomi.mone.log.manager.service.extension.dictionary.DictionaryExtensionService.DEFAULT_DICTIONARY_EXTENSION_SERVICE_KEY; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/4/12 10:40 - */ -@Slf4j -public class DictionaryExtensionServiceFactory { - - private static String factualServiceName; - - public static DictionaryExtensionService getDictionaryExtensionService() { - factualServiceName = Config.ins().get("directory.extension.service", DEFAULT_DICTIONARY_EXTENSION_SERVICE_KEY); - log.debug("DictionaryExtensionServiceFactory factualServiceName:{}", factualServiceName); - return Ioc.ins().getBean(factualServiceName); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/resource/DefaultResourceExtensionService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/resource/DefaultResourceExtensionService.java deleted file mode 100644 index db0593e3b..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/resource/DefaultResourceExtensionService.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.extension.resource; - -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.api.enums.MQSourceEnum; -import com.xiaomi.mone.log.api.enums.MiddlewareEnum; -import com.xiaomi.mone.log.api.model.bo.MiLogResource; -import com.xiaomi.mone.log.api.model.vo.ResourceUserSimple; -import com.xiaomi.mone.log.manager.common.context.MoneUserContext; -import com.xiaomi.mone.log.manager.dao.MilogSpaceDao; -import com.xiaomi.mone.log.manager.model.pojo.*; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.YES; -import static com.xiaomi.mone.log.manager.service.extension.resource.ResourceExtensionService.DEFAULT_RESOURCE_EXTENSION_SERVICE_KEY; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/4/11 10:01 - */ -@Service(name = DEFAULT_RESOURCE_EXTENSION_SERVICE_KEY) -@Slf4j -public class DefaultResourceExtensionService implements ResourceExtensionService { - - @Resource - private MilogSpaceDao logSpaceDao; - - @Override - public List userShowAuthority(List configList) { - return configList; - } - - @Override - public void filterEsQueryWrapper(QueryWrapper queryWrapper) { - - } - - @Override - public List generateResourceLabels(String id) { - return Lists.newArrayList(); - } - - @Override - public void addResourcePreProcessing(List resourceLabels, MiLogResource miLogResource) { - - } - - @Override - public void addEsResourcePreProcessing(MilogEsClusterDO esClusterDO) { - if (MoneUserContext.getCurrentUser().getIsAdmin()) { - esClusterDO.setIsDefault(YES); - } - } - - @Override - public void addResourceMiddleProcessing(MiLogResource miLogResource) { -// mqConfigService.createCommonTagTopic(miLogResource.getAk(), miLogResource.getSk(), miLogResource.getClusterName(), -// miLogResource.getServiceUrl(), StringUtils.EMPTY, miLogResource.getOrgId(), -// miLogResource.getBrokerName()); - } - - @Override - public void addResourcePostProcessing(MilogMiddlewareConfig milogMiddlewareConfig) { - /** - * Currently, one configuration is shared by default - */ - if (MoneUserContext.getCurrentUser().getIsAdmin()) { - milogMiddlewareConfig.setIsDefault(YES); - } - } - - @Override - public boolean userResourceListPre(Integer logTypeCode) { - return false; - } - - @Override - public List currentUserConfigFilter(List middlewareConfigs) { - if (MoneUserContext.getCurrentUser().getIsAdmin()) { - return middlewareConfigs.stream().filter(milogMiddlewareConfig -> Objects.equals(YES, milogMiddlewareConfig.getIsDefault())).collect(Collectors.toList()); - } - return middlewareConfigs; - } - - @Override - public boolean resourceNotRequiredInit(Integer logTypeCode, List middlewareMqConfigs, List middlewareEsConfigs, List esIndexDOList) { - return CollectionUtils.isNotEmpty(middlewareMqConfigs) && - CollectionUtils.isNotEmpty(middlewareEsConfigs) && - CollectionUtils.isNotEmpty(esIndexDOList); - } - - @Override - public boolean resourceShowStatusFlag(ResourceUserSimple configResource) { - if (MoneUserContext.getCurrentUser().getIsAdmin()) { - configResource.setShowFlag(Boolean.FALSE); - return false; - } - configResource.setShowFlag(Boolean.TRUE); - return Boolean.TRUE; - } - - @Override - public Integer getResourceCode() { - return MiddlewareEnum.ROCKETMQ.getCode(); - } - - @Override - public void deleteMqResourceProcessing(MilogLogTailDo mt, MilogLogStoreDO logStoreDO) { - - } - - @Override - public List getMqResourceCodeList() { - return Arrays.stream(MQSourceEnum.values()).map(MQSourceEnum::getCode).collect(Collectors.toList()); - } - - @Override - public String queryHostName(String ip) { - return ip; - } - - @Override - public List getSpaceIdsByNameExcluded(String spaceName) { - List spaceIds; - if (StringUtils.isNotBlank(spaceName)) { - List spaceDOS = logSpaceDao.queryByName(spaceName); - spaceIds = spaceDOS.stream() - .map(MilogSpaceDO::getId) - .toList(); - } else { - spaceIds = logSpaceDao.getAll().stream().map(MilogSpaceDO::getId).collect(Collectors.toList()); - } - return spaceIds; - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/resource/ResourceExtensionService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/resource/ResourceExtensionService.java deleted file mode 100644 index 4ff8141b7..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/resource/ResourceExtensionService.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.extension.resource; - -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.xiaomi.mone.log.api.model.bo.MiLogResource; -import com.xiaomi.mone.log.api.model.vo.ResourceUserSimple; -import com.xiaomi.mone.log.manager.model.pojo.*; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/4/11 10:00 - */ -public interface ResourceExtensionService { - - String DEFAULT_RESOURCE_EXTENSION_SERVICE_KEY = "defaultResourceExtensionService"; - - List userShowAuthority(List configList); - - void filterEsQueryWrapper(QueryWrapper queryWrapper); - - List generateResourceLabels(String id); - - void addResourcePreProcessing(List resourceLabels, MiLogResource miLogResource); - - void addEsResourcePreProcessing(MilogEsClusterDO esClusterDO); - - void addResourceMiddleProcessing(MiLogResource miLogResource); - - void addResourcePostProcessing(MilogMiddlewareConfig milogMiddlewareConfig); - - boolean userResourceListPre(Integer logTypeCode); - - List currentUserConfigFilter(List middlewareConfigs); - - boolean resourceNotRequiredInit(Integer logTypeCode, List middlewareMqConfigs, List middlewareEsConfigs, List esIndexDOList); - - boolean resourceShowStatusFlag(ResourceUserSimple configResource); - - Integer getResourceCode(); - - void deleteMqResourceProcessing(MilogLogTailDo mt, MilogLogStoreDO logStoreDO); - - List getMqResourceCodeList(); - - String queryHostName(String ip); - - List getSpaceIdsByNameExcluded(String spaceName); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/resource/ResourceExtensionServiceFactory.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/resource/ResourceExtensionServiceFactory.java deleted file mode 100644 index a0e402867..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/resource/ResourceExtensionServiceFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.extension.resource; - -import com.xiaomi.mone.log.common.Config; -import com.xiaomi.youpin.docean.Ioc; -import lombok.extern.slf4j.Slf4j; - -import static com.xiaomi.mone.log.manager.service.extension.resource.ResourceExtensionService.DEFAULT_RESOURCE_EXTENSION_SERVICE_KEY; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/4/11 10:02 - */ -@Slf4j -public class ResourceExtensionServiceFactory { - - private static String factualServiceName; - - public static ResourceExtensionService getResourceExtensionService() { - factualServiceName = Config.ins().get("resource.extension.service", DEFAULT_RESOURCE_EXTENSION_SERVICE_KEY); - log.debug("ResourceExtensionServiceFactory factualServiceName:{}", factualServiceName); - return Ioc.ins().getBean(factualServiceName); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/store/DefaultStoreExtensionService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/store/DefaultStoreExtensionService.java deleted file mode 100644 index ec8a07be2..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/store/DefaultStoreExtensionService.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.extension.store; - -import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; -import com.baomidou.mybatisplus.core.toolkit.Wrappers; -import com.xiaomi.mone.log.api.enums.LogStorageTypeEnum; -import com.xiaomi.mone.log.api.enums.OperateEnum; -import com.xiaomi.mone.log.api.model.vo.ResourceUserSimple; -import com.xiaomi.mone.log.manager.common.ManagerConstant; -import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; -import com.xiaomi.mone.log.manager.domain.EsIndexTemplate; -import com.xiaomi.mone.log.manager.mapper.MilogEsClusterMapper; -import com.xiaomi.mone.log.manager.model.dto.EsInfoDTO; -import com.xiaomi.mone.log.manager.model.dto.LogStorageData; -import com.xiaomi.mone.log.manager.model.pojo.MilogEsClusterDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogMiddlewareConfig; -import com.xiaomi.mone.log.manager.model.vo.LogStoreParam; -import com.xiaomi.mone.log.manager.service.impl.MilogMiddlewareConfigServiceImpl; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; - -import static com.xiaomi.mone.log.common.Constant.YES; -import static com.xiaomi.mone.log.manager.service.extension.store.StoreExtensionService.DEFAULT_STORE_EXTENSION_SERVICE_KEY; - -/** - * @author wtt - * @version 1.0 - * @description Store resource binding - * @date 2023/4/10 16:19 - */ -@Service(name = DEFAULT_STORE_EXTENSION_SERVICE_KEY) -@Slf4j -public class DefaultStoreExtensionService implements StoreExtensionService { - - @Resource - private MilogMiddlewareConfigServiceImpl resourceConfigService; - - @Resource - private MilogMiddlewareConfigServiceImpl milogMiddlewareConfigService; - - @Resource - private MilogEsClusterMapper milogEsClusterMapper; - - @Resource - private EsIndexTemplate esIndexTemplate; - - @Resource - private MilogLogstoreDao logStoreDao; - - @Resource - private DorisLogStorageService dorisLogStorageService; - - @Override - public boolean storeInfoCheck(LogStoreParam param) { - return false; - } - - @Override - public void storeResourceBinding(MilogLogStoreDO storeDO, LogStoreParam storeParam, OperateEnum operateEnum) { - if (StringUtils.isNotEmpty(storeDO.getEsIndex()) && null != storeDO.getMqResourceId() && null != storeDO.getEsClusterId()) { - //custom resources operate - customResources(storeDO, storeParam); - return; - } - bindMqResource(storeDO, storeParam); - bindStorageResource(storeDO, storeParam); - } - - private void bindStorageResource(MilogLogStoreDO storeDO, LogStoreParam storeParam) { - ResourceUserSimple resourceUserConfig = resourceConfigService.userResourceList(storeParam.getMachineRoom(), storeParam.getLogType()); - if (resourceUserConfig.getInitializedFlag()) { - // select the ES cluster - storeParam.setEsResourceId(storeParam.getEsResourceId() != null ? storeParam.getEsResourceId() : getDefaultEsClusterId()); - - MilogEsClusterDO esClusterDO = milogEsClusterMapper.selectById(storeParam.getEsResourceId()); - LogStorageTypeEnum storageTypeEnum = LogStorageTypeEnum.queryByName(esClusterDO.getLogStorageType()); - if (storageTypeEnum == LogStorageTypeEnum.DORIS) { - return; - } - - EsInfoDTO esInfo = esIndexTemplate.getEsInfo(storeParam.getEsResourceId(), storeParam.getLogType(), null); - storeParam.setEsIndex(StringUtils.defaultIfBlank(storeParam.getEsIndex(), esInfo.getIndex())); - storeDO.setEsClusterId(esInfo.getClusterId()); - storeDO.setEsIndex(storeParam.getEsIndex()); - } - } - - private Long getDefaultEsClusterId() { - LambdaQueryWrapper lambdaQueryWrapper = Wrappers.lambdaQuery(); - lambdaQueryWrapper.eq(MilogEsClusterDO::getIsDefault, YES); - - return milogEsClusterMapper.selectList(lambdaQueryWrapper) - .stream() - .map(MilogEsClusterDO::getId) - .reduce((first, second) -> second) - .orElse(null); - } - - private void bindMqResource(MilogLogStoreDO storeDO, LogStoreParam storeParam) { - ResourceUserSimple resourceUserConfig = resourceConfigService.userResourceList(storeParam.getMachineRoom(), storeParam.getLogType()); - if (resourceUserConfig.getInitializedFlag() && null == storeParam.getMqResourceId()) { - MilogMiddlewareConfig milogMiddlewareConfig = milogMiddlewareConfigService.queryMiddlewareConfigDefault(storeParam.getMachineRoom()); - storeDO.setMqResourceId(milogMiddlewareConfig.getId()); - storeParam.setMqResourceId(milogMiddlewareConfig.getId()); - } - } - - private void customResources(MilogLogStoreDO ml, LogStoreParam command) { - if (null != command.getMqResourceId()) { - ml.setMqResourceId(command.getMqResourceId()); - } - if (null != command.getEsResourceId()) { - ml.setEsClusterId(command.getEsResourceId()); - } - if (StringUtils.isNotBlank(command.getEsIndex())) { - ml.setEsIndex(command.getEsIndex()); - } - } - - @Override - public void postProcessing(MilogLogStoreDO storeDO, LogStoreParam cmd, OperateEnum operateEnum) { - if (isDorisStorage(storeDO)) { - processDorisStorage(storeDO, cmd, operateEnum); - } - } - - private boolean isDorisStorage(MilogLogStoreDO storeDO) { - MilogEsClusterDO esClusterDO = milogEsClusterMapper.selectById(storeDO.getEsClusterId()); - LogStorageTypeEnum storageTypeEnum = LogStorageTypeEnum.queryByName(esClusterDO.getLogStorageType()); - return storageTypeEnum == LogStorageTypeEnum.DORIS; - } - - private void processDorisStorage(MilogLogStoreDO ml, LogStoreParam cmd, OperateEnum operateEnum) { - LogStorageData storageData = LogStorageData.builder() - .storeId(ml.getId()).build(); - - switch (operateEnum) { - case DELETE_OPERATE: - deleteDorisTable(ml, storageData); - break; - case ADD_OPERATE: - addDorisTable(cmd, storageData); - break; - case UPDATE_OPERATE: - updateDorisTable(ml, cmd, storageData); - break; - default: - // other operations can be processed according to actual needs - } - - updateEsIndexIfNeeded(ml, storageData, operateEnum); - } - - - private void deleteDorisTable(MilogLogStoreDO ml, LogStorageData storageData) { - storageData.setClusterId(ml.getEsClusterId()); - dorisLogStorageService.deleteTable(storageData); - } - - - private void addDorisTable(LogStoreParam cmd, LogStorageData storageData) { - storageData.setLogType(cmd.getLogType()); - storageData.setClusterId(cmd.getEsResourceId()); - storageData.setLogStoreName(cmd.getLogstoreName()); - storageData.setKeys(cmd.getKeyList()); - storageData.setColumnTypes(cmd.getColumnTypeList()); - dorisLogStorageService.createTable(storageData); - } - - private void updateDorisTable(MilogLogStoreDO ml, LogStoreParam cmd, LogStorageData storageData) { - storageData.setUpdateKeys(cmd.getKeyList()); - storageData.setUpdateColumnTypes(cmd.getColumnTypeList()); - storageData.setKeys(ml.getKeyList()); - storageData.setColumnTypes(ml.getColumnTypeList()); - storageData.setLogStoreName(ml.getLogstoreName()); - storageData.setUpdateStoreName(cmd.getLogstoreName()); - dorisLogStorageService.updateTable(storageData); - } - - private void updateEsIndexIfNeeded(MilogLogStoreDO ml, LogStorageData storageData, OperateEnum operateEnum) { - if (OperateEnum.DELETE_OPERATE == operateEnum) { - return; - } - String tableName = dorisLogStorageService.buildTableName(storageData.getClusterId(), storageData.getStoreId()); - if (!StringUtils.equals(tableName, ml.getEsIndex())) { - ml.setEsClusterId(storageData.getClusterId()); - ml.setEsIndex(tableName); - logStoreDao.updateMilogLogStore(ml); - } - } - - @Override - public boolean sendConfigSwitch(LogStoreParam param) { - return true; - } - - @Override - public void deleteStorePostProcessing(MilogLogStoreDO logStoreD) { - - } - - @Override - public String getMangerEsLabel() { - return ManagerConstant.ES_LABEL; - } - - @Override - public boolean updateLogStore(MilogLogStoreDO ml) { - return logStoreDao.updateMilogLogStore(ml); - } - - @Override - public boolean isNeedSendMsgType(Integer logType) { - return true; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/store/DorisLogStorageService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/store/DorisLogStorageService.java deleted file mode 100644 index 3d85a4856..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/store/DorisLogStorageService.java +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.extension.store; - -import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; -import com.baomidou.mybatisplus.core.toolkit.Wrappers; -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.common.Constant; -import com.xiaomi.mone.log.manager.common.exception.MilogManageException; -import com.xiaomi.mone.log.manager.mapper.MilogEsClusterMapper; -import com.xiaomi.mone.log.manager.mapper.MilogEsIndexMapper; -import com.xiaomi.mone.log.manager.model.dto.LogStorageData; -import com.xiaomi.mone.log.manager.model.pojo.MilogEsClusterDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogEsIndexDO; -import com.xiaomi.youpin.docean.Ioc; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; -import javax.sql.DataSource; -import java.sql.*; -import java.util.*; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.GSON; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/11/10 14:29 - */ -@Service -@Slf4j -public class DorisLogStorageService implements LogStorageService { - - private static final String ALTER_TABLE_PREFIX = "ALTER TABLE %s "; - private static final String DROP_COLUMN_FORMAT = "DROP COLUMN %s"; - private static final String ADD_COLUMN_FORMAT = "ADD COLUMN %s %s"; - private static final String CHANGE_COLUMN_FORMAT = "MODIFY COLUMN %s %s"; - private static final String DELETE_TABLE_FORMAT = "DROP TABLE IF EXISTS %s"; - - - @Resource - private MilogEsClusterMapper milogEsClusterMapper; - - @Resource - private MilogEsIndexMapper esIndexMapper; - - @Override - public boolean createTable(LogStorageData storageData) { - MilogEsClusterDO esClusterDO = milogEsClusterMapper.selectById(storageData.getClusterId()); - if (null == esClusterDO) { - throw new MilogManageException("doris data config not exist"); - } - DataSource dataSource = Ioc.ins().getBean(Constant.LOG_STORAGE_SERV_BEAN_PRE + storageData.getClusterId()); - - String tableName = buildTableName(storageData.getClusterId(), storageData.getStoreId()); - String createTableGrammar = buildCreateTableGrammar(tableName, storageData.getKeys(), storageData.getColumnTypes()); - - log.info("createTable,tableName:{},sql:{}", tableName, createTableGrammar); - try (Statement statement = dataSource.getConnection().createStatement()) { - statement.execute(createTableGrammar); - } // Automatically closes statement - catch (SQLException e) { - log.error("createTable error,data:{},tableName:{},sql:{}", GSON.toJson(storageData), tableName, createTableGrammar, e); - throw new MilogManageException("createTable error:" + e.getMessage()); - } - addLogStorageTable(storageData, tableName); - return true; - } - - private void addLogStorageTable(LogStorageData storageData, String tableName) { - MilogEsIndexDO logStorageData = new MilogEsIndexDO(); - logStorageData.setClusterId(storageData.getClusterId()); - logStorageData.setLogType(storageData.getLogType()); - logStorageData.setIndexName(tableName); - esIndexMapper.insert(logStorageData); - } - - private String buildCreateTableGrammar(String tableName, String keys, String columnTypes) { - String CREATE_TABLE_TEMPLATE = "CREATE TABLE %s (%s) DISTRIBUTED BY HASH(`%s`) BUCKETS 1 PROPERTIES\n" + - "(\n" + - "\"replication_num\" = \"1\"\n" + - ");"; - - Map fieldMap = buildFieldMap(keys, columnTypes); - StringJoiner columnDefinitions = new StringJoiner(", "); - fieldMap.forEach((key, value) -> - columnDefinitions.add(String.format("`%s` %s", key, value)) - ); - - return String.format( - CREATE_TABLE_TEMPLATE, - tableName, - columnDefinitions, - "timestamp" - ); - } - - @Override - public boolean updateTable(LogStorageData storageData) { - String tableName = buildTableName(storageData.getClusterId(), storageData.getStoreId()); - if (noChanges(storageData)) { - return false; - } - DataSource dataSource = Ioc.ins().getBean(Constant.LOG_STORAGE_SERV_BEAN_PRE + storageData.getClusterId()); - try { - Connection connection = dataSource.getConnection(); - deleteColumns(connection, tableName, storageData.getKeys(), storageData.getUpdateKeys()); - addColumns(connection, tableName, storageData.getKeys(), storageData.getUpdateKeys(), storageData.getUpdateColumnTypes()); - changeColumns(connection, tableName, storageData.getKeys(), storageData.getColumnTypes(), storageData.getUpdateKeys(), storageData.getUpdateColumnTypes()); - } catch (Exception e) { - log.error("updateTable error,data:{}", GSON.toJson(storageData), e); - throw new MilogManageException("updateTable error:" + e.getMessage()); - } - updateLogStorageTable(storageData, tableName); - return true; - } - - private void updateLogStorageTable(LogStorageData storageData, String tableName) { - LambdaQueryWrapper lambdaQueryWrapper = Wrappers.lambdaQuery() - .eq(MilogEsIndexDO::getClusterId, storageData.getClusterId()) - .eq(MilogEsIndexDO::getIndexName, tableName); - List logEsIndexDOS = esIndexMapper.selectList(lambdaQueryWrapper); - for (MilogEsIndexDO logEsIndexDO : logEsIndexDOS) { - if (!Objects.equals(storageData.getLogType(), logEsIndexDO.getLogType())) { - logEsIndexDO.setLogType(storageData.getLogType()); - esIndexMapper.updateById(logEsIndexDO); - } - } - } - - private boolean noChanges(LogStorageData storageData) { - if (StringUtils.isEmpty(storageData.getUpdateKeys()) || StringUtils.isEmpty(storageData.getUpdateColumnTypes())) { - return true; - } - return StringUtils.equals(storageData.getKeys(), storageData.getUpdateKeys()) && - StringUtils.equals(storageData.getColumnTypes(), storageData.getUpdateColumnTypes()); - } - - private void deleteColumns(Connection connection, String tableName, String currentKeys, String updateKeys) throws - Exception { - List cleanKeyList = getCleanKeyList(currentKeys); - List updateCleanKeyList = getCleanKeyList(updateKeys); - List deleteKeys = cleanKeyList.stream().filter(key -> !updateCleanKeyList.contains(key)).collect(Collectors.toList()); - - if (CollectionUtils.isNotEmpty(deleteKeys)) { - String delColumns = deleteKeys.stream().map(key -> String.format(DROP_COLUMN_FORMAT, key)) - .collect(Collectors.joining(",")); - executeAlterTable(connection, tableName, delColumns); - } - } - - private void addColumns(Connection connection, String tableName, String keys, String updateKeys, String updateColumnTypes) throws Exception { - List cleanKeyList = getCleanKeyList(updateKeys); - List addKeys = cleanKeyList.stream().filter(key -> !getCleanKeyList(keys).contains(key)) - .collect(Collectors.toList()); - - if (CollectionUtils.isNotEmpty(addKeys)) { - Map fieldMap = buildFieldMap(updateKeys, updateColumnTypes); - String addColumns = addKeys.stream() - .map(key -> String.format(ADD_COLUMN_FORMAT, key, fieldMap.get(key))) - .collect(Collectors.joining(",")); - executeAlterTable(connection, tableName, addColumns); - } - } - - private void changeColumns(Connection connection, String tableName, String currentKeys, String currentColumnTypes, String updateKeys, String updateColumnTypes) throws Exception { - Map originFieldMap = buildFieldMap(currentKeys, currentColumnTypes); - Map currentFieldMap = buildFieldMap(updateKeys, updateColumnTypes); - - List updateFields = currentFieldMap.entrySet().stream() - .filter(entry -> !StringUtils.equals(entry.getValue(), originFieldMap.get(entry.getKey()))) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - - if (CollectionUtils.isNotEmpty(updateFields)) { - String changeColumns = updateFields.stream() - .map(key -> String.format(CHANGE_COLUMN_FORMAT, key, currentFieldMap.get(key))) - .collect(Collectors.joining(",")); - executeAlterTable(connection, tableName, changeColumns); - } - } - - private void executeAlterTable(Connection connection, String tableName, String columnStatements) throws - Exception { - String sql = String.format(ALTER_TABLE_PREFIX + "%s;", tableName, columnStatements); - try (Statement statement = connection.createStatement()) { - statement.execute(sql); - } finally { - log.error("executeAlterTable error,sql:{}", sql); - } - } - - private List getCleanKeyList(String keys) { - return Arrays.stream(keys.split(",")) - .map(s -> s.split(":")[0]) - .collect(Collectors.toList()); - } - - - @Override - public boolean deleteTable(LogStorageData storageData) { - String tableName = buildTableName(storageData.getClusterId(), storageData.getStoreId()); - DataSource dataSource = Ioc.ins().getBean(Constant.LOG_STORAGE_SERV_BEAN_PRE + storageData.getClusterId()); - - try (Statement statement = dataSource.getConnection().createStatement()) { - - String createTableGrammar = String.format(DELETE_TABLE_FORMAT, tableName); - statement.execute(createTableGrammar); - - } // Automatically closes statement - catch (SQLException e) { - throw new MilogManageException("deleteTable error:" + e.getMessage()); - } - deleteLogStorageTable(storageData, tableName); - return true; - } - - @Override - public List getColumnList(Long clusterId, String tableName) { - List columnList = Lists.newArrayList(); - DataSource dataSource = Ioc.ins().getBean(Constant.LOG_STORAGE_SERV_BEAN_PRE + clusterId); - try { - Connection connection = dataSource.getConnection(); - DatabaseMetaData metaData = connection.getMetaData(); - - try (ResultSet resultSet = metaData.getColumns(null, null, tableName, null)) { - while (resultSet.next()) { - String columnName = resultSet.getString("COLUMN_NAME"); - columnList.add(columnName); - } - } - } catch (Exception e) { - log.error("getColumnList error,clusterId:{},tableName:{}", clusterId, tableName, e); - } - log.info("getColumnList,,clusterId:{},tableName:{},columnList:{}", clusterId, tableName, GSON.toJson(columnList)); - return columnList; - } - - private void deleteLogStorageTable(LogStorageData storageData, String tableName) { - LambdaQueryWrapper lambdaQueryWrapper = Wrappers.lambdaQuery() - .eq(MilogEsIndexDO::getLogType, storageData.getLogType()) - .eq(MilogEsIndexDO::getClusterId, storageData.getClusterId()) - .eq(MilogEsIndexDO::getIndexName, tableName); - esIndexMapper.delete(lambdaQueryWrapper); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/store/EsLogStorageService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/store/EsLogStorageService.java deleted file mode 100644 index 832da1b60..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/store/EsLogStorageService.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.extension.store; - -import com.xiaomi.mone.log.manager.model.dto.LogStorageData; - -import java.util.ArrayList; -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/11/10 14:28 - */ -public class EsLogStorageService implements LogStorageService { - - @Override - public boolean createTable(LogStorageData logStorageData) { - return false; - } - - @Override - public boolean updateTable(LogStorageData logStorageData) { - return false; - } - - @Override - public boolean deleteTable(LogStorageData logStorageData) { - return false; - } - - @Override - public List getColumnList(Long clusterId, String tableName) { - return new ArrayList<>(); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/store/LogStorageService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/store/LogStorageService.java deleted file mode 100644 index 1944cbb34..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/store/LogStorageService.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.extension.store; - -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.manager.model.dto.LogStorageData; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/11/10 14:26 - */ -public interface LogStorageService { - - String DORIS_TABLE_PREFIX = "hera_log_doris_table"; - - boolean createTable(LogStorageData logStorageData); - - boolean updateTable(LogStorageData logStorageData); - - boolean deleteTable(LogStorageData logStorageData); - - /** - * get the schema collection of the table - * - * @param tableName - * @return - */ - List getColumnList(Long clusterId, String tableName); - - - default String buildTableName(Long clusterId, Long storeId) { - return String.format("%s_%s_%s", DORIS_TABLE_PREFIX, clusterId, storeId); - } - - String BIGINT = "BIGINT"; - String VARCHAR = "VARCHAR(1024)"; - String VARCHAR_LARGE = "STRING"; - List Large_Field_Properties = Lists.newArrayList("message", "logsource"); - - - default Map buildFieldMap(String keys, String columnTypes) { - - List cleanKeyList = Arrays.stream(keys.split(",")) - .map(s -> s.split(":")[0]) - .collect(Collectors.toList()); - - List columnTypeList = Arrays.asList(columnTypes.split(",")); - - return IntStream.range(0, cleanKeyList.size()) - .boxed() - .collect(Collectors.toMap( - i -> cleanKeyList.get(i), - i -> mapFieldType(columnTypeList.get(i).toLowerCase(), cleanKeyList.get(i)) - )); - } - - default String mapFieldType(String fieldType, String field) { - if (Large_Field_Properties.contains(field)) { - return VARCHAR_LARGE; - } - switch (fieldType) { - case "date": - case "long": - return BIGINT; - case "keyword": - case "text": - case "ip": - return VARCHAR; - default: - return fieldType; - } - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/store/StoreExtensionService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/store/StoreExtensionService.java deleted file mode 100644 index afb9f0631..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/store/StoreExtensionService.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.extension.store; - -import com.xiaomi.mone.log.api.enums.OperateEnum; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.vo.LogStoreParam; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/4/10 16:13 - */ -public interface StoreExtensionService { - - String DEFAULT_STORE_EXTENSION_SERVICE_KEY = "defaultStoreExtensionService"; - - boolean storeInfoCheck(LogStoreParam param); - - /** - * resource bind - * - * @param ml - * @param cmd - * @param operateEnum - */ - void storeResourceBinding(MilogLogStoreDO ml, LogStoreParam cmd, OperateEnum operateEnum); - - /** - * Additional post processing - * - * @param ml - * @param cmd - */ - void postProcessing(MilogLogStoreDO ml, LogStoreParam cmd, OperateEnum operateEnum); - - /** - * Send Configuration Switch - * - * @return - */ - boolean sendConfigSwitch(LogStoreParam param); - - void deleteStorePostProcessing(MilogLogStoreDO logStoreD); - - String getMangerEsLabel(); - - boolean updateLogStore(MilogLogStoreDO ml); - - boolean isNeedSendMsgType(Integer logType); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/store/StoreExtensionServiceFactory.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/store/StoreExtensionServiceFactory.java deleted file mode 100644 index e0dcc2506..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/store/StoreExtensionServiceFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.extension.store; - -import com.xiaomi.mone.log.common.Config; -import com.xiaomi.youpin.docean.Ioc; -import lombok.extern.slf4j.Slf4j; - -import static com.xiaomi.mone.log.manager.service.extension.store.StoreExtensionService.DEFAULT_STORE_EXTENSION_SERVICE_KEY; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/4/10 16:41 - */ -@Slf4j -public class StoreExtensionServiceFactory { - - private static String factualServiceName; - - public static StoreExtensionService getStoreExtensionService() { - factualServiceName = Config.ins().get("store.extension.service", DEFAULT_STORE_EXTENSION_SERVICE_KEY); - log.debug("StoreExtensionServiceFactory factualServiceName:{}", factualServiceName); - return Ioc.ins().getBean(factualServiceName); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/tail/DefaultTailExtensionService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/tail/DefaultTailExtensionService.java deleted file mode 100644 index b0ad61a56..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/tail/DefaultTailExtensionService.java +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.extension.tail; - -import com.google.common.collect.Lists; -import org.apache.ozhera.app.api.model.HeraSimpleEnv; -import org.apache.ozhera.app.api.response.AppBaseInfo; -import com.xiaomi.mone.log.api.enums.OperateEnum; -import com.xiaomi.mone.log.api.enums.ProjectTypeEnum; -import com.xiaomi.mone.log.manager.dao.MilogLogTailDao; -import com.xiaomi.mone.log.manager.model.bo.LogTailParam; -import com.xiaomi.mone.log.manager.model.dto.MilogAppEnvDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.mone.log.manager.model.pojo.MilogMiddlewareConfig; -import com.xiaomi.mone.log.manager.service.extension.agent.MilogAgentServiceImpl; -import com.xiaomi.mone.log.manager.service.impl.HeraAppEnvServiceImpl; -import com.xiaomi.mone.log.manager.service.impl.LogTailServiceImpl; -import com.xiaomi.mone.log.manager.service.impl.MilogAppMiddlewareRelServiceImpl; -import com.xiaomi.mone.log.manager.service.nacos.FetchStreamMachineService; -import com.xiaomi.mone.log.manager.service.nacos.impl.StreamConfigNacosPublisher; -import com.xiaomi.mone.log.model.LogtailConfig; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.manager.service.extension.agent.MilogAgentService.DEFAULT_AGENT_EXTENSION_SERVICE_KEY; -import static com.xiaomi.mone.log.manager.service.extension.tail.TailExtensionService.DEFAULT_TAIL_EXTENSION_SERVICE_KEY; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/4/10 17:06 - */ -@Service(name = DEFAULT_TAIL_EXTENSION_SERVICE_KEY) -@Slf4j -public class DefaultTailExtensionService implements TailExtensionService { - - @Resource - private LogTailServiceImpl logTailService; - - @Resource - private MilogAppMiddlewareRelServiceImpl milogAppMiddlewareRelService; - - @Resource - private HeraAppEnvServiceImpl heraAppEnvService; - - @Resource(name = DEFAULT_AGENT_EXTENSION_SERVICE_KEY) - private MilogAgentServiceImpl milogAgentService; - - @Resource - private MilogLogTailDao milogLogtailDao; - - @Override - public boolean tailHandlePreprocessingSwitch(MilogLogStoreDO milogLogStore, LogTailParam param) { - return true; - } - - @Override - public boolean bindMqResourceSwitch(MilogLogStoreDO logStore, Integer appType) { - return Objects.equals(ProjectTypeEnum.MIONE_TYPE.getCode().intValue(), appType); - } - - @Override - public boolean bindPostProcessSwitch(Long storeId) { - return false; - } - - @Override - public void postProcessing() { - - } - - @Override - public void defaultBindingAppTailConfigRel(Long id, Long milogAppId, Long middleWareId, String topicName, Integer batchSendSize) { - milogAppMiddlewareRelService.defaultBindingAppTailConfigRel(id, milogAppId, middleWareId, topicName, batchSendSize); - } - - @Override - public void defaultBindingAppTailConfigRelPostProcess(Long spaceId, Long storeId, Long tailId, Long milogAppId, Long storeMqResourceId) { - - } - - @Override - public void sendMessageOnCreate(LogTailParam param, MilogLogTailDo mt, Long milogAppId, boolean supportedConsume) { - /** - * Send configuration information ---log-agent - */ - if (param.getCollectionReady()) { - CompletableFuture.runAsync(() -> logTailService.sengMessageToAgent(milogAppId, mt)); - } - /** - * Send final configuration information ---log-stream -- View the log template type, if it is OpenTelemetry logs, only send MQ and do not consume - */ - if (supportedConsume) { - logTailService.sengMessageToStream(mt, OperateEnum.ADD_OPERATE.getCode()); - } - } - - @Override - public void updateSendMsg(MilogLogTailDo milogLogtailDo, List oldIps, boolean supportedConsume) { - /** - * Synchronous log-agent - */ - if (milogLogtailDo.getCollectionReady()) { - CompletableFuture.runAsync(() -> milogAgentService.publishIncrementConfig(milogLogtailDo.getId(), milogLogtailDo.getMilogAppId(), milogLogtailDo.getIps())); - } - /** - * Synchronous log-stream If it is OpenTelemetry logs, only send MQ and do not consume it - */ - if (supportedConsume) { -// List middlewareRels = milogAppMiddlewareRelDao.queryByCondition(milogLogtailDo.getMilogAppId(), null, milogLogtailDo.getId()); -// createConsumerGroup(milogLogtailDo.getSpaceId(), milogLogtailDo.getStoreId(), milogLogtailDo.getId(), milogMiddlewareConfigDao.queryById(middlewareRels.get(0).getMiddlewareId()), milogLogtailDo.getMilogAppId(), false); - logTailService.sengMessageToStream(milogLogtailDo, OperateEnum.UPDATE_OPERATE.getCode()); - } - logTailService.compareChangeDelIps(milogLogtailDo.getId(), milogLogtailDo.getLogPath(), milogLogtailDo.getIps(), oldIps); - } - - @Override - public void logTailDoExtraFiled(MilogLogTailDo milogLogtailDo, MilogLogStoreDO logStoreDO, LogTailParam logTailParam) { - milogLogtailDo.setIps(logTailParam.getIps()); - } - - @Override - public void logTailConfigExtraField(LogtailConfig logtailConfig, MilogMiddlewareConfig middlewareConfig) { - - } - - @Override - public void logTailDelPostProcess(MilogLogStoreDO logStoreDO, MilogLogTailDo milogLogtailDo) { - - } - - @Override - public List getEnInfosByAppId(AppBaseInfo appBaseInfo, Long milogAppId, Integer deployWay, String machineRoom) { - List heraSimpleEnvs = null; - try { - heraSimpleEnvs = heraAppEnvService.querySimpleEnvAppBaseInfoId(milogAppId.intValue()); - } catch (Exception e) { - log.error(String.format("query ip error:milogAppId:%s,deployWay:%s", milogAppId, deployWay), e); - } - if (CollectionUtils.isNotEmpty(heraSimpleEnvs)) { - return heraSimpleEnvs.stream().map(envBo -> MilogAppEnvDTO.builder().label(envBo.getName()).value(envBo.getId()).ips(envBo.getIps()).build()).collect(Collectors.toList()); - } - return Lists.newArrayList(); - } - - @Override - public boolean decorateTailDTOValId(Integer logType, Integer appType) { - return true; - } - - @Override - public List getStreamMachineUniqueList(Integer projectTypeCode, String motorRoomEn) { - return Lists.newArrayList(); - } - - @Override - public String deleteCheckProcessPre(Long id) { - return StringUtils.EMPTY; - } - - @Override - public String validLogPath(LogTailParam param) { - if (Objects.equals(ProjectTypeEnum.MIONE_TYPE.getCode(), param.getAppType())) { - // Verify the log file with the same name - List appLogTails = milogLogtailDao.queryByMilogAppAndEnv(param.getMilogAppId(), param.getEnvId()); - for (int i = 0; i < appLogTails.size() && null == param.getId(); i++) { - if (appLogTails.get(i).getLogPath().equals(param.getLogPath())) { - return "The current deployment environment for the file " + param.getLogPath() + " Log collection is configured with the following aliases:" + appLogTails.get(i).getTail(); - } - } - } - return StringUtils.EMPTY; - } - - @Override - public void publishStreamConfigPostProcess(StreamConfigNacosPublisher streamConfigNacosPublisher, Long spaceId, String motorRoomEn) { - - } - - @Override - public List fetchStreamUniqueKeyList(FetchStreamMachineService fetchStreamMachineService, Long spaceId, String motorRoomEn) { - List mioneStreamIpList = fetchStreamMachineService.streamMachineUnique(); - if (CollectionUtils.isEmpty(mioneStreamIpList)) { - mioneStreamIpList = getStreamMachineUniqueList(null, motorRoomEn); - } - return mioneStreamIpList; - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/tail/TailExtensionService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/tail/TailExtensionService.java deleted file mode 100644 index a6d83994d..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/tail/TailExtensionService.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.extension.tail; - -import org.apache.ozhera.app.api.response.AppBaseInfo; -import com.xiaomi.mone.log.manager.model.bo.LogTailParam; -import com.xiaomi.mone.log.manager.model.dto.MilogAppEnvDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.mone.log.manager.model.pojo.MilogMiddlewareConfig; -import com.xiaomi.mone.log.manager.service.nacos.FetchStreamMachineService; -import com.xiaomi.mone.log.manager.service.nacos.impl.StreamConfigNacosPublisher; -import com.xiaomi.mone.log.model.LogtailConfig; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/4/10 17:06 - */ -public interface TailExtensionService { - String DEFAULT_TAIL_EXTENSION_SERVICE_KEY = "defaultTailExtensionService"; - - boolean tailHandlePreprocessingSwitch(MilogLogStoreDO milogLogStore, LogTailParam param); - - boolean bindMqResourceSwitch(MilogLogStoreDO logStore, Integer appType); - - boolean bindPostProcessSwitch(Long storeId); - - void postProcessing(); - - void defaultBindingAppTailConfigRel(Long id, Long milogAppId, Long middleWareId, String topicName, Integer batchSendSize); - - void defaultBindingAppTailConfigRelPostProcess(Long spaceId, Long storeId, Long tailId, Long milogAppId, Long storeMqResourceId); - - void sendMessageOnCreate(LogTailParam param, MilogLogTailDo mt, Long milogAppId, boolean supportedConsume); - - void updateSendMsg(MilogLogTailDo milogLogtailDo, List oldIps, boolean supportedConsume); - - void logTailDoExtraFiled(MilogLogTailDo milogLogtailDo, MilogLogStoreDO logStoreDO, LogTailParam logTailParam); - - void logTailConfigExtraField(LogtailConfig logtailConfig, MilogMiddlewareConfig middlewareConfig); - - void logTailDelPostProcess(MilogLogStoreDO logStoreDO, MilogLogTailDo milogLogtailDo); - - List getEnInfosByAppId(AppBaseInfo appBaseInfo, Long milogAppId, Integer deployWay, String machineRoom); - - boolean decorateTailDTOValId(Integer logType, Integer appType); - - List getStreamMachineUniqueList(Integer projectTypeCode, String motorRoomEn); - - String deleteCheckProcessPre(Long id); - - String validLogPath(LogTailParam param); - - void publishStreamConfigPostProcess(StreamConfigNacosPublisher streamConfigNacosPublisher, Long spaceId, String motorRoomEn); - - List fetchStreamUniqueKeyList(FetchStreamMachineService fetchStreamMachineService, Long spaceId, String motorRoomEn); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/tail/TailExtensionServiceFactory.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/tail/TailExtensionServiceFactory.java deleted file mode 100644 index 6031aebc0..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/tail/TailExtensionServiceFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.extension.tail; - -import com.xiaomi.mone.log.common.Config; -import com.xiaomi.youpin.docean.Ioc; -import lombok.extern.slf4j.Slf4j; - -import static com.xiaomi.mone.log.manager.service.extension.tail.TailExtensionService.DEFAULT_TAIL_EXTENSION_SERVICE_KEY; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/4/10 17:15 - */ -@Slf4j -public class TailExtensionServiceFactory { - - private static String factualServiceName; - - public static TailExtensionService getTailExtensionService() { - factualServiceName = Config.ins().get("tail.extension.service", DEFAULT_TAIL_EXTENSION_SERVICE_KEY); - log.debug("TailExtensionServiceFactory factualServiceName:{}", factualServiceName); - return Ioc.ins().getBean(factualServiceName); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/AgentConfigServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/AgentConfigServiceImpl.java deleted file mode 100644 index acaac2fd8..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/AgentConfigServiceImpl.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import com.google.gson.Gson; -import com.xiaomi.mone.log.api.model.meta.LogCollectMeta; -import com.xiaomi.mone.log.api.service.AgentConfigService; -import com.xiaomi.mone.log.manager.service.extension.agent.MilogAgentService; -import com.xiaomi.mone.log.manager.service.extension.agent.MilogAgentServiceFactory; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/8/4 19:42 - */ -@Slf4j -@Service -@com.xiaomi.youpin.docean.plugin.dubbo.anno.Service(interfaceClass = AgentConfigService.class, group = "$dubbo.env.group") -public class AgentConfigServiceImpl implements AgentConfigService { - - private MilogAgentService milogAgentService; - - public void init() { - milogAgentService = MilogAgentServiceFactory.getAgentExtensionService(); - } - - /** - * 1.Query all applications that have access logs on the physical machine - * 2.Package information - * - * @param ip - * @return - */ - @Override - public LogCollectMeta getLogCollectMetaFromManager(String ip) { - log.info("getLogCollectMetaFromManager begin:{}", ip); - try { - long begin = System.currentTimeMillis(); - if (null == milogAgentService) { - init(); - } - LogCollectMeta logCollectMeta = milogAgentService.getLogCollectMetaFromManager(ip); - log.info("getLogCollectMetaFromManager end:{} {} {}", ip, new Gson().toJson(logCollectMeta), (System.currentTimeMillis() - begin)); - return logCollectMeta; - } catch (Exception e) { - log.error("getLogCollectMetaFromManager error,ip:{}", ip, e); - } - return new LogCollectMeta(); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/EsDataServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/EsDataServiceImpl.java deleted file mode 100644 index a3c211a56..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/EsDataServiceImpl.java +++ /dev/null @@ -1,684 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import cn.hutool.core.date.DateUtil; -import cn.hutool.core.lang.Pair; -import com.alibaba.fastjson.JSON; -import com.google.common.collect.Lists; -import org.apache.ozhera.app.api.model.HeraAppEnvData; -import com.xiaomi.mone.es.EsClient; -import com.xiaomi.mone.log.api.enums.LogStorageTypeEnum; -import com.xiaomi.mone.log.api.model.dto.TraceLogDTO; -import com.xiaomi.mone.log.api.model.vo.TraceLogQuery; -import com.xiaomi.mone.log.api.service.LogDataService; -import com.xiaomi.mone.log.common.Constant; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.exception.CommonError; -import com.xiaomi.mone.log.manager.common.context.MoneUserContext; -import com.xiaomi.mone.log.manager.common.exception.MilogManageException; -import com.xiaomi.mone.log.manager.common.utils.ExportUtils; -import com.xiaomi.mone.log.manager.dao.MilogLogTailDao; -import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; -import com.xiaomi.mone.log.manager.dao.MilogSpaceDao; -import com.xiaomi.mone.log.manager.domain.EsCluster; -import com.xiaomi.mone.log.manager.domain.SearchLog; -import com.xiaomi.mone.log.manager.domain.TraceLog; -import com.xiaomi.mone.log.manager.mapper.MilogEsClusterMapper; -import com.xiaomi.mone.log.manager.model.dto.EsStatisticResult; -import com.xiaomi.mone.log.manager.model.dto.LogDTO; -import com.xiaomi.mone.log.manager.model.dto.LogDataDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogEsClusterDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.mone.log.manager.model.pojo.MilogSpaceDO; -import com.xiaomi.mone.log.manager.model.vo.LogContextQuery; -import com.xiaomi.mone.log.manager.model.vo.LogQuery; -import com.xiaomi.mone.log.manager.model.vo.RegionTraceLogQuery; -import com.xiaomi.mone.log.manager.model.vo.TraceAppLogUrlQuery; -import com.xiaomi.mone.log.manager.service.EsDataBaseService; -import com.xiaomi.mone.log.manager.service.EsDataService; -import com.xiaomi.mone.log.manager.service.HeraAppEnvService; -import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionService; -import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionServiceFactory; -import com.xiaomi.mone.log.parse.LogParser; -import com.xiaomi.youpin.docean.Ioc; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.common.StringUtils; -import com.xiaomi.youpin.docean.plugin.config.anno.Value; -import com.xiaomi.youpin.docean.plugin.dubbo.anno.Reference; -import com.xiaomi.youpin.docean.plugin.es.EsService; -import lombok.extern.slf4j.Slf4j; -import org.apache.poi.hssf.usermodel.HSSFWorkbook; -import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; -import org.springframework.util.CollectionUtils; -import org.springframework.util.StopWatch; -import run.mone.excel.ExportExcel; - -import javax.annotation.Resource; -import javax.sql.DataSource; -import java.io.IOException; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.*; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.DEFAULT_OPERATOR; -import static com.xiaomi.mone.log.common.Constant.GSON; -import static com.xiaomi.mone.log.manager.common.utils.ManagerUtil.*; -import static com.xiaomi.mone.log.parse.LogParser.*; -import static org.elasticsearch.search.sort.SortOrder.ASC; -import static org.elasticsearch.search.sort.SortOrder.DESC; - -@Slf4j -@Service -@com.xiaomi.youpin.docean.plugin.dubbo.anno.Service(interfaceClass = LogDataService.class) -public class EsDataServiceImpl implements EsDataService, LogDataService, EsDataBaseService { - - @Resource - private MilogLogstoreDao logstoreDao; - - @Resource - private MilogLogTailDao tailDao; - - @Resource - private MilogSpaceDao spaceDao; - - @Resource - private EsCluster esCluster; - - @Resource - private TraceLog traceLog; - - @Resource - private SearchLog searchLog; - - @Resource - private MilogEsClusterMapper milogEsClusterMapper; - - @Value(value = "$hera.url") - private String heraUrl; - - @Reference(interfaceClass = LogDataService.class, group = "$dubbo.youpin.group", check = false, timeout = 5000) - private LogDataService logDataService; - - private CommonExtensionService commonExtensionService; - - public void init() { - commonExtensionService = CommonExtensionServiceFactory.getCommonExtensionService(); - } - - private Set noHighLightSet = new HashSet<>(); - - private Set hidenFiledSet = new HashSet<>(); - - @Resource - private HeraAppEnvService heraAppEnvService; - public static List>> requiredFields = Lists.newArrayList( - Pair.of(ES_KEY_MAP_MESSAGE, Pair.of("text", 1)), - Pair.of(ES_KEY_MAP_LOG_SOURCE, Pair.of("text", 1)), - Pair.of(ES_KEY_MAP_TAIL_ID, Pair.of("integer", 3)) - ); - - { - noHighLightSet.add("logstore"); -// noHighLightSet.add("logsource"); - noHighLightSet.add("tail"); -// noHighLightSet.add("timestamp"); -// noHighLightSet.add("linenumber"); - - hidenFiledSet.add("mqtag"); - hidenFiledSet.add("mqtopic"); - hidenFiledSet.add("logstore"); - hidenFiledSet.add("linenumber"); - hidenFiledSet.add("filename"); - } - - /** - * Read the data of the ES index - * - * @param logQuery - * @return - */ - @Override - public Result logQuery(LogQuery logQuery) { - String operator = MoneUserContext.getCurrentUser() != null ? MoneUserContext.getCurrentUser().getUser() : DEFAULT_OPERATOR; - - String logInfo = String.format("queryText:%s, user:%s, logQuery:%s", logQuery.getFullTextSearch(), operator, logQuery); - log.info("query simple param:{}", logInfo); - - StopWatch stopWatch = new StopWatch("HERA-LOG-QUERY"); - - try { - MilogLogStoreDO milogLogstoreDO = logstoreDao.queryById(logQuery.getStoreId()); - if (milogLogstoreDO == null) { - log.warn("[EsDataService.logQuery] not find logStore:[{}]", logQuery.getLogstore()); - return Result.failParam("not found[" + logQuery.getLogstore() + "]The corresponding data"); - } - LogStorageTypeEnum storageTypeEnum = queryLogStorageTypeEnum(milogLogstoreDO.getEsClusterId()); - LogDTO dto = new LogDTO(); - List keyList = getKeyList(milogLogstoreDO.getKeyList(), milogLogstoreDO.getColumnTypeList()); - - if (LogStorageTypeEnum.DORIS == storageTypeEnum) { - return dorisDataQuery(logQuery, milogLogstoreDO, dto); - } else { - return elasticDataQuery(milogLogstoreDO, logQuery, dto, keyList, stopWatch); - } - } catch (Throwable e) { - log.error("Log query error, log search error,logQuery:[{}],user:[{}]", logQuery, MoneUserContext.getCurrentUser(), e); - return Result.failParam(e.getMessage()); - } - } - - private LogStorageTypeEnum queryLogStorageTypeEnum(Long esClusterId) { - MilogEsClusterDO esClusterDO = milogEsClusterMapper.selectById(esClusterId); - LogStorageTypeEnum storageTypeEnum = LogStorageTypeEnum.queryByName(esClusterDO.getLogStorageType()); - return storageTypeEnum; - } - - private Result dorisDataQuery(LogQuery logQuery, MilogLogStoreDO milogLogstoreDO, LogDTO dto) { - try { - DataSource dataSource = Ioc.ins().getBean(Constant.LOG_STORAGE_SERV_BEAN_PRE + milogLogstoreDO.getEsClusterId()); - List> tableColumnDTOS = queryResult(logQuery, milogLogstoreDO, dataSource); - dorisDataToLog(tableColumnDTOS, dto); - return Result.success(dto); - } catch (Exception e) { - log.error("Doris data query error", e); - return Result.failParam(e.getMessage()); - } - } - - private Result elasticDataQuery(MilogLogStoreDO milogLogstoreDO, LogQuery logQuery, - LogDTO dto, List keyList, StopWatch stopWatch) throws IOException { - EsService esService = esCluster.getEsService(milogLogstoreDO.getEsClusterId()); - - String esIndexName = commonExtensionService.getSearchIndex(logQuery.getStoreId(), milogLogstoreDO.getEsIndex()); - if (esService == null || StringUtils.isEmpty(esIndexName)) { - log.warn("[EsDataService.logQuery] logStore:[{}] configuration exceptions", logQuery.getLogstore()); - return Result.failParam("logStore configuration exceptions"); - } - // Build query parameters - BoolQueryBuilder boolQueryBuilder = searchLog.getQueryBuilder(logQuery, getKeyColonPrefix(milogLogstoreDO.getKeyList())); - SearchSourceBuilder builder = assembleSearchSourceBuilder(logQuery, keyList, boolQueryBuilder); - - SearchRequest searchRequest = new SearchRequest(new String[]{esIndexName}, builder); - // query - stopWatch.start("search-query"); - SearchResponse searchResponse = esService.search(searchRequest); - stopWatch.stop(); - - dto.setSourceBuilder(builder); - if (stopWatch.getLastTaskTimeMillis() > 7 * 1000) { - log.warn("##LONG-COST-QUERY##{} cost:{} ms, msg:{}", stopWatch.getLastTaskName(), stopWatch.getLastTaskTimeMillis()); - } - //Result transformation - stopWatch.start("data-assemble"); - transformSearchResponse(searchResponse, dto, keyList); - stopWatch.stop(); - - if (stopWatch.getTotalTimeMillis() > 15 * 1000) { - log.warn("##LONG-COST-QUERY##{} cost:{} ms, msg:{}", "gt15s", stopWatch.getTotalTimeMillis()); - } - return Result.success(dto); - } - - private void dorisDataToLog(List> tableColumnDTOS, LogDTO logDTO) { - List logDataList = Lists.newArrayList(); - for (Map columnMap : tableColumnDTOS) { - - LogDataDTO logData = new LogDataDTO(); - - logData.setValue(LogParser.esKeyMap_timestamp, columnMap.get(LogParser.esKeyMap_timestamp)); - for (String key : columnMap.keySet()) { - if (!hidenFiledSet.contains(key)) { - logData.setValue(key, columnMap.get(key)); - } - } - logData.setIp(columnMap.get(LogParser.esKeyMap_logip) == null ? "" : String.valueOf(columnMap.get(LogParser.esKeyMap_logip))); - logData.setFileName(columnMap.get(LogParser.esKyeMap_fileName) == null ? "" : String.valueOf(columnMap.get(LogParser.esKyeMap_fileName))); - logData.setLineNumber(columnMap.get(LogParser.esKeyMap_lineNumber) == null ? "" : String.valueOf(columnMap.get(LogParser.esKeyMap_lineNumber))); - logData.setTimestamp(columnMap.get(LogParser.esKeyMap_timestamp) == null ? "" : String.valueOf(columnMap.get(LogParser.esKeyMap_timestamp))); - logData.setLogOfString(JSON.toJSONString(logData.getLogOfKV())); - // Package highlighted - logDataList.add(logData); - } - logDTO.setThisSortValue(null); - logDTO.setLogDataDTOList(logDataList); - } - - private List> queryResult(LogQuery logQuery, MilogLogStoreDO milogLogstoreDO, DataSource dataSource) throws SQLException { - List> columns = new ArrayList<>(); - - String querySql = buildQuerySql(logQuery, milogLogstoreDO); - - try (Statement statement = dataSource.getConnection().createStatement(); - ResultSet resultSet = statement.executeQuery(querySql)) { - - ResultSetMetaData metaData = resultSet.getMetaData(); - int columnCount = metaData.getColumnCount(); - - while (resultSet.next()) { - Map dataMap = new HashMap<>(); - for (int i = 1; i <= columnCount; i++) { - dataMap.put(metaData.getColumnName(i), resultSet.getObject(i)); - } - columns.add(dataMap); - } - } - return columns; - } - - private static String buildQuerySql(LogQuery logQuery, MilogLogStoreDO milogLogstoreDO) { - String sqlPrefix = getConditional(logQuery); - String sortSql = StringUtils.isNotEmpty(logQuery.getSortKey()) ? String.format("ORDER BY %s", logQuery.getSortKey()) : ""; - if (StringUtils.isNotEmpty(sortSql) && !logQuery.getAsc()) { - sortSql += " DESC"; - } - String limitSql = String.format("LIMIT %s, %s", (logQuery.getPage() - 1) * logQuery.getPageSize(), logQuery.getPageSize()); - if (StringUtils.isNotEmpty(logQuery.getFullTextSearch())) { - return String.format("SELECT * FROM %s WHERE %s AND %s %s %s", - milogLogstoreDO.getEsIndex(), sqlPrefix, logQuery.getFullTextSearch(), sortSql, limitSql); - } else { - return String.format("SELECT * FROM %s WHERE %s %s %s", - milogLogstoreDO.getEsIndex(), sqlPrefix, sortSql, limitSql); - } - } - - private static String buildQuerySqlConditional(LogQuery logQuery, MilogLogStoreDO milogLogstoreDO) { - String sqlPrefix = getConditional(logQuery); - if (StringUtils.isNotEmpty(logQuery.getFullTextSearch())) { - return String.format("SELECT * FROM %s WHERE %s AND %s", - milogLogstoreDO.getEsIndex(), sqlPrefix, logQuery.getFullTextSearch()); - } else { - return String.format("SELECT * FROM %s WHERE %s ", - milogLogstoreDO.getEsIndex(), sqlPrefix); - } - } - - private static String getConditional(LogQuery logQuery) { - String sqlPrefix = String.format("timestamp >= %s AND timestamp <= %s", logQuery.getStartTime(), logQuery.getEndTime()); - - if (StringUtils.isNotEmpty(logQuery.getTail())) { - String tailIdFields = Arrays.stream(logQuery.getTail().split(",")).map(tail -> org.apache.commons.lang3.StringUtils.wrap(tail, "\"")).collect(Collectors.joining(",")); - String tailSql = String.format(" AND tail IN (%s)", tailIdFields); - sqlPrefix += tailSql; - } - return sqlPrefix; - } - - - private SearchSourceBuilder assembleSearchSourceBuilder(LogQuery logQuery, List keyList, BoolQueryBuilder boolQueryBuilder) { - SearchSourceBuilder builder = new SearchSourceBuilder(); - builder.query(boolQueryBuilder); - - builder.sort(commonExtensionService.getSortedKey(logQuery, logQuery.getSortKey()), logQuery.getAsc() ? ASC : DESC); - if (null != logQuery.getPage()) { - builder.from((logQuery.getPage() - 1) * logQuery.getPageSize()); - } - builder.size(logQuery.getPageSize()); - // highlight - builder.highlighter(getHighlightBuilder(keyList)); - builder.timeout(TimeValue.timeValueMinutes(2L)); - return builder; - } - - private void transformSearchResponse(SearchResponse searchResponse, final LogDTO logDTO, List keyList) { - SearchHit[] hits = searchResponse.getHits().getHits(); - if (hits == null || hits.length == 0) { - log.info("es query result is empty,es response:{}", searchResponse); - return; - } - List logDataList = Lists.newArrayList(); - for (SearchHit hit : hits) { - LogDataDTO logData = hit2DTO(hit, keyList); - // Package highlighted - logData.setHighlight(getHightlinghtMap(hit)); - logDataList.add(logData); - } - logDTO.setThisSortValue(hits[hits.length - 1].getSortValues()); - logDTO.setLogDataDTOList(logDataList); - } - - // highlight - private HighlightBuilder getHighlightBuilder(List keyList) { - HighlightBuilder highlightBuilder = new HighlightBuilder(); - for (Pair> requiredField : requiredFields) { - if (!keyList.contains(requiredField.getKey())) { - keyList.add(requiredField.getKey()); - } - } - for (String key : keyList) { - if (noHighLightSet.contains(key)) { - continue; - } - HighlightBuilder.Field highlightField = new HighlightBuilder.Field(key); - highlightBuilder.field(highlightField); - } - return highlightBuilder; - } - - - /** - * Insert data - * - * @param indexName - * @param data - * @return - */ - @Override - public void insertDoc(String indexName, Map data) throws IOException { - EsService esService = esCluster.getEsService(null); - esService.insertDoc(indexName, data); - } - - @Override - public Result EsStatistic(LogQuery logQuery) { - try { - EsStatisticResult result = new EsStatisticResult(); - result.setName(constructEsStatisticRet(logQuery)); - MilogLogStoreDO logStore = logstoreDao.queryById(logQuery.getStoreId()); - if (logStore == null) { - return new Result<>(CommonError.UnknownError.getCode(), "not found logstore", null); - } - - // get interval - String interval = searchLog.esHistogramInterval(logQuery.getEndTime() - logQuery.getStartTime()); - String esIndex = commonExtensionService.getSearchIndex(logStore.getId(), logStore.getEsIndex()); - - LogStorageTypeEnum storageTypeEnum = queryLogStorageTypeEnum(logStore.getEsClusterId()); - if (LogStorageTypeEnum.DORIS == storageTypeEnum) { - handleDorisStat(result, logQuery, logStore); - } else { - handleEsStat(result, logQuery, logStore, esIndex, interval); - } - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), result); - } catch (ElasticsearchStatusException e) { - log.error("Log query errors and log bar chart statistics report errors:[{}], Error type[{}], logQuery:[{}], user:[{}]", e, e.status(), logQuery, MoneUserContext.getCurrentUser(), e); - return Result.failParam("ES resource permissions are misconfigured, please check the username password or token"); - } catch (Exception e) { - log.error("Log query errors and log bar chart statistics report errors[{}],logQuery:[{}],user:[{}]", e, logQuery, MoneUserContext.getCurrentUser(), e); - return Result.failParam("Search term input error, please check"); - } - - } - - private void handleDorisStat(EsStatisticResult result, LogQuery logQuery, MilogLogStoreDO logStore) throws SQLException { - DataSource dataSource = Ioc.ins().getBean(Constant.LOG_STORAGE_SERV_BEAN_PRE + logStore.getEsClusterId()); - String querySql = buildQuerySqlConditional(logQuery, logStore); - String staticSql = buildDorisStatSql(querySql); - log.info("staticSql:{}", staticSql); - List timestamps = Lists.newArrayList(); - List counts = executeDorisStatQuery(dataSource, staticSql, timestamps); - - result.setCounts(counts); - result.setTimestamps(timestamps); - result.setQueryBuilder(staticSql); - result.calTotalCounts(); - } - - private static String buildDorisStatSql(String querySql) { - return "SELECT DATE_FORMAT(FROM_UNIXTIME(`timestamp` / 1000), '%Y-%m-%d %H:%i:%s') AS time_bucket, " + - "COUNT(*) AS data_count " + - "FROM (" + querySql + ") data " + - "GROUP BY time_bucket " + - "ORDER BY time_bucket"; - } - - private List executeDorisStatQuery(DataSource dataSource, String staticSql, List timestamps) throws SQLException { - List counts = new ArrayList<>(); - try (Statement statement = dataSource.getConnection().createStatement(); - ResultSet resultSet = statement.executeQuery(staticSql)) { - - while (resultSet.next()) { - timestamps.add(resultSet.getString("time_bucket")); - counts.add(resultSet.getLong("data_count")); - } - } - return counts; - } - - private void handleEsStat(EsStatisticResult result, LogQuery logQuery, MilogLogStoreDO logStore, String esIndex, String interval) throws IOException { - EsService esService = esCluster.getEsService(logStore.getEsClusterId()); - if (esService == null || StringUtils.isEmpty(esIndex)) { - throw new MilogManageException("Log Store or tail configuration exceptions"); - } - if (!StringUtils.isEmpty(interval)) { - BoolQueryBuilder queryBuilder = searchLog.getQueryBuilder(logQuery, getKeyColonPrefix(logStore.getKeyList())); - String histogramField = commonExtensionService.queryDateHistogramField(logQuery.getStoreId()); - EsClient.EsRet esRet = esService.dateHistogram(esIndex, histogramField, interval, logQuery.getStartTime(), logQuery.getEndTime(), queryBuilder); - result.setCounts(esRet.getCounts()); - result.setTimestamps(esRet.getTimestamps()); - result.setQueryBuilder(queryBuilder); - result.calTotalCounts(); - } else { - throw new MilogManageException("The minimum time interval is 10s"); - } - } - - - public String constructEsStatisticRet(LogQuery logquery) { - StringBuilder sb = new StringBuilder(); - if (!StringUtils.isEmpty(logquery.getLogstore())) { - sb.append("logstore:").append(logquery.getLogstore()).append(";"); - } - if (!StringUtils.isEmpty(logquery.getFullTextSearch())) { - sb.append("fullTextSearch:").append(logquery.getFullTextSearch()).append(";"); - } - return sb.toString(); - } - - - /** - * Get trace logs - * - * @param logQuery - * @return - */ - @Override - public TraceLogDTO getTraceLog(TraceLogQuery logQuery) { - try { - log.info("getTraceLog,param data:{}", GSON.toJson(logQuery)); - return traceLog.getTraceLog(logQuery.getAppId(), logQuery.getTraceId(), "", logQuery.getGenerationTime(), logQuery.getLevel()); - } catch (Exception e) { - log.error("Log query error, query trace log error, logQuery:[{}]", e, GSON.toJson(logQuery), e); - return TraceLogDTO.emptyData(); - } - } - - public Result getTraceAppLogUrl(TraceAppLogUrlQuery query) { - List tailDoList = tailDao.queryByAppId(query.getAppId()); - handleEnvId(query); - if (null != query.getEnvId() && !CollectionUtils.isEmpty(tailDoList)) { - tailDoList = tailDoList.stream().filter(logTailDo -> Objects.equals(query.getEnvId(), logTailDo.getEnvId())).collect(Collectors.toList()); - } - if (tailDoList == null || tailDoList.isEmpty()) { - return Result.failParam("The application is not connected to the log"); - } - String tailName = ""; - for (MilogLogTailDo tail : tailDoList) { - tailName += tail.getTail() + ","; - } - tailName = tailName.substring(0, tailName.length() - 1); - Long storeId = tailDoList.get(0).getStoreId(); - MilogLogStoreDO storeDO = logstoreDao.queryById(storeId); - MilogSpaceDO spaceDO = spaceDao.queryById(storeDO.getSpaceId()); - Long startTime = (query.getTimestamp() / 1000) - (1000 * 60 * 10); - Long endTime = (query.getTimestamp() / 1000) + (1000 * 60 * 10); - - String url = String.format("%s/project-milog/user/space-tree?spaceId=%s&inputV=traceId:%s&storeId=%s&tailName=%s&type=search&startTime=%s&endTime=%s", heraUrl, spaceDO.getId(), query.getTraceId(), storeDO.getId(), tailName, startTime, endTime); - return Result.success(url); - } - - private void handleEnvId(TraceAppLogUrlQuery traceAppLogUrlQuery) { - if (null != traceAppLogUrlQuery.getEnvId() && isOpenEnv()) { - List heraAppEnvDataList = heraAppEnvService.queryEnvById(null, null, traceAppLogUrlQuery.getEnvId()); - if (org.apache.commons.collections.CollectionUtils.isNotEmpty(heraAppEnvDataList)) { - traceAppLogUrlQuery.setEnvId(heraAppEnvDataList.getFirst().getId()); - } - } - } - - - /** - * Obtain trace logs in the data center - * - * @param regionTraceLogQuery - * @return - */ - @Override - public Result queryRegionTraceLog(RegionTraceLogQuery regionTraceLogQuery) throws IOException { - return Result.success(traceLog.getTraceLog(null, regionTraceLogQuery.getTraceId(), regionTraceLogQuery.getRegion(), "", "")); - } - - /** - * Get trace logs - * - * @param logQuery - * @return - */ - public TraceLogDTO getTraceLogFromDubbo(TraceLogQuery logQuery) throws IOException { - return logDataService.getTraceLog(logQuery); - } - - public Result getDocContext(LogContextQuery logContextQuery) { - SearchRequest searchRequest = null; - try { - if (searchLog.isLegalParam(logContextQuery) == false) { - return Result.failParam("Required parameters are missing"); - } - MilogLogStoreDO milogLogstoreDO = logstoreDao.getByName(logContextQuery.getLogstore()); - if (milogLogstoreDO.getEsClusterId() == null || StringUtils.isEmpty(milogLogstoreDO.getEsIndex())) { - return Result.failParam("store Configuration exceptions"); - } - EsService esService = esCluster.getEsService(milogLogstoreDO.getEsClusterId()); - String esIndexName = milogLogstoreDO.getEsIndex(); - List keyList = getKeyList(milogLogstoreDO.getKeyList(), milogLogstoreDO.getColumnTypeList()); - LogDTO dto = new LogDTO(); - List logDataList = new ArrayList<>(); - int times = 1, pageSize = logContextQuery.getPageSize(); - Long lineNumberSearchAfter = logContextQuery.getLineNumber(); - List logOrder = new ArrayList<>(); - logOrder.add(logContextQuery.getType()); - if (0 == logContextQuery.getType()) { - times = 2; - pageSize = pageSize / 2; - logOrder.remove(0); - logOrder.add(2); - logOrder.add(1); - } - for (int t = 0; t < times; t++) { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.filter(QueryBuilders.termQuery(LogParser.esKeyMap_logip, logContextQuery.getIp())); - boolQueryBuilder.filter(QueryBuilders.termQuery(LogParser.esKyeMap_fileName, logContextQuery.getFileName())); - SearchSourceBuilder builder = new SearchSourceBuilder(); - builder.query(boolQueryBuilder); - if (1 == logOrder.get(t)) { - // 1-after - builder.sort(LogParser.esKeyMap_timestamp, ASC); - builder.sort(LogParser.esKeyMap_lineNumber, ASC); - } else if (2 == logOrder.get(t)) { - // 2-before - builder.sort(LogParser.esKeyMap_timestamp, DESC); - builder.sort(LogParser.esKeyMap_lineNumber, DESC); - } - if (0 == logContextQuery.getType() && 2 == logOrder.get(t)) { - builder.searchAfter(new Object[]{logContextQuery.getTimestamp(), lineNumberSearchAfter + 1}); - } else { - builder.searchAfter(new Object[]{logContextQuery.getTimestamp(), lineNumberSearchAfter}); - } - builder.size(pageSize); - searchRequest = new SearchRequest(esIndexName); - searchRequest.source(builder); - SearchResponse searchResponse; - searchResponse = esService.search(searchRequest); - SearchHit[] hits = searchResponse.getHits().getHits(); - if (hits == null || hits.length == 0) { - continue; - } - if (1 == logOrder.get(t)) { - for (int i = 0; i < hits.length; i++) { - logDataList.add(this.hit2DTO(hits[i], keyList)); - } - } else if (2 == logOrder.get(t)) { - for (int i = hits.length - 1; i >= 0; i--) { - logDataList.add(this.hit2DTO(hits[i], keyList)); - } - } - } - dto.setLogDataDTOList(logDataList); - return Result.success(dto); - } catch (Exception e) { - log.error("Log query error, log context error, logContextQuery:[{}], searchRequest:[{}], user:[{}]", logContextQuery, searchRequest, MoneUserContext.getCurrentUser(), e); - return Result.failParam("System error, please try again"); - } - } - - private LogDataDTO hit2DTO(SearchHit hit, List keyList) { - LogDataDTO logData = new LogDataDTO(); - Map ferry = hit.getSourceAsMap(); - long time = 0; - if (ferry.containsKey("time") && null != ferry.get("time") && StringUtils.isNotBlank(ferry.get("time").toString())) { - try { - time = DateUtil.parse(ferry.get("time").toString()).toTimestamp().getTime(); - } catch (Exception e) { - log.error("Log query error, log context error,time:{}", ferry.get("time"), e); - } - } - if (!ferry.containsKey(LogParser.esKeyMap_timestamp) || null == ferry.get(LogParser.esKeyMap_timestamp)) { - logData.setValue(LogParser.esKeyMap_timestamp, time); - } else { - logData.setValue(LogParser.esKeyMap_timestamp, ferry.get(LogParser.esKeyMap_timestamp)); - } - for (String key : keyList) { - if (!hidenFiledSet.contains(key)) { - logData.setValue(key, ferry.get(key)); - } - } - logData.setIp(ferry.get(LogParser.esKeyMap_logip) == null ? "" : String.valueOf(ferry.get(LogParser.esKeyMap_logip))); - logData.setFileName(ferry.get(LogParser.esKyeMap_fileName) == null ? "" : String.valueOf(ferry.get(LogParser.esKyeMap_fileName))); - logData.setLineNumber(ferry.get(LogParser.esKeyMap_lineNumber) == null ? "" : String.valueOf(ferry.get(LogParser.esKeyMap_lineNumber))); - logData.setTimestamp(ferry.get(LogParser.esKeyMap_timestamp) == null ? String.valueOf(time) : String.valueOf(ferry.get(LogParser.esKeyMap_timestamp))); - logData.setLogOfString(JSON.toJSONString(logData.getLogOfKV())); - return logData; - } - - - public void logExport(LogQuery logQuery) throws Exception { - // Generate Excel - int maxLogNum = 10000; - logQuery.setPageSize(maxLogNum); - Result logDTOResult = this.logQuery(logQuery); - List> exportData = logDTOResult.getCode() != CommonError.Success.getCode() || logDTOResult.getData().getLogDataDTOList() == null || logDTOResult.getData().getLogDataDTOList().isEmpty() ? null : logDTOResult.getData().getLogDataDTOList().stream().map(logDataDto -> ExportUtils.SplitTooLongContent(logDataDto)).collect(Collectors.toList()); - HSSFWorkbook excel = ExportExcel.HSSFWorkbook4Map(exportData, generateTitle(logQuery)); - // Download - String fileName = String.format("%s_log.xls", logQuery.getLogstore()); - searchLog.downLogFile(excel, fileName); - } - - private String generateTitle(LogQuery logQuery) { - return String.format("%sLogs, search terms:[%s],time range%d-%d", logQuery.getLogstore(), logQuery.getFullTextSearch() == null ? "" : logQuery.getFullTextSearch(), logQuery.getStartTime(), logQuery.getEndTime()); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/EsIndexOperateServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/EsIndexOperateServiceImpl.java deleted file mode 100644 index e95e9b567..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/EsIndexOperateServiceImpl.java +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.api.enums.LogStorageTypeEnum; -import com.xiaomi.mone.log.manager.domain.EsCluster; -import com.xiaomi.mone.log.manager.mapper.MilogEsClusterMapper; -import com.xiaomi.mone.log.manager.model.pojo.MilogEsClusterDO; -import com.xiaomi.mone.log.manager.service.EsIndexOperateService; -import com.xiaomi.mone.log.utils.DateUtils; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.plugin.es.EsService; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.elasticsearch.action.admin.indices.alias.Alias; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.client.indices.*; -import org.elasticsearch.cluster.metadata.AliasMetadata; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.Template; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentType; - -import javax.annotation.Resource; -import java.io.IOException; -import java.util.*; - -import static com.xiaomi.mone.log.common.Constant.GSON; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2024/1/19 17:11 - */ -@Slf4j -@Service -public class EsIndexOperateServiceImpl implements EsIndexOperateService { - - @Resource - private MilogEsClusterMapper esClusterMapper; - - @Resource - private EsCluster esCluster; - - private List containKeys = Lists.newArrayList("mione", "zgq"); - - - @Override - public void createIndexPre() { - /** - * 1. First query the es information in the table - * 2. Query query index template - * 3.Modify the alias of the current index template - * 4. Query whether the index with sequence exists, create it if it does not exist - */ - List esClusterDOS = esClusterMapper.selectAll(); - for (MilogEsClusterDO esClusterDO : esClusterDOS) { - try { - LogStorageTypeEnum storageTypeEnum = LogStorageTypeEnum.queryByName(esClusterDO.getLogStorageType()); - if (LogStorageTypeEnum.ELASTICSEARCH == storageTypeEnum) { - EsService esService = esCluster.getEsService(esClusterDO.getId()); - if (esService != null) { - List indexTemplates = getIndexTemplate(esService); - String today = DateUtils.getTime(); - String tomorrow = DateUtils.getTime(1); - for (String indexTemplateName : indexTemplates) { - log.info("index template name:{}", indexTemplateName); - if (!indexTemplateName.startsWith(".") && - containKeys.stream().anyMatch(indexTemplateName::contains)) { - createOrCheckIndex(esService, indexTemplateName, today); - createOrCheckIndex(esService, indexTemplateName, tomorrow); - } - } - } - } - } catch (Exception e) { - log.error("createIndexPre,error,param:{}", GSON.toJson(esClusterDO), e); - - } - } - } - - /** - * The problem of not being able to obtain templates when compatible with high and low versions - * - * @param esService - * @return - * @throws IOException - */ - private List getIndexTemplate(EsService esService) throws IOException { - List templateNames = new ArrayList<>(); - try { - RestHighLevelClient client = esService.getEsClient().getEsOriginalClient(); - for (String containKey : containKeys) { - - GetComposableIndexTemplateRequest templateRequest = new GetComposableIndexTemplateRequest(String.format("*%s*", containKey)); - GetComposableIndexTemplatesResponse indexTemplatesResponse = client.indices() - .getIndexTemplate(templateRequest, RequestOptions.DEFAULT); - Map indexTemplates = indexTemplatesResponse.getIndexTemplates(); - - for (Map.Entry mapEntry : indexTemplates.entrySet()) { - String name = mapEntry.getKey(); - ComposableIndexTemplate value = mapEntry.getValue(); - Template template = value.template(); - - String today = DateUtils.getTime(); - String newRolloverAlias = String.format("%s-%s", name, today); - if (!templateNames.contains(name)) { - templateNames.add(name); - if (null != template) { - Settings settings = template.settings(); - String rolloverAlias = settings.get("index.lifecycle.rollover_alias"); - - if (!StringUtils.equals(rolloverAlias, newRolloverAlias)) { - updateTemplate(template, newRolloverAlias, name, client); - } - } else { - updateTemplate(null, newRolloverAlias, name, client); - } - } - } - } - } catch (Exception e) { - log.info("getIndexTemplate (_index_template error,use _template command)", e); - GetIndexTemplatesRequest indexTemplatesRequest = new GetIndexTemplatesRequest(); - List indexTemplates = esService.getIndexTemplates(indexTemplatesRequest); - - for (IndexTemplateMetadata template : indexTemplates) { - if (containKeys.stream().anyMatch(data -> template.name().contains(data)) && !templateNames.contains(template.name())) { - templateNames.add(template.name()); - Settings settings = template.settings(); - String rolloverAlias = settings.get("index.lifecycle.rollover_alias"); - if (rolloverAlias != null) { - String name = template.name(); - String today = DateUtils.getTime(); - String newRolloverAlias = String.format("%s-%s", name, today); - if (!StringUtils.equals(rolloverAlias, newRolloverAlias)) { - updateTemplate(esService, template); - } - } else { - updateTemplate(esService, template); - } - } - } - } - return templateNames; - } - - private static void updateTemplate(Template template, String newRolloverAlias, String name, RestHighLevelClient client) throws IOException { - // Get existing settings - Settings.Builder updatedSettings = Settings.builder().put(template.settings()); - - // Add or update index.lifecycle.rollover_alias - updatedSettings.put("index.lifecycle.rollover_alias", newRolloverAlias); - - // Update rollover alias - IndexTemplateMetadata.Builder updatedTemplateBuilder = IndexTemplateMetadata.builder(name); - updatedTemplateBuilder.settings(Settings.builder() - .put(template.settings()) - .put("index.lifecycle.rollover_alias", newRolloverAlias)); - - Template newTemplate = new Template(updatedSettings.build(), template.mappings(), template.aliases()); - - PutComposableIndexTemplateRequest request = new PutComposableIndexTemplateRequest() - .name(name); - ComposableIndexTemplate composableIndexTemplate = - new ComposableIndexTemplate(Arrays.asList(name + "-*"), newTemplate, null, null, null, null); - request.indexTemplate(composableIndexTemplate); - - // Send request to update template - client.indices().putIndexTemplate(request, RequestOptions.DEFAULT); - log.info("update ComposableIndexTemplate:{} success,", name); - } - - - private void updateTemplate(EsService esService, IndexTemplateMetadata template) throws IOException { - String name = template.name(); - String today = DateUtils.getTime(); - String newRolloverAlias = String.format("%s-%s", name, today); - // Get existing settings - Settings.Builder updatedSettings = Settings.builder().put(template.settings()); - - // Add or update index.lifecycle.rollover_alias - updatedSettings.put("index.lifecycle.rollover_alias", newRolloverAlias); - - // create a new put index template request instance - PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest(template.name()); - - // copy the contents of Index Template Metadata to Put Index Template Request - putRequest.patterns(template.patterns()); - putRequest.order(template.order()); - putRequest.settings(updatedSettings.build()); - putRequest.mapping(template.mappings().getSourceAsMap()); - putRequest.version(template.version()); - - // set aliases - Iterator> iterator = template.aliases().iterator(); - while (iterator.hasNext()) { - ObjectObjectCursor aliasEntry = iterator.next(); - AliasMetadata aliasMetadata = aliasEntry.value; - Alias alias = new Alias(aliasEntry.key); - alias.routing(aliasMetadata.getIndexRouting()); - alias.isHidden(aliasMetadata.isHidden()); - if (aliasMetadata.writeIndex() != null) { - alias.writeIndex(aliasMetadata.writeIndex()); - } - alias.searchRouting(aliasMetadata.searchRouting()); - putRequest.alias(alias); - } - - esService.getEsClient().getEsOriginalClient().indices().putTemplate(putRequest, RequestOptions.DEFAULT); - } - - private void createOrCheckIndex(EsService esService, String templateName, String dateSuffix) throws IOException { - String indexCreateName = String.format("%s-%s-000001", templateName, dateSuffix); - GetIndexRequest todayIndexRequest = new GetIndexRequest(indexCreateName); - try { - boolean dateIndexExists = esService.getEsClient().getEsOriginalClient().indices().exists(todayIndexRequest, RequestOptions.DEFAULT); - - if (!dateIndexExists) { - CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexCreateName); - Alias alias = new Alias(String.format("%s-%s", templateName, dateSuffix)); - alias.writeIndex(true); - createIndexRequest.alias(alias); - - createIndexRequest.source("{\n" + - " \"settings\": {\n" + - " \"index.lifecycle.rollover_alias\": \"" + String.format("%s-%s", templateName, dateSuffix) + "\"\n" + - " }\n" + - "}", XContentType.JSON); - - esService.createIndex(createIndexRequest); - log.error("index:{} create success", indexCreateName); - } - } catch (Exception e) { - log.error("createOrCheckIndex error, index:{}", indexCreateName, e); - } - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/EsIndexTemplateServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/EsIndexTemplateServiceImpl.java deleted file mode 100644 index 80f98a172..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/EsIndexTemplateServiceImpl.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import com.xiaomi.mone.log.manager.domain.EsCluster; -import com.xiaomi.mone.log.manager.domain.EsIndexTemplate; -import com.xiaomi.mone.log.manager.domain.LogTemplate; -import com.xiaomi.mone.log.manager.model.vo.UpdateIndexTemplateCommand; -import com.xiaomi.mone.log.manager.service.EsIndexTemplateService; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Resource; -import java.io.IOException; - -@Slf4j -@Service -public class EsIndexTemplateServiceImpl implements EsIndexTemplateService { - @Resource - private EsCluster esCluster; - - @Resource - private EsIndexTemplate esIndexTemplate; - - @Resource - private LogTemplate logTemplate; - - /** - * Update the index template - * - * @param command - * @return - */ - @Override - public boolean updateIndexTemplate(UpdateIndexTemplateCommand command) throws IOException { - return false; - } - - /** - * Create an index - * - * @param templateName - * @return - */ - @Override - public boolean createIndex(String templateName) throws IOException { - return esIndexTemplate.createIndex(templateName); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/HeraAppEnvServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/HeraAppEnvServiceImpl.java deleted file mode 100644 index 3c578ab13..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/HeraAppEnvServiceImpl.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import org.apache.ozhera.app.api.model.HeraAppEnvData; -import org.apache.ozhera.app.api.model.HeraSimpleEnv; -import org.apache.ozhera.app.api.service.HeraAppEnvOutwardService; -import com.xiaomi.mone.log.manager.service.HeraAppEnvService; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.plugin.dubbo.anno.Reference; -import lombok.extern.slf4j.Slf4j; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/11/12 11:48 - */ -@Slf4j -@Service -public class HeraAppEnvServiceImpl implements HeraAppEnvService { - - @Reference(interfaceClass = HeraAppEnvOutwardService.class, group = "$dubbo.env.group", check = false) - private HeraAppEnvOutwardService heraAppEnvOutwardService; - - @Override - public List querySimpleEnvAppBaseInfoId(Integer appBaseId) { - return heraAppEnvOutwardService.querySimpleEnvAppBaseInfoId(appBaseId); - } - - @Override - public List queryEnvById(Long id, Long heraAppId, Long envId) { - return heraAppEnvOutwardService.queryEnvById(id, heraAppId, envId); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/HeraAppServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/HeraAppServiceImpl.java deleted file mode 100644 index ade0725ed..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/HeraAppServiceImpl.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import org.apache.ozhera.app.api.model.HeraAppBaseInfoModel; -import org.apache.ozhera.app.api.model.HeraAppBaseInfoParticipant; -import org.apache.ozhera.app.api.model.HeraAppBaseQuery; -import org.apache.ozhera.app.api.response.AppBaseInfo; -import com.xiaomi.mone.log.manager.service.HeraAppService; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.plugin.dubbo.anno.Reference; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.List; -import java.util.concurrent.TimeUnit; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/11/11 17:00 - */ -@Slf4j -@Service -public class HeraAppServiceImpl implements HeraAppService { - - @Reference(interfaceClass = org.apache.ozhera.app.api.service.HeraAppService.class, group = "$dubbo.env.group", check = false, timeout = 5000) - private org.apache.ozhera.app.api.service.HeraAppService heraAppService; - - private static final Cache CACHE_LOCAL = CacheBuilder.newBuilder() - .maximumSize(100) - .expireAfterWrite(3, TimeUnit.MINUTES) - .build(); - - private static final Cache> CACHE_LOCAL_IDS = CacheBuilder.newBuilder() - .maximumSize(50) - .expireAfterWrite(5, TimeUnit.MINUTES) - .build(); - - @Override - public List queryAppInfoWithLog(String appName, Integer type) { - return heraAppService.queryAppInfoWithLog(appName, type); - } - - @Override - public List queryAllExistsApp() { - return heraAppService.queryAllExistsApp(); - } - - @Override - public AppBaseInfo queryById(Long id) { - AppBaseInfo appBaseInfo = CACHE_LOCAL.getIfPresent(id); - if (null == appBaseInfo) { - appBaseInfo = heraAppService.queryById(id); - if (null != appBaseInfo) { - CACHE_LOCAL.put(id, appBaseInfo); - } - } - return appBaseInfo; - } - - @Override - public List queryByIds(List ids) { - String key = StringUtils.join(ids, ","); - List appBaseInfos = CACHE_LOCAL_IDS.getIfPresent(key); - if (CollectionUtils.isEmpty(appBaseInfos)) { - appBaseInfos = heraAppService.queryByIds(ids); - if (CollectionUtils.isNotEmpty(appBaseInfos)) { - CACHE_LOCAL_IDS.put(key, appBaseInfos); - } - } - return appBaseInfos; - } - - @Override - public AppBaseInfo queryByAppId(Long appId, Integer type) { - return heraAppService.queryByAppId(appId, type); - } - - @Override - public AppBaseInfo queryByAppIdPlatFormType(String bindId, Integer platformTypeCode) { - return heraAppService.queryByAppIdPlatFormType(bindId, platformTypeCode); - } - - @Override - public AppBaseInfo queryByIamTreeId(Long iamTreeId, String bingId, Integer platformType) { - return heraAppService.queryByIamTreeId(iamTreeId, bingId, platformType); - } - - @Override - public Long countByParticipant(HeraAppBaseQuery query) { - return heraAppService.countByParticipant(query); - } - - @Override - public List queryByParticipant(HeraAppBaseQuery query) { - return heraAppService.queryByParticipant(query); - } - - @Override - public Long count(HeraAppBaseInfoModel baseInfo) { - return heraAppService.count(baseInfo); - } - - @Override - public List query(HeraAppBaseInfoModel baseInfo, Integer pageCount, Integer pageNum) { - return heraAppService.query(baseInfo, pageCount, pageNum); - } - - @Override - public HeraAppBaseInfoModel getById(Integer id) { - return heraAppService.getById(id); - } - - @Override - public int delById(Integer id) { - return heraAppService.delById(id); - } - - @Override - public Long getAppCount() { - return heraAppService.getAppCount(); - } -} - diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/HeralogHomePageServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/HeralogHomePageServiceImpl.java deleted file mode 100644 index 288a72aa8..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/HeralogHomePageServiceImpl.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.xiaomi.mone.log.manager.service.impl; - -import com.google.common.collect.Lists; -import org.apache.ozhera.app.api.response.AppBaseInfo; -import com.xiaomi.mone.log.common.Config; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.exception.CommonError; -import com.xiaomi.mone.log.manager.dao.MilogLogTailDao; -import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; -import com.xiaomi.mone.log.manager.dao.MilogStoreSpaceAuthDao; -import com.xiaomi.mone.log.manager.model.dto.MapDTO; -import com.xiaomi.mone.log.manager.model.dto.MilogSpaceTreeDTO; -import com.xiaomi.mone.log.manager.model.dto.UnAccessAppDTO; -import com.xiaomi.mone.log.manager.model.dto.ValueDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.mone.log.manager.model.pojo.MilogStoreSpaceAuth; -import com.xiaomi.mone.log.manager.service.HeralogHomePageService; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.jetbrains.annotations.Nullable; - -import javax.annotation.Resource; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - -@Slf4j -@Service -public class HeralogHomePageServiceImpl implements HeralogHomePageService { - - @Resource - private MilogLogTailDao milogLogtailDao; - - @Resource - private HeraAppServiceImpl heraAppService; - - @Resource - private MilogLogstoreDao milogLogstoreDao; - - @Resource - private MilogStoreSpaceAuthDao milogStoreSpaceAuthDao; - - private List> milogpattern; - - { - String pattern = Config.ins().get("milogpattern", ""); - String[] split = pattern.split(","); - ArrayList> valueDTOS = new ArrayList<>(); - for (String s : split) { - valueDTOS.add(new ValueDTO<>(s)); - } - milogpattern = valueDTOS; - } - - @Override - public Result> milogAccess() { - Long total = heraAppService.getAppCount(); - int access = milogLogtailDao.appCount(); - HashMap map = new HashMap<>(); - map.put("total", total); - map.put("access", access); - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), map); - } - - @Override - public Result> unAccessAppList() { - List appBaseInfos = heraAppService.queryAllExistsApp(); - Map appMap = appBaseInfos.stream() - .collect(Collectors.toMap(AppBaseInfo::getId, AppBaseInfo::getAppName)); - List hasAccessAppId = milogLogtailDao.queryAllAppId(); - ArrayList list = new ArrayList<>(); - for (Map.Entry app : appMap.entrySet()) { - if (!hasAccessAppId.contains(app.getKey())) { - list.add(new UnAccessAppDTO(app.getKey().longValue(), app.getValue())); - } - } - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), list); - } - - @Override - public Result> getMilogSpaceTree(Long spaceId) { - List stores = getMilogLogStoreDOS(spaceId); - List spaceTreeDTOS = stores.stream().map(milogLogstoreDO -> { - Long logstoreDOId = milogLogstoreDO.getId(); - MilogSpaceTreeDTO milogSpaceTreeDTO = new MilogSpaceTreeDTO(); - milogSpaceTreeDTO.setLabel(milogLogstoreDO.getLogstoreName()); - milogSpaceTreeDTO.setValue(logstoreDOId); - List logTailDos = milogLogtailDao.getMilogLogtailByStoreId(logstoreDOId); - if (CollectionUtils.isNotEmpty(logTailDos)) { - List> collect = logTailDos.stream().map(logTailDo -> { - MapDTO mapDTO = new MapDTO(); - mapDTO.setValue(logTailDo.getId()); - mapDTO.setLabel(logTailDo.getTail()); - return mapDTO; - }).collect(Collectors.toList()); - milogSpaceTreeDTO.setChildren(collect); - } - return milogSpaceTreeDTO; - }).collect(Collectors.toList()); - return Result.success(spaceTreeDTOS); - } - - /** - * Query the store that originally belonged to the space, and query the authorized store - * - * @param spaceId - * @return - */ - @Override - @Nullable - public List getMilogLogStoreDOS(Long spaceId) { - List storeDOS = Lists.newArrayList(); - List stores = milogLogstoreDao.getMilogLogstoreBySpaceId(spaceId); - List storeSpaceAuths = milogStoreSpaceAuthDao.queryStoreIdsBySpaceId(spaceId); - if (CollectionUtils.isNotEmpty(stores)) { - storeDOS = stores; - } - if (CollectionUtils.isNotEmpty(storeSpaceAuths)) { - List collect = storeSpaceAuths.stream() - .map(storeSpaceAuth -> milogLogstoreDao.queryById(storeSpaceAuth.getStoreId())) - .filter(Objects::nonNull).collect(Collectors.toList()); - storeDOS.addAll(collect); - } - return storeDOS; - } - - @Override - public Result>> getMiloglogAccessPattern() { - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), milogpattern); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/KafkaMqConfigService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/KafkaMqConfigService.java deleted file mode 100644 index fa37cfba8..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/KafkaMqConfigService.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.manager.model.dto.DictionaryDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogAppMiddlewareRel; -import com.xiaomi.mone.log.manager.service.CommonRocketMqService; -import com.xiaomi.mone.log.manager.service.MqConfigService; -import com.xiaomi.mone.log.utils.KafkaUtils; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.plugin.config.anno.Value; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.kafka.clients.admin.*; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.common.KafkaFuture; - -import java.util.*; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/11/29 11:37 - */ -@Slf4j -@Service -public class KafkaMqConfigService implements MqConfigService, CommonRocketMqService { - - @Value("$kafka.use.ssl") - private String kafkaUseSsl; - - @Value("$kafka.sll.location") - private String kafkaSllLocation; - - @Override - public MilogAppMiddlewareRel.Config generateConfig(String ak, String sk, String nameServer, String serviceUrl, String authorization, String orgId, String teamId, Long exceedId, String name, String source, Long id) { - MilogAppMiddlewareRel.Config config = new MilogAppMiddlewareRel.Config(); - - Properties properties = new Properties(); - - if (StringUtils.isNotEmpty(ak) && StringUtils.isNotEmpty(sk) && Objects.equals("true", kafkaUseSsl)) { - properties.putAll(KafkaUtils.getSslKafkaProperties(nameServer, ak, sk, kafkaSllLocation)); - } else if (StringUtils.isNotEmpty(ak) && StringUtils.isNotEmpty(sk)) { - properties.putAll(KafkaUtils.getVpc9094KafkaProperties(nameServer, ak, sk)); - } else { - properties.putAll(KafkaUtils.getDefaultKafkaProperties(nameServer)); - } - properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, nameServer); - String topicName = generateSimpleTopicName(id, name); - // create AdminClient - try (AdminClient adminClient = AdminClient.create(properties)) { - List topics = Lists.newArrayList(topicName).stream().map(topic -> new NewTopic(topic, Optional.of(1), - Optional.of((short) 1))).collect(Collectors.toList()); - CreateTopicsResult result = adminClient.createTopics(topics); - // Wait for the theme to be created - Map> values = result.values(); - for (Map.Entry> entry : values.entrySet()) { - try { - entry.getValue().get(); // Wait for the creation operation to complete - log.info("Topic:{},created successfully.", entry.getKey()); - } catch (InterruptedException | ExecutionException e) { - log.error("Failed to create topic:{}", entry.getKey(), e); - } - } - } catch (Exception e) { - log.error("create kafka topic error,topic:{}", topicName, e); - } - config.setTopic(topicName); - config.setPartitionCnt(1); - return config; - } - - @Override - public List queryExistsTopic(String ak, String sk, String nameServer, String serviceUrl, String authorization, String orgId, String teamId) { - Properties properties = new Properties(); - - if (StringUtils.isNotEmpty(ak) && StringUtils.isNotEmpty(sk) && Objects.equals("true", kafkaUseSsl)) { - properties.putAll(KafkaUtils.getSslKafkaProperties(nameServer, ak, sk, kafkaSllLocation)); - } else if (StringUtils.isNotEmpty(ak) && StringUtils.isNotEmpty(sk)) { - properties.putAll(KafkaUtils.getVpc9094KafkaProperties(nameServer, ak, sk)); - } else { - properties.putAll(KafkaUtils.getDefaultKafkaProperties(nameServer)); - } - properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, nameServer); - - // create AdminClient - try (AdminClient adminClient = AdminClient.create(properties)) { - // get topic list - Set topics = getTopicList(adminClient); - return topics.stream().map(data -> { - DictionaryDTO dictionaryDTO = new DictionaryDTO<>(); - dictionaryDTO.setLabel(data); - dictionaryDTO.setValue(data); - return dictionaryDTO; - }).collect(Collectors.toList()); - } catch (Exception e) { - log.error("query kafka topic list error", e); - } - return null; - } - - private static Set getTopicList(AdminClient adminClient) throws ExecutionException, InterruptedException { - // Configure ListTopicsOptions - ListTopicsOptions options = new ListTopicsOptions(); - options.listInternal(true); - // Get topic list - ListTopicsResult topicsResult = adminClient.listTopics(options); - return topicsResult.names().get(); - } - - @Override - public List createCommonTagTopic(String ak, String sk, String nameServer, String serviceUrl, String authorization, String orgId, String teamId) { - return null; - } - - @Override - public boolean CreateGroup(String ak, String sk, String nameServer) { - - return false; - } - - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogAnalyseService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogAnalyseService.java deleted file mode 100644 index 219ec0bd1..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogAnalyseService.java +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.exception.CommonError; -import com.xiaomi.mone.log.manager.common.context.MoneUserContext; -import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; -import com.xiaomi.mone.log.manager.domain.AnalyseLog; -import com.xiaomi.mone.log.manager.mapper.MilogAnalyseDashboardGraphRefMapper; -import com.xiaomi.mone.log.manager.mapper.MilogAnalyseDashboardMapper; -import com.xiaomi.mone.log.manager.mapper.MilogAnalyseGraphMapper; -import com.xiaomi.mone.log.manager.mapper.MilogAnalyseGraphTypeMapper; -import com.xiaomi.mone.log.manager.model.convert.DGRefConvert; -import com.xiaomi.mone.log.manager.model.convert.DashboardConvert; -import com.xiaomi.mone.log.manager.model.convert.GraphConvert; -import com.xiaomi.mone.log.manager.model.convert.GraphTypeConvert; -import com.xiaomi.mone.log.manager.model.dto.*; -import com.xiaomi.mone.log.manager.model.pojo.*; -import com.xiaomi.mone.log.manager.model.vo.*; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Resource; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -@Slf4j -@Service -public class LogAnalyseService { - @Resource - private MilogAnalyseDashboardGraphRefMapper dgRefMapper; - - @Resource - private MilogAnalyseDashboardMapper dashboardMapper; - - @Resource - private MilogAnalyseGraphMapper graphMapper; - - @Resource - private MilogAnalyseGraphTypeMapper graphTypeMapper; - - @Resource - private AnalyseLog analyseLog; - - @Resource - private MilogLogstoreDao logstoreDao; - - private static final List ignoreKeyList = new ArrayList<>(); - - { - ignoreKeyList.add("message"); - ignoreKeyList.add("logsource"); - ignoreKeyList.add("timestamp"); - ignoreKeyList.add("traceId"); - ignoreKeyList.add("line"); - } - - public Result getDashboardGraph(LogAnalyseQuery logAnalyseQuery) { - List dashboardList = dashboardMapper.getByStoreId(logAnalyseQuery.getStoreId()); - if (dashboardList == null || dashboardList.isEmpty()) { - return Result.success(); - } - MilogAnalyseDashboardDO dashboard = dashboardList.get(0); - List dashboardGraphList = graphMapper.getDashboardGraph(dashboard.getId()); - return Result.success(DashboardConvert.INSTANCE.fromDO(dashboard, dashboardGraphList)); - } - - public Result createGraph(CreateGraphCmd cmd) { - MilogAnalyseGraphDO graphDO = GraphConvert.INSTANCE.toDO(cmd); - graphDO.setCreator(MoneUserContext.getCurrentUser().getUser()); - graphDO.setUpdater(MoneUserContext.getCurrentUser().getUser()); - long thisTime = System.currentTimeMillis(); - graphDO.setCreateTime(thisTime); - graphDO.setUpdateTime(thisTime); - graphMapper.insert(graphDO); - return Result.success(graphDO.getId()); - } - - public Result updateGraph(UpdateGraphCmd cmd) { - MilogAnalyseGraphDO graphDO = GraphConvert.INSTANCE.toDO(cmd); - graphDO.setUpdater(MoneUserContext.getCurrentUser().getUser()); - graphDO.setUpdateTime(System.currentTimeMillis()); - int res = graphMapper.updateById(graphDO); - return res == 1 ? Result.success(true) : Result.fail(CommonError.ParamsError); - } - - public Result deleteGraph(Long graphId) { - graphMapper.deleteById(graphId); - dgRefMapper.deleteGraphRef(graphId); - return Result.success(true); - } - - public Result> searchGraph(GraphQuery query) { - List graphList = graphMapper.search(query); - return Result.success(graphList); - } - - public Result ref(DGRefCmd cmd) { - Long refed = dgRefMapper.isRefed(cmd.getDashboardId(), cmd.getGraphId()); - if (refed == null || refed == 0) { - int res = dgRefMapper.insert(DGRefConvert.INSTANCE.toDo(cmd)); - return res == 1 ? Result.success(true) : Result.fail(CommonError.ParamsError); - } - return Result.fail(CommonError.ParamsError.getCode(), "The chart already exists for this dashboard"); - } - - public Result delRef(DGRefDelCmd cmd) { - int res = dgRefMapper.delRef(cmd.getDashboardId(), cmd.getGraphId()); - return res == 1 ? Result.success(true) : Result.fail(CommonError.ParamsError); - } - - public Result updateRef(DGRefUpdateCmd cmd) { - if (cmd.getDashboardId() == null || cmd.getGraphList() == null || cmd.getGraphList().isEmpty()) { - return Result.fail(CommonError.ParamsError); - } - for (DGRefUpdateCmd.DGRefDetailUpdateCmd graph : cmd.getGraphList()) { - MilogAnalyseDashboardGraphRefDO refOld = dgRefMapper.getRef(cmd.getDashboardId(), graph.getGraphId()); - if (refOld == null) { - return Result.fail(CommonError.ParamsError); - } - MilogAnalyseDashboardGraphRefDO refNew = DGRefConvert.INSTANCE.toDo(graph); - refNew.setId(refOld.getId()); - dgRefMapper.updateById(refNew); - } - return Result.success(true); - } - - public Result data(LogAnalyseDataQuery query) throws IOException { - LogAnalyseDataDTO dto = analyseLog.getData(query); - return Result.success(dto); - } - - public Result dataPre(LogAnalyseDataPreQuery query) throws IOException { - LogAnalyseDataDTO dto = analyseLog.getData(query); - return Result.success(dto); - } - - public Result createDashboard(CreateDashboardCmd cmd) { - MilogAnalyseDashboardDO dashboardDO = DashboardConvert.INSTANCE.toDO(cmd); - dashboardDO.setCreateTime(System.currentTimeMillis()); - dashboardDO.setCreator(MoneUserContext.getCurrentUser().getUser()); - - // Set the default name, and change it later if you need multiple dashboards - MilogLogStoreDO logstore = logstoreDao.queryById(cmd.getStoreId()); - if (logstore == null) { - return Result.failParam("Create failed, logstore does not exist"); - } - dashboardDO.setName(logstore.getLogstoreName() + "dashboard"); - - dashboardMapper.insert(dashboardDO); - return Result.success(dashboardDO.getId()); - } - - public Result> type() { - List gtaphTypeDOList = graphTypeMapper.selectList(null); - List graphTypeDTOList = GraphTypeConvert.INSTANCE.toDTOList(gtaphTypeDOList); - return Result.success(graphTypeDTOList); - } - - - public Result> supportKey(Long storeId) { - if (storeId == null) { - return Result.fail(CommonError.ParamsError); - } - MilogLogStoreDO logStore = logstoreDao.queryById(storeId); - if (logStore == null) { - return Result.fail(CommonError.ParamsError); - } - String[] keyDescripArray = logStore.getKeyList().split(","); - List keyList = new ArrayList<>(); - String key; - for (String keyDes : keyDescripArray) { - key = keyDes.split(":")[0]; - if (ignoreKeyList.contains(key)) { - continue; - } - keyList.add(key); - } - return Result.success(keyList); - } - -} \ No newline at end of file diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogCountServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogCountServiceImpl.java deleted file mode 100644 index 615c793fe..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogCountServiceImpl.java +++ /dev/null @@ -1,291 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.domain.EsCluster; -import com.xiaomi.mone.log.manager.domain.Tpc; -import com.xiaomi.mone.log.manager.mapper.MilogLogCountMapper; -import com.xiaomi.mone.log.manager.mapper.MilogLogstailMapper; -import com.xiaomi.mone.log.manager.model.dto.LogtailCollectTopDTO; -import com.xiaomi.mone.log.manager.model.dto.LogtailCollectTrendDTO; -import com.xiaomi.mone.log.manager.model.dto.SpaceCollectTopDTO; -import com.xiaomi.mone.log.manager.model.dto.SpaceCollectTrendDTO; -import com.xiaomi.mone.log.manager.model.pojo.LogCountDO; -import com.xiaomi.mone.log.manager.service.LogCountService; -import com.xiaomi.mone.log.utils.DateUtils; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.common.StringUtils; -import com.xiaomi.youpin.docean.plugin.es.EsService; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.elasticsearch.client.core.CountRequest; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.builder.SearchSourceBuilder; - -import javax.annotation.Resource; -import java.io.IOException; -import java.text.NumberFormat; -import java.text.SimpleDateFormat; -import java.util.*; - -@Slf4j -@Service -public class LogCountServiceImpl implements LogCountService { - @Resource - private MilogLogCountMapper logCountMapper; - - @Resource - private MilogLogstailMapper logtailMapper; - - @Resource - private EsCluster esCluster; - - @Resource - private Tpc tpc; - - private List logtailCollectTopList = new ArrayList<>(); - - private Map>> logtailCollectTrendMap = new HashMap<>(); - - private List spaceCollectTopList = new ArrayList<>(); - - private Map> spaceCollectTrendCache = new HashMap<>(); - - /** - * Top log collections - * - * @return - */ - @Override - public Result> collectTop() { - return Result.success(logtailCollectTopList); - } - - /** - * Log collection trends - * - * @param tailId - * @return - */ - @Override - public Result> collectTrend(Long tailId) { - String thisDay = DateUtils.getDaysAgo(1); - if (!logtailCollectTrendMap.containsKey(thisDay) || !logtailCollectTrendMap.get(thisDay).containsKey(tailId)) { - collectTrendCount(tailId); - } - return Result.success(logtailCollectTrendMap.get(thisDay).get(tailId)); - } - - @Override - public void collectTopCount() { - List> res = logCountMapper.collectTopCount(DateUtils.getDaysAgo(7), DateUtils.getDaysAgo(1)); - List dtoList = new ArrayList(); - LogtailCollectTopDTO dto; - for (Map count : res) { - dto = new LogtailCollectTopDTO(); - dto.setTail(String.valueOf(count.get("tail"))); - dto.setNumber(getLogNumberFormat(Long.parseLong(count.get("number").toString()))); - dtoList.add(dto); - } - logtailCollectTopList = dtoList; - } - - @Override - public void collectTrendCount(Long tailId) { - String yesterday = DateUtils.getDaysAgo(1); - synchronized (this) { - if (logtailCollectTrendMap.containsKey(yesterday) && logtailCollectTrendMap.get(yesterday).containsKey(tailId)) { - return; - } - if (!logtailCollectTrendMap.containsKey(yesterday)) { - logtailCollectTrendMap.remove(DateUtils.getDaysAgo(2)); - logtailCollectTrendMap.put(yesterday, new HashMap<>()); - } - } - synchronized (tailId) { - if (logtailCollectTrendMap.get(yesterday).containsKey(tailId)) { - return; - } - List> resList = logCountMapper.collectTrend(DateUtils.getDaysAgo(7), DateUtils.getDaysAgo(1), tailId); - List trendlist = new ArrayList<>(); - LogtailCollectTrendDTO dto; - for (Map res : resList) { - dto = new LogtailCollectTrendDTO(); - dto.setDay(String.valueOf(res.get("day")).split("-")[2] + "day"); - dto.setNumber(String.valueOf(res.get("number"))); - dto.setShowNumber(getLogNumberFormat(Long.parseLong(String.valueOf(res.get("number"))))); - trendlist.add(dto); - } - logtailCollectTrendMap.get(yesterday).put(tailId, trendlist); - } - } - - @Override - public void collectLogCount(String thisDay) throws IOException { - log.info("Statistics log starts"); - logCountMapper.deleteThisDay(thisDay); - if (StringUtils.isEmpty(thisDay)) { - thisDay = new SimpleDateFormat("yyyy-MM-dd").format(new Date()); - } - Long thisDayFirstMillisecond = DateUtils.getThisDayFirstMillisecond(thisDay); - List logCountDOList = new ArrayList(); - LogCountDO logCountDO; - EsService esService; - Long total; - String esIndex; - List> tailList = logtailMapper.getAllTailForCount(); - for (Map tail : tailList) { - try { - esIndex = String.valueOf(tail.get("es_index")); - if (StringUtils.isEmpty(esIndex) || tail.get("es_cluster_id") == null) { - total = 0l; - esIndex = ""; - } else { - esService = esCluster.getEsService(Long.parseLong(String.valueOf(tail.get("es_cluster_id")))); - if (esService == null) { - log.warn("Statistics logs warn,tail:{} the logs are not counted and the ES client is not generated", tail); - continue; - } - SearchSourceBuilder builder = new SearchSourceBuilder(); - BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); - boolQueryBuilder.filter(QueryBuilders.termQuery("tail", tail.get("tail"))); - boolQueryBuilder.filter(QueryBuilders.rangeQuery("timestamp").from(thisDayFirstMillisecond).to(thisDayFirstMillisecond + DateUtils.dayms - 1)); - builder.query(boolQueryBuilder); - // statistics - CountRequest countRequest = new CountRequest(); - countRequest.indices(esIndex); - countRequest.source(builder); - total = esService.count(countRequest); - } - logCountDO = new LogCountDO(); - logCountDO.setTailId(Long.parseLong(String.valueOf(tail.get("id")))); - logCountDO.setEsIndex(esIndex); - logCountDO.setNumber(total); - logCountDO.setDay(thisDay); - logCountDOList.add(logCountDO); - } catch (Exception e) { - log.error("collectLogCount error,thisDay:{}", thisDay, e); - } - } - long res = 0; - if (CollectionUtils.isNotEmpty(logCountDOList)) { - res = logCountMapper.batchInsert(logCountDOList); - } - log.info("End of statistics log,Should be counted{}，Total statistics{}", tailList.size(), res); - } - - @Override - public boolean isLogtailCountDone(String day) { - Long logTailCountDone = logCountMapper.isLogtailCountDone(day); - return !Objects.equals(logTailCountDone, 0L); - } - - @Override - public void deleteHistoryLogCount() { - Calendar calendar = Calendar.getInstance(); - calendar.add(Calendar.DAY_OF_MONTH, -70); - String deleteBeforedDay = new SimpleDateFormat("yyyy-MM-dd").format(calendar.getTime()); - logCountMapper.deleteBeforeDay(deleteBeforedDay); - } - - @Override - public void collectLogDelete(String day) { - logCountMapper.deleteThisDay(day); - } - - @Override - public void collectTrendRefresh() { - logtailCollectTrendMap.clear(); - } - - private String getLogNumberFormat(long number) { - NumberFormat format = NumberFormat.getInstance(); - format.setMaximumFractionDigits(2); - format.setMinimumFractionDigits(2); - if (number >= 100000000) { - return format.format((float) number / 100000000) + "hundred million"; - } else if (number >= 1000000) { - return format.format((float) number / 1000000) + "million"; - } else if (number >= 10000) { - return format.format((float) number / 10000) + "ten thousand"; - } else { - return number + "strip"; - } - } - - @Override - public void showLogCountCache() { - log.info("logTopList:{}", logtailCollectTopList); - log.info("logTrendMap:{}", logtailCollectTrendMap); - } - - @Override - public Result> collectSpaceTop() { - return Result.success(spaceCollectTopList); - } - - @Override - public void collectSpaceTopCount() { - List> spaceTopList = logCountMapper.collectSpaceCount(DateUtils.getDaysAgo(7), DateUtils.getDaysAgo(1)); - List dtoList = new ArrayList<>(); - SpaceCollectTopDTO dto; - for (Map spaceTop : spaceTopList) { - dto = new SpaceCollectTopDTO(); - dto.setSpaceName(String.valueOf(spaceTop.get("spaceName"))); - dto.setNumber(getLogNumberFormat(Long.parseLong(spaceTop.get("number").toString()))); - dto.setOrgName(tpc.getSpaceLastOrg(Long.parseLong(String.valueOf(spaceTop.get("spaceId"))))); - dtoList.add(dto); - } - spaceCollectTopList = dtoList; - } - - @Override - public void collectSpaceTrend() { - List> spaceTrendList = logCountMapper.collectSpaceTrend(DateUtils.getDaysAgo(7), DateUtils.getDaysAgo(1)); - if (CollectionUtils.isEmpty(spaceTrendList)) { - return; - } - Map> cache = new HashMap<>(); - List dtoList = new ArrayList<>(); - SpaceCollectTrendDTO dto; - Long lastSpaceId = Long.parseLong(String.valueOf(spaceTrendList.get(0).get("spaceId"))); - Long thisSpaceId; - for (Map spaceTrend : spaceTrendList) { - thisSpaceId = Long.parseLong(String.valueOf(spaceTrend.get("spaceId"))); - if (!thisSpaceId.equals(lastSpaceId)) { - cache.put(lastSpaceId, dtoList); - dtoList = new ArrayList<>(); - } - dto = new SpaceCollectTrendDTO(); - dto.setSpaceName(String.valueOf(spaceTrend.get("spaceName"))); - dto.setNumber(String.valueOf(spaceTrend.get("number"))); - dto.setShowNumber(getLogNumberFormat(Long.parseLong(String.valueOf(spaceTrend.get("number"))))); - dto.setOrgName(tpc.getSpaceLastOrg(Long.parseLong(String.valueOf(spaceTrend.get("spaceId"))))); - dto.setDay(String.valueOf(spaceTrend.get("day")).split("-")[2] + "day"); - dtoList.add(dto); - lastSpaceId = thisSpaceId; - } - spaceCollectTrendCache = cache; - } - - @Override - public Result> spaceCollectTrend(Long spaceId) { - return Result.success(spaceCollectTrendCache.get(spaceId)); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogProcessServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogProcessServiceImpl.java deleted file mode 100644 index 7111979f6..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogProcessServiceImpl.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import com.xiaomi.mone.log.api.model.vo.TailLogProcessDTO; -import com.xiaomi.mone.log.api.model.vo.UpdateLogProcessCmd; -import com.xiaomi.mone.log.api.service.LogProcessService; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.dao.MilogLogTailDao; -import com.xiaomi.mone.log.manager.domain.LogProcess; -import com.xiaomi.mone.log.manager.mapper.MilogLogProcessMapper; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogProcessDOMybatis; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.Nullable; - -import javax.annotation.Resource; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -@Slf4j -@Service -public class LogProcessServiceImpl implements LogProcessService { - - @Resource - private MilogLogProcessMapper processMapper; - - @Resource - private LogProcess logProcess; - - @Resource - private MilogLogTailDao logtailDao; - - /** - * Update log collection progress - * - * @param cmd - */ - @Override - public void updateLogProcess(UpdateLogProcessCmd cmd) { - if (null != logProcess) { - logProcess.updateLogProcess(cmd); - } - } - - public MilogLogProcessDOMybatis getByIdFramework(Long id) { - return processMapper.selectById(id); - } - - /** - * Get the log collection progress of the store - * - * @param type - * @param value - * @return - */ - public Result> getStoreLogProcess(String type, String value) { - if (StringUtils.isEmpty(type) || StringUtils.isEmpty(value)) { - return Result.failParam("Type and value cannot be empty"); - } - List dtoList; - switch (type) { - case "store": - dtoList = logProcess.getStoreLogProcess(Long.parseLong(value), ""); - break; - case "tail": - dtoList = logProcess.getTailLogProcess(Long.parseLong(value), ""); - break; - case "ip": - String[] params = value.split(","); - dtoList = logProcess.getStoreLogProcess(Long.parseLong(params[0]), params[1]); - break; - case "tail&ip": - String[] params2 = value.split(","); - dtoList = logProcess.getTailLogProcess(Long.parseLong(params2[0]), params2[1]); - break; - default: - return Result.failParam("The type type is not legal"); - } - return Result.success(dtoList); - } - - public Result> getColProcessImperfect(Double progressRation) { - if (null == progressRation) { - return Result.failParam("The parameter cannot be empty"); - } - return Result.success(upColProcessTailName(logProcess.getColProcessImperfect(progressRation))); - } - - @Nullable - private List upColProcessTailName(List colProcessImperfect) { - // Create a mapping of tailId to CollectDetail - Map tailIdToDetailMap = new HashMap<>(); - - // Process and update details - colProcessImperfect.parallelStream().forEach(collectDetail -> { - String tailId = collectDetail.getTailId(); - if (StringUtils.isNotBlank(tailId)) { - try { - MilogLogTailDo logTailDo = logtailDao.queryById(Long.valueOf(tailId)); - if (logTailDo != null) { - collectDetail.setTailName(logTailDo.getTail()); - } - - // Update the map with the latest details - tailIdToDetailMap.merge(tailId, collectDetail, (existingDetail, newDetail) -> { - existingDetail.getIpList().addAll(newDetail.getIpList()); - existingDetail.getIpList().stream().distinct().collect(Collectors.toList()); - existingDetail.getFileProgressDetails().addAll(newDetail.getFileProgressDetails()); - return existingDetail; - }); - } catch (Exception e) { - log.error("process data error", e); - } - } - }); - - // Return the values from the map as a list - return new ArrayList<>(tailIdToDetailMap.values()); - - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogQueryServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogQueryServiceImpl.java deleted file mode 100644 index 96774b8b1..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogQueryServiceImpl.java +++ /dev/null @@ -1,449 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import cn.hutool.core.date.DateUtil; -import cn.hutool.core.date.StopWatch; -import com.google.common.collect.Lists; -import com.google.gson.Gson; -import com.xiaomi.mone.es.EsClient; -import com.xiaomi.mone.log.api.model.dto.TraceLogDTO; -import com.xiaomi.mone.log.api.model.vo.TraceLogQuery; -import com.xiaomi.mone.log.api.service.LogDataService; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.exception.CommonError; -import com.xiaomi.mone.log.manager.common.context.MoneUserContext; -import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; -import com.xiaomi.mone.log.manager.domain.EsCluster; -import com.xiaomi.mone.log.manager.domain.SearchLog; -import com.xiaomi.mone.log.manager.domain.TraceLog; -import com.xiaomi.mone.log.manager.model.dto.EsStatisticResult; -import com.xiaomi.mone.log.manager.model.dto.LogDTO; -import com.xiaomi.mone.log.manager.model.dto.LogDataDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.vo.LogContextQuery; -import com.xiaomi.mone.log.manager.model.vo.LogQuery; -import com.xiaomi.mone.log.manager.model.vo.RegionTraceLogQuery; -import com.xiaomi.mone.log.manager.service.EsDataBaseService; -import com.xiaomi.mone.log.manager.service.LogQueryService; -import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionService; -import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionServiceFactory; -import com.xiaomi.mone.log.parse.LogParser; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.common.StringUtils; -import com.xiaomi.youpin.docean.plugin.es.EsService; -import lombok.extern.slf4j.Slf4j; -import org.apache.poi.hssf.usermodel.HSSFWorkbook; -import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; -import run.mone.excel.ExportExcel; - -import javax.annotation.Resource; -import java.io.IOException; -import java.util.*; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.GSON; -import static com.xiaomi.mone.log.manager.common.utils.ManagerUtil.getKeyColonPrefix; -import static com.xiaomi.mone.log.manager.common.utils.ManagerUtil.getKeyList; -import static org.elasticsearch.search.sort.SortOrder.ASC; -import static org.elasticsearch.search.sort.SortOrder.DESC; - -@Slf4j -@Service -@com.xiaomi.youpin.docean.plugin.dubbo.anno.Service(interfaceClass = LogDataService.class) -public class LogQueryServiceImpl implements LogQueryService, LogDataService, EsDataBaseService { - - @Resource - private MilogLogstoreDao logstoreDao; - - @Resource - private EsCluster esCluster; - - @Resource - private TraceLog traceLog; - - @Resource - private SearchLog searchLog; - - private Set noHighLightSet = new HashSet<>(); - - private Set hidenFiledSet = new HashSet<>(); - - private CommonExtensionService commonExtensionService; - - public void init() { - commonExtensionService = CommonExtensionServiceFactory.getCommonExtensionService(); - } - - { - noHighLightSet.add("logstore"); - noHighLightSet.add("logsource"); - noHighLightSet.add("tail"); - noHighLightSet.add("timestamp"); - noHighLightSet.add("linenumber"); - - hidenFiledSet.add("mqtag"); - hidenFiledSet.add("mqtopic"); - hidenFiledSet.add("logstore"); - hidenFiledSet.add("linenumber"); - hidenFiledSet.add("filename"); - } - - /** - * Read the data of the ES index - * - * @param logQuery - * @return - */ - @Override - public Result logQuery(LogQuery logQuery) { - String logInfo = String.format("queryText:%s, user:%s, logQuery:%s", logQuery.getFullTextSearch(), MoneUserContext.getCurrentUser().getUser(), logQuery); - log.info("query simple param:{}", logInfo); - - SearchRequest searchRequest = null; - StopWatch stopWatch = new StopWatch("HERA-LOG-QUERY"); - try { - stopWatch.start("before-query"); - MilogLogStoreDO logStore = logstoreDao.getByName(logQuery.getLogstore()); - if (logStore == null) { - log.warn("[EsDataService.logQuery] not find logstore:[{}]", logQuery.getLogstore()); - return Result.failParam("Not found [" + logQuery.getLogstore() + "]The corresponding data"); - } - EsService esService = esCluster.getEsService(logStore.getEsClusterId()); - String esIndexName = commonExtensionService.getSearchIndex(logQuery.getStoreId(), logStore.getEsIndex()); - if (esService == null || StringUtils.isEmpty(esIndexName)) { - log.warn("[EsDataService.logQuery] logStore:[{}] Configuration exceptions", logQuery.getLogstore()); - return Result.failParam("logStore configuration exception"); - } - List keyList = getKeyList(logStore.getKeyList(), logStore.getColumnTypeList()); - // Build query parameters - BoolQueryBuilder boolQueryBuilder = searchLog.getQueryBuilder(logQuery, getKeyColonPrefix(logStore.getKeyList())); - SearchSourceBuilder builder = new SearchSourceBuilder(); - builder.query(boolQueryBuilder); - LogDTO dto = new LogDTO(); - stopWatch.stop(); - - // query - stopWatch.start("bool-query"); - builder.sort(commonExtensionService.getSortedKey(logQuery, logQuery.getSortKey()), logQuery.getAsc() ? ASC : DESC); -// if ("cn".equals(milogLogstoreDO.getMachineRoom())) { -// builder.sort(LogParser.esKeyMap_lineNumber, logQuery.getAsc() ? ASC : DESC); -// } - // page - if (logQuery.getBeginSortValue() != null && logQuery.getBeginSortValue().length != 0) { - builder.searchAfter(logQuery.getBeginSortValue()); - } - builder.size(logQuery.getPageSize()); - // highlight - builder.highlighter(getHighlightBuilder(keyList)); - builder.timeout(TimeValue.timeValueMinutes(1L)); - searchRequest = new SearchRequest(esIndexName); - searchRequest.source(builder); - dto.setSourceBuilder(builder); - SearchResponse searchResponse = esService.search(searchRequest); - stopWatch.stop(); - if (stopWatch.getLastTaskTimeMillis() > 7 * 1000) { - log.warn("##LONG-COST-QUERY##{} cost:{} ms, msg:{}", stopWatch.getLastTaskName(), stopWatch.getLastTaskTimeMillis(), logInfo); - } - - //Result transformation - stopWatch.start("after-query"); - transformSearchResponse(searchResponse, dto, keyList); - - stopWatch.stop(); - if (stopWatch.getTotalTimeMillis() > 15 * 1000) { - log.warn("##LONG-COST-QUERY##{} cost:{} ms, msg:{}", "gt15s", stopWatch.getLastTaskTimeMillis(), logInfo); - } - - return Result.success(dto); - } catch (ElasticsearchStatusException e) { - log.error("Log query error, log search error, error type[{}], logQuery:[{}], searchRequest:[{}], user:[{}]", e.status(), logQuery, searchRequest, MoneUserContext.getCurrentUser(), e); - return Result.failParam("If the permissions of ES resources are configured incorrectly, check the username and password or token"); - } catch (Throwable e) { - log.error("Log query errors and log search errors,logQuery:[{}],searchRequest:[{}],user:[{}]", logQuery, searchRequest, MoneUserContext.getCurrentUser(), e); - return Result.failParam("Search term input error, please check"); - } - } - - private void transformSearchResponse(SearchResponse searchResponse, final LogDTO logDTO, List keyList) { - SearchHit[] hits = searchResponse.getHits().getHits(); - if (hits == null || hits.length == 0) { - return; - } - List logDataList = Lists.newArrayList(); - for (SearchHit hit : hits) { - LogDataDTO logData = hit2DTO(hit, keyList); - // Package highlighted - logData.setHighlight(getHightlinghtMap(hit)); - logDataList.add(logData); - } - logDTO.setThisSortValue(hits[hits.length - 1].getSortValues()); - logDTO.setLogDataDTOList(logDataList); - } - - // highlight - private HighlightBuilder getHighlightBuilder(List keyList) { - HighlightBuilder highlightBuilder = new HighlightBuilder(); - for (String key : keyList) { - if (noHighLightSet.contains(key)) { - continue; - } - HighlightBuilder.Field highlightField = new HighlightBuilder.Field(key); - highlightBuilder.field(highlightField); - } - return highlightBuilder; - } - - @Override - public Result EsStatistic(LogQuery logQuery) { - try { - EsStatisticResult result = new EsStatisticResult(); - result.setName(constractEsStatisticRet(logQuery)); - MilogLogStoreDO logStore = logstoreDao.getByName(logQuery.getLogstore()); - if (logStore == null) { - return new Result<>(CommonError.UnknownError.getCode(), "not found logstore", null); - } - // get interval - String interval = searchLog.esHistogramInterval(logQuery.getEndTime() - logQuery.getStartTime()); - EsService esService = esCluster.getEsService(logStore.getEsClusterId()); - String esIndex = logStore.getEsIndex(); - if (esService == null || StringUtils.isEmpty(esIndex)) { - return Result.failParam("Log Store or tail configuration exceptions"); - } - if (!StringUtils.isEmpty(interval)) { - BoolQueryBuilder queryBuilder = searchLog.getQueryBuilder(logQuery, getKeyColonPrefix(logStore.getKeyList())); - EsClient.EsRet esRet = esService.dateHistogram(esIndex, interval, logQuery.getStartTime(), logQuery.getEndTime(), queryBuilder); - result.setCounts(esRet.getCounts()); - result.setTimestamps(esRet.getTimestamps()); - result.setQueryBuilder(queryBuilder); - result.calTotalCounts(); - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), result); - } else { - return new Result<>(CommonError.UnknownError.getCode(), "The minimum time interval is 10s", null); - } - } catch (ElasticsearchStatusException e) { - log.error("Log query errors and log bar chart statistics report errors:[{}], Error type[{}], logQuery:[{}], user:[{}]", e, e.status(), logQuery, MoneUserContext.getCurrentUser(), e); - return Result.failParam("If the permissions of ES resources are configured incorrectly, check the username and password or token"); - } catch (Exception e) { - log.error("Log query errors and log bar chart statistics report errors[{}],logQuery:[{}],user:[{}]", e, logQuery, MoneUserContext.getCurrentUser(), e); - return Result.failParam("Search term input error, please check"); - } - } - - private String constractEsStatisticRet(LogQuery logquery) { - StringBuilder sb = new StringBuilder(); - if (!StringUtils.isEmpty(logquery.getLogstore())) { - sb.append("logstore:").append(logquery.getLogstore()).append(";"); - } - if (!StringUtils.isEmpty(logquery.getFullTextSearch())) { - sb.append("fullTextSearch:").append(logquery.getFullTextSearch()).append(";"); - } - return sb.toString(); - } - - private String esHistogramInterval(Long duration) { - duration = duration / 1000; - if (duration > 24 * 60 * 60) { - duration = duration / 100; - return duration + "s"; - } else if (duration > 12 * 60 * 60) { - duration = duration / 80; - return duration + "s"; - } else if (duration > 6 * 60 * 60) { - duration = duration / 60; - return duration + "s"; - } else if (duration > 60 * 60) { - duration = duration / 50; - return duration + "s"; - } else if (duration > 30 * 60) { - duration = duration / 40; - return duration + "s"; - } else if (duration > 10 * 60) { - duration = duration / 30; - return duration + "s"; - } else if (duration > 5 * 60) { - duration = duration / 25; - return duration + "s"; - } else if (duration > 3 * 60) { - duration = duration / 20; - return duration + "s"; - } else if (duration > 60) { - duration = duration / 15; - return duration + "s"; - } else if (duration > 10) { - duration = duration / 10; - return duration + "s"; - } else { - return ""; - } - } - - /** - * Get trace logs - * - * @param logQuery - * @return - */ - @Override - public TraceLogDTO getTraceLog(TraceLogQuery logQuery) { - try { - log.info("getTraceLog,param data:{}", GSON.toJson(logQuery)); - return traceLog.getTraceLog(logQuery.getAppId(), logQuery.getTraceId(), "", logQuery.getGenerationTime(), logQuery.getLevel()); - } catch (Exception e) { - log.error("Log query error, query trace log error, logQuery:[{}]", e, GSON.toJson(logQuery), e); - return TraceLogDTO.emptyData(); - } - } - - /** - * Obtain trace logs in the data center - * - * @param regionTraceLogQuery - * @return - */ - @Override - public Result queryRegionTraceLog(RegionTraceLogQuery regionTraceLogQuery) throws IOException { - return Result.success(traceLog.getTraceLog(null, regionTraceLogQuery.getTraceId(), regionTraceLogQuery.getRegion(), "", "")); - } - - @Override - public Result getDocContext(LogContextQuery logContextQuery) { - SearchRequest searchRequest = null; - try { - if (searchLog.isLegalParam(logContextQuery) == false) { - return Result.failParam("Required parameters are missing"); - } - MilogLogStoreDO logStore = logstoreDao.getByName(logContextQuery.getLogstore()); - if (logStore.getEsClusterId() == null || StringUtils.isEmpty(logStore.getEsIndex())) { - return Result.failParam("Store configuration exception"); - } - EsService esService = esCluster.getEsService(logStore.getEsClusterId()); - String esIndexName = logStore.getEsIndex(); - List keyList = getKeyList(logStore.getKeyList(), logStore.getColumnTypeList()); - LogDTO dto = new LogDTO(); - List logDataList = new ArrayList<>(); - int times = 1, pageSize = logContextQuery.getPageSize(); - Long lineNumberSearchAfter = logContextQuery.getLineNumber(); - List logOrder = new ArrayList<>(); - logOrder.add(logContextQuery.getType()); - if (0 == logContextQuery.getType()) { - times = 2; - pageSize = pageSize / 2; - logOrder.remove(0); - logOrder.add(2); - logOrder.add(1); - } - for (int t = 0; t < times; t++) { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.filter(QueryBuilders.termQuery(LogParser.esKeyMap_logip, logContextQuery.getIp())); - boolQueryBuilder.filter(QueryBuilders.termQuery(LogParser.esKyeMap_fileName, logContextQuery.getFileName())); - SearchSourceBuilder builder = new SearchSourceBuilder(); - builder.query(boolQueryBuilder); - if (1 == logOrder.get(t)) { - // 1-after - builder.sort(LogParser.esKeyMap_timestamp, ASC); - builder.sort(LogParser.esKeyMap_lineNumber, ASC); - } else if (2 == logOrder.get(t)) { - // 2-before - builder.sort(LogParser.esKeyMap_timestamp, DESC); - builder.sort(LogParser.esKeyMap_lineNumber, DESC); - } - if (0 == logContextQuery.getType() && 2 == logOrder.get(t)) { - builder.searchAfter(new Object[]{logContextQuery.getTimestamp(), lineNumberSearchAfter + 1}); - } else { - builder.searchAfter(new Object[]{logContextQuery.getTimestamp(), lineNumberSearchAfter}); - } - builder.size(pageSize); - searchRequest = new SearchRequest(esIndexName); - searchRequest.source(builder); - SearchResponse searchResponse; - searchResponse = esService.search(searchRequest); - SearchHit[] hits = searchResponse.getHits().getHits(); - if (hits == null || hits.length == 0) { - continue; - } - if (1 == logOrder.get(t)) { - for (int i = 0; i < hits.length; i++) { - logDataList.add(this.hit2DTO(hits[i], keyList)); - } - } else if (2 == logOrder.get(t)) { - for (int i = hits.length - 1; i >= 0; i--) { - logDataList.add(this.hit2DTO(hits[i], keyList)); - } - } - } - dto.setLogDataDTOList(logDataList); - return Result.success(dto); - } catch (Exception e) { - log.error("Log query error and log context error:[{}], logContextQuery:[{}], searchRequest:[{}], user:[{}]", e, logContextQuery, searchRequest, MoneUserContext.getCurrentUser()); - return Result.failParam("System error, please try again"); - } - } - - private LogDataDTO hit2DTO(SearchHit hit, List keyList) { - LogDataDTO logData = new LogDataDTO(); - Map ferry = hit.getSourceAsMap(); - - long time = 0; - if (ferry.containsKey("time") && null != ferry.get("time") && StringUtils.isNotBlank(ferry.get("time").toString())) { - time = DateUtil.parse(ferry.get("time").toString()).toTimestamp().getTime(); - } - if (null == ferry.get(LogParser.esKeyMap_timestamp)) { - logData.setValue(LogParser.esKeyMap_timestamp, time); - } else { - logData.setValue(LogParser.esKeyMap_timestamp, ferry.get(LogParser.esKeyMap_timestamp)); - } - for (String key : keyList) { - if (!hidenFiledSet.contains(key)) { - logData.setValue(key, ferry.get(key)); - } - } - logData.setIp(ferry.get(LogParser.esKeyMap_logip) == null ? "" : String.valueOf(ferry.get(LogParser.esKeyMap_logip))); - logData.setFileName(ferry.get(LogParser.esKyeMap_fileName) == null ? "" : String.valueOf(ferry.get(LogParser.esKyeMap_fileName))); - logData.setLineNumber(ferry.get(LogParser.esKeyMap_lineNumber) == null ? "" : String.valueOf(ferry.get(LogParser.esKeyMap_lineNumber))); - logData.setTimestamp(ferry.get(LogParser.esKeyMap_timestamp) == null ? String.valueOf(time) : String.valueOf(ferry.get(LogParser.esKeyMap_timestamp))); - logData.setLogOfString(new Gson().toJson(logData.getLogOfKV())); - return logData; - } - - public void logExport(LogQuery logQuery) throws Exception { - // Generate Excel - int maxLogNum = 10000; - logQuery.setPageSize(maxLogNum); - Result logDTOResult = this.logQuery(logQuery); - List> exportData = - logDTOResult.getCode() != CommonError.Success.getCode() - || logDTOResult.getData().getLogDataDTOList() == null - || logDTOResult.getData().getLogDataDTOList().isEmpty() ? - null : logDTOResult.getData().getLogDataDTOList().stream().map(LogDataDTO::getLogOfKV).collect(Collectors.toList()); - HSSFWorkbook excel = ExportExcel.HSSFWorkbook4Map(exportData, generateTitle(logQuery)); - // download - String fileName = String.format("%s_log.xls", logQuery.getLogstore()); - searchLog.downLogFile(excel, fileName); - } - - private String generateTitle(LogQuery logQuery) { - return String.format("%s Logs, search terms:[%s],time range %d-%d", logQuery.getLogstore(), logQuery.getFullTextSearch() == null ? "" : logQuery.getFullTextSearch(), logQuery.getStartTime(), logQuery.getEndTime()); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogSearchSaveServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogSearchSaveServiceImpl.java deleted file mode 100644 index 8b2e87653..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogSearchSaveServiceImpl.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import com.xiaomi.mone.log.api.enums.FavouriteSearchEnum; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.common.context.MoneUserContext; -import com.xiaomi.mone.log.manager.mapper.MilogLogSearchSaveMapper; -import com.xiaomi.mone.log.manager.model.convert.SearchSaveConvert; -import com.xiaomi.mone.log.manager.model.dto.SearchSaveDTO; -import com.xiaomi.mone.log.manager.model.page.PageInfo; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogSearchSaveDO; -import com.xiaomi.mone.log.manager.model.vo.SearchSaveInsertCmd; -import com.xiaomi.mone.log.manager.model.vo.SearchSaveUpdateCmd; -import com.xiaomi.mone.log.manager.service.LogSearchSaveService; -import com.xiaomi.youpin.docean.anno.Service; - -import javax.annotation.Resource; -import java.util.List; - -/** - * - * @author wanghaoyang - * @since 2022-03-29 - */ -@Service -public class LogSearchSaveServiceImpl implements LogSearchSaveService { - - @Resource - private MilogLogSearchSaveMapper logSearchSaveMapper; - - public Result> list(Long storeId, Integer pageNum, Integer pageSize) { - Long count = logSearchSaveMapper.countByStoreId(storeId); - List list = logSearchSaveMapper.selectByStoreId(storeId, pageSize * (pageNum - 1), pageSize); - List searchSaveDTOList = SearchSaveConvert.INSTANCE.fromDOList(list); - PageInfo res = new PageInfo<>(pageNum, pageSize, count.intValue(), searchSaveDTOList); - return Result.success(res); - } - - public SearchSaveDTO getById(Long id) { - return SearchSaveConvert.INSTANCE.fromDO(logSearchSaveMapper.selectById(id)); - } - - public Result save(SearchSaveInsertCmd cmd) { - if (cmd.getSort() == null) { - return Result.failParam("The classification field SORT cannot be empty"); - } - switch (FavouriteSearchEnum.queryByCode(cmd.getSort())) { - case TEXT: - if (isRepeatName(cmd.getName())) { - return Result.failParam("Names cannot be duplicated"); - } - break; - case STORE: - Integer isMyFavouriteStore = logSearchSaveMapper.isMyFavouriteStore(MoneUserContext.getCurrentUser().getUser(), cmd.getStoreId()); - if (isMyFavouriteStore >= 1) { - return Result.failParam("Bookmarked"); - } - case TAIL: - Integer isMyFavouriteTail = logSearchSaveMapper.isMyFavouriteTail(MoneUserContext.getCurrentUser().getUser(), cmd.getTailId()); - if (isMyFavouriteTail >= 1) { - return Result.failParam("Bookmarked"); - } - break; - } - - MilogLogSearchSaveDO logSearchSaveDO = SearchSaveConvert.INSTANCE.toDO(cmd); - long current = System.currentTimeMillis(); - logSearchSaveDO.setCreateTime(current); - logSearchSaveDO.setUpdateTime(current); - String user = MoneUserContext.getCurrentUser().getUser(); - logSearchSaveDO.setCreator(user); - logSearchSaveDO.setUpdater(user); - Integer maxOrder = logSearchSaveMapper.getMaxOrder(user, cmd.getSort()); - logSearchSaveDO.setOrderNum(maxOrder == null ? 100 : maxOrder + 100); - int insert = logSearchSaveMapper.insert(logSearchSaveDO); - return Result.success(insert); - } - - public Result update(SearchSaveUpdateCmd cmd) { - MilogLogSearchSaveDO milogLogSearchSaveDO = logSearchSaveMapper.selectById(cmd.getId()); - if (milogLogSearchSaveDO == null) { - return Result.failParam("Data not found"); - } - if (!cmd.getName().equals(milogLogSearchSaveDO.getName()) && isRepeatName(cmd.getName())) { - return Result.failParam("Names cannot be duplicated"); - } - milogLogSearchSaveDO.setName(cmd.getName()); - milogLogSearchSaveDO.setQueryText(cmd.getQueryText()); - milogLogSearchSaveDO.setIsFixTime(cmd.getIsFixTime()); - milogLogSearchSaveDO.setStartTime(cmd.getStartTime()); - milogLogSearchSaveDO.setEndTime(cmd.getEndTime()); - milogLogSearchSaveDO.setCommon(cmd.getCommon()); - milogLogSearchSaveDO.setUpdateTime(System.currentTimeMillis()); - milogLogSearchSaveDO.setUpdater(MoneUserContext.getCurrentUser().getUser()); - int i = logSearchSaveMapper.updateById(milogLogSearchSaveDO); - return Result.success(i); - } - - public Result removeById(Long id) { - int i = logSearchSaveMapper.removeById(id); - return Result.success(i); - } - - private boolean isRepeatName(String name) { - Long count = logSearchSaveMapper.countByStoreAndName(name, MoneUserContext.getCurrentUser().getUser()); - return count >= 1; - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogSpaceServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogSpaceServiceImpl.java deleted file mode 100644 index 31d1fc7d6..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogSpaceServiceImpl.java +++ /dev/null @@ -1,342 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import cn.hutool.core.collection.CollectionUtil; -import com.alibaba.nacos.common.utils.CollectionUtils; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.api.enums.LogStructureEnum; -import com.xiaomi.mone.log.api.enums.MachineRegionEnum; -import com.xiaomi.mone.log.api.enums.OperateEnum; -import com.xiaomi.mone.log.api.enums.ProjectSourceEnum; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.exception.CommonError; -import com.xiaomi.mone.log.manager.common.context.MoneUserContext; -import com.xiaomi.mone.log.manager.common.exception.MilogManageException; -import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; -import com.xiaomi.mone.log.manager.dao.MilogSpaceDao; -import com.xiaomi.mone.log.manager.domain.Tpc; -import com.xiaomi.mone.log.manager.model.MilogSpaceParam; -import com.xiaomi.mone.log.manager.model.convert.MilogSpaceConvert; -import com.xiaomi.mone.log.manager.model.dto.MapDTO; -import com.xiaomi.mone.log.manager.model.dto.MilogSpaceDTO; -import com.xiaomi.mone.log.manager.model.page.PageInfo; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogSpaceDO; -import com.xiaomi.mone.log.manager.service.BaseService; -import com.xiaomi.mone.log.manager.service.LogSpaceService; -import com.xiaomi.mone.log.manager.user.MoneUser; -import com.xiaomi.mone.tpc.common.enums.NodeUserRelTypeEnum; -import com.xiaomi.mone.tpc.common.enums.UserTypeEnum; -import com.xiaomi.mone.tpc.common.vo.NodeVo; -import com.xiaomi.mone.tpc.common.vo.PageDataVo; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.common.StringUtils; -import com.xiaomi.youpin.docean.plugin.db.Transactional; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Resource; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -@Service -@Slf4j -public class LogSpaceServiceImpl extends BaseService implements LogSpaceService { - - @Resource - private MilogSpaceDao milogSpaceDao; - - @Resource - private MilogLogstoreDao milogLogstoreDao; - - @Resource - private TpcSpaceAuthService spaceAuthService; - - @Resource - private LogTailServiceImpl logTailService; - - @Resource - private Tpc tpc; - - private static final Cache>> SPACE_ALL_CACHE = CacheBuilder.newBuilder() - .maximumSize(100) - .expireAfterWrite(3, TimeUnit.MINUTES) - .build(); - - /** - * new - * - * @param param - * @return - */ - @Override - public Result newMilogSpace(MilogSpaceParam param) { - if (Objects.isNull(param) || StringUtils.isBlank(param.getSpaceName())) { - return Result.failParam("Parameter error"); - } - - String spaceName = param.getSpaceName(); - if (milogSpaceDao.verifyExistByName(spaceName)) { - return Result.failParam("There is a space name of the same name"); - } - - MilogSpaceDO milogSpaceDO = wrapMilogSpaceDO(param); - wrapBaseCommon(milogSpaceDO, OperateEnum.ADD_OPERATE); - - MilogSpaceDO dbDO = milogSpaceDao.newMilogSpace(milogSpaceDO); - if (Objects.isNull(dbDO.getId())) { - return Result.failParam("Space is not saved successfully, please try again"); - } - String creator = MoneUserContext.getCurrentUser().getUser(); - List otherAdmins = Lists.newArrayList(); - if (CollectionUtils.isNotEmpty(param.getAdmins())) { - creator = param.getAdmins().get(0); - if (param.getAdmins().size() > 1) { - otherAdmins = CollectionUtil.sub(param.getAdmins(), 1, param.getAdmins().size()); - } - } - com.xiaomi.youpin.infra.rpc.Result tpcResult = spaceAuthService.saveSpacePerm(dbDO, creator); - addMemberAsync(dbDO.getId(), otherAdmins); - - if (tpcResult == null || tpcResult.getCode() != 0) { - milogSpaceDao.deleteMilogSpace(dbDO.getId()); - log.error("The new space is not associated with a permission system,space:[{}], tpcResult:[{}]", dbDO, tpcResult); - return Result.failParam("Space is not associated with a permission system"); - } - - return Result.success(); - } - - private void addMemberAsync(Long spaceId, List otherAdmins) { - if (CollectionUtil.isNotEmpty(otherAdmins)) { - List> adminAsyncResult = otherAdmins.stream() - .map(admin -> CompletableFuture.runAsync(() -> - spaceAuthService.addSpaceMember(spaceId, admin, UserTypeEnum.CAS_TYPE.getCode(), NodeUserRelTypeEnum.MANAGER.getCode()))) - .toList(); - CompletableFuture.allOf(adminAsyncResult.toArray(new CompletableFuture[0])).join(); - } - } - - private MilogSpaceDO wrapMilogSpaceDO(MilogSpaceParam param) { - MilogSpaceDO milogSpaceDO = new MilogSpaceDO(); - milogSpaceDO.setSpaceName(param.getSpaceName()); - milogSpaceDO.setDescription(param.getDescription()); - return milogSpaceDO; - } - - /** - * getById - * - * @param id - * @return - */ - @Override - public Result getMilogSpaceById(Long id) { - if (null == id) { - return new Result<>(CommonError.ParamsError.getCode(), "ID cannot be empty"); - } - MilogSpaceDO milogSpace = milogSpaceDao.getMilogSpaceById(id); - if (null != milogSpace) { - MilogSpaceDTO milogSpaceDTO = new MilogSpaceDTO(); - milogSpaceDTO.setCreator(milogSpace.getCreator()); - milogSpaceDTO.setCtime(milogSpace.getCtime()); - milogSpaceDTO.setDescription(milogSpace.getDescription()); - milogSpaceDTO.setId(milogSpace.getId()); - milogSpaceDTO.setTenantId(milogSpace.getTenantId()); - milogSpaceDTO.setTenantName("todo"); - milogSpaceDTO.setUtime(milogSpace.getUtime()); - milogSpaceDTO.setSpaceName(milogSpace.getSpaceName()); - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), milogSpaceDTO); - } - return new Result<>(CommonError.NOT_EXISTS_DATA.getCode(), CommonError.NOT_EXISTS_DATA.getMessage()); - } - - /** - * Paginated queries - * - * @param spaceName - * @param page - * @param pageSize - * @return - */ - @Override - public Result> getMilogSpaceByPage(String spaceName, Long tenantId, Integer page, Integer pageSize) { - com.xiaomi.youpin.infra.rpc.Result> userPermSpacePage = spaceAuthService.getUserPermSpace(spaceName, page, pageSize); - PageInfo spaceDTOPageInfo = MilogSpaceConvert.INSTANCE.fromTpcPage(userPermSpacePage.getData()); - return Result.success(spaceDTOPageInfo); - } - - @Override - public Result>> getMilogSpaces(Long tenantId) { - int pageNum = 1; - int defaultPageSize = 300; - String defaultSpaceKey = buildCacheKey(tenantId); - List> cachedResult = SPACE_ALL_CACHE.getIfPresent(defaultSpaceKey); - - // return cached result if available - if (CollectionUtils.isNotEmpty(cachedResult)) { - return Result.success(cachedResult); - } - - List nodeVos = fetchAllNodeVos(pageNum, defaultPageSize); - - // transform nodeVos to MapDTO list - List> result = nodeVos.parallelStream() - .map(nodeVo -> new MapDTO<>(nodeVo.getNodeName(), nodeVo.getOutId())) - .collect(Collectors.toList()); - - // cache the result - SPACE_ALL_CACHE.put(defaultSpaceKey, result); - return Result.success(result); - } - - private static String buildCacheKey(Long tenantId) { - MoneUser currentUser = MoneUserContext.getCurrentUser(); - return String.format("%s-%s", (tenantId == null) ? "local-key" : tenantId.toString(), currentUser.getUser()); - } - - private List fetchAllNodeVos(int pageNum, int pageSize) { - List nodeVos = new ArrayList<>(); - - while (true) { - com.xiaomi.youpin.infra.rpc.Result> response = spaceAuthService.getUserPermSpace("", pageNum, pageSize); - - if (response.getCode() != 0) { - throw new MilogManageException("query space from tpc error"); - } - - PageDataVo pageData = response.getData(); - if (pageData == null || CollectionUtils.isEmpty(pageData.getList())) { - break; - } - - nodeVos.addAll(pageData.getList()); - - if (pageData.getList().size() < pageSize) { - break; - } - pageNum++; - } - - return nodeVos; - } - - /** - * update - * - * @param param - * @return - */ - @Transactional - @Override - public Result updateMilogSpace(MilogSpaceParam param) { - if (null == param || StringUtils.isBlank(param.getSpaceName())) { - return new Result<>(CommonError.ParamsError.getCode(), "Parameter error", ""); - } - - if (milogSpaceDao.verifyExistByName(param.getSpaceName(), param.getId())) { - return new Result<>(CommonError.UnknownError.getCode(), "There is a space name of the same name", ""); - } - - MilogSpaceDO milogSpace = milogSpaceDao.queryById(param.getId()); - if (null == milogSpace) { - return new Result<>(CommonError.ParamsError.getCode(), "logSpace does not exist", ""); - } - - if (Objects.equals(param.getSpaceName(), milogSpace.getSpaceName()) && - Objects.equals(param.getDescription(), milogSpace.getDescription())) { - return Result.success("the logSpace data has not changed"); - } - - if (!tpc.hasPerm(MoneUserContext.getCurrentUser(), param.getId())) { - return Result.fail(CommonError.UNAUTHORIZED); - } - wrapMilogSpace(milogSpace, param); - wrapBaseCommon(milogSpace, OperateEnum.UPDATE_OPERATE); - - if (milogSpaceDao.update(milogSpace)) { - com.xiaomi.youpin.infra.rpc.Result tpcResult = spaceAuthService.updateSpaceTpc(param, MoneUserContext.getCurrentUser().getUser()); - if (tpcResult == null || tpcResult.getCode() != 0) { - log.error("Modify the space permission system not associated with it,space:[{}], tpcResult:[{}]", milogSpace, tpcResult); - return Result.success("To modify the unassociated permission system of space, contact the server-side performance group"); - } - return Result.success(); - } else { - log.warn("[MilogSpaceService.updateMilogSpace] update MilogSpace err,spaceName:{},spaceId:{}", param.getSpaceName(), param.getId()); - return new Result<>(CommonError.UnknownError.getCode(), CommonError.UnknownError.getMessage(), ""); - } - } - - @Transactional - @Override - public Result deleteMilogSpace(Long id) { - if (null == id) { - return new Result<>(CommonError.ParamsError.getCode(), "ID cannot be empty", ""); - } - if (!tpc.hasPerm(MoneUserContext.getCurrentUser(), id)) { - return Result.fail(CommonError.UNAUTHORIZED); - } - MilogSpaceDO milogSpace = milogSpaceDao.getMilogSpaceById(id); - if (null == milogSpace) { - return new Result<>(CommonError.ParamsError.getCode(), "logSpace does not exist", ""); - } - List stores = milogLogstoreDao.getMilogLogstoreBySpaceId(id); - if (stores != null && stores.size() != 0) { - return new Result<>(CommonError.ParamsError.getCode(), "There is a store under this space and cannot be deleted", ""); - } - if (milogSpaceDao.deleteMilogSpace(id)) { - logTailService.deleteConfigRemote(id, id, MachineRegionEnum.CN_MACHINE.getEn(), LogStructureEnum.SPACE); - - com.xiaomi.youpin.infra.rpc.Result tpcResult = spaceAuthService.deleteSpaceTpc(id, MoneUserContext.getCurrentUser().getUser(), MoneUserContext.getCurrentUser().getUserType()); - if (tpcResult == null || tpcResult.getCode() != 0) { - log.error("Remove the space without associated permission system,space:[{}], tpcResult:[{}]", milogSpace, tpcResult); - return Result.failParam("To delete a space system that is not associated with it, contact the server performance group"); - } - return Result.success(); - } else { - log.warn("[MilogSpaceService.deleteMilogSpace] delete MilogSpace err,spaceId:{}", id); - return Result.failParam("Space deletion failed, please try again"); - } - } - - @Override - public Result setSpacePermission(Long spaceId, String permDeptIds) { - if (spaceId == null || StringUtils.isEmpty(permDeptIds)) { - return Result.fail(CommonError.ParamsError); - } - MilogSpaceDO space = milogSpaceDao.getMilogSpaceById(spaceId); - if (!permDeptIds.contains(space.getCreateDeptId())) { - return Result.fail(CommonError.ParamsError.getCode(), "The Create Department permission cannot be revoked"); - } - space.setPermDeptId(permDeptIds); - boolean update = milogSpaceDao.update(space); - return update ? Result.success() : Result.fail(CommonError.UnknownError); - } - - @Override - public MilogSpaceDO buildMiLogSpace(MilogSpaceParam cmd, String appCreator) { - MilogSpaceDO ms = new MilogSpaceDO(); - wrapMilogSpace(ms, cmd, ProjectSourceEnum.ONE_SOURCE.getSource()); - wrapBaseCommon(ms, OperateEnum.ADD_OPERATE, appCreator); - return ms; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogStoreServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogStoreServiceImpl.java deleted file mode 100644 index c91afd232..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogStoreServiceImpl.java +++ /dev/null @@ -1,338 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import cn.hutool.core.bean.BeanUtil; -import cn.hutool.core.lang.Pair; -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.api.enums.LogStructureEnum; -import com.xiaomi.mone.log.api.enums.LogTypeEnum; -import com.xiaomi.mone.log.api.enums.OperateEnum; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.exception.CommonError; -import com.xiaomi.mone.log.manager.common.context.MoneUserContext; -import com.xiaomi.mone.log.manager.common.validation.StoreValidation; -import com.xiaomi.mone.log.manager.dao.MilogLogTailDao; -import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; -import com.xiaomi.mone.log.manager.domain.EsIndexTemplate; -import com.xiaomi.mone.log.manager.domain.LogTail; -import com.xiaomi.mone.log.manager.mapper.MilogEsIndexMapper; -import com.xiaomi.mone.log.manager.model.convert.MilogLogstoreConvert; -import com.xiaomi.mone.log.manager.model.dto.EsInfoDTO; -import com.xiaomi.mone.log.manager.model.dto.LogStoreDTO; -import com.xiaomi.mone.log.manager.model.dto.MapDTO; -import com.xiaomi.mone.log.manager.model.dto.MenuDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogEsIndexDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.mone.log.manager.model.pojo.MilogMiddlewareConfig; -import com.xiaomi.mone.log.manager.model.vo.LogStoreParam; -import com.xiaomi.mone.log.manager.service.BaseService; -import com.xiaomi.mone.log.manager.service.LogStoreService; -import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionService; -import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionServiceFactory; -import com.xiaomi.mone.log.manager.service.extension.resource.ResourceExtensionService; -import com.xiaomi.mone.log.manager.service.extension.resource.ResourceExtensionServiceFactory; -import com.xiaomi.mone.log.manager.service.extension.store.StoreExtensionService; -import com.xiaomi.mone.log.manager.service.extension.store.StoreExtensionServiceFactory; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; -import java.util.*; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.SUCCESS_MESSAGE; -import static com.xiaomi.mone.log.manager.service.impl.EsDataServiceImpl.requiredFields; - -/** - * @author milog - */ -@Service -@Slf4j -public class LogStoreServiceImpl extends BaseService implements LogStoreService { - - @Resource - private MilogLogstoreDao logStoreDao; - - @Resource - private MilogLogTailDao milogLogtailDao; - - @Resource - private EsIndexTemplate esIndexTemplate; - - @Resource - private StoreValidation storeValidation; - - @Resource - private MilogMiddlewareConfigServiceImpl milogMiddlewareConfigService; - - @Resource - private MilogEsIndexMapper milogEsIndexMapper; - - @Resource - private LogTail logTail; - - @Resource - private LogTailServiceImpl logTailService; - - private StoreExtensionService storeExtensionService; - - private ResourceExtensionService resourceExtensionService; - - private CommonExtensionService commonExtensionService; - - /** - * init method - */ - public void init() { - storeExtensionService = StoreExtensionServiceFactory.getStoreExtensionService(); - resourceExtensionService = ResourceExtensionServiceFactory.getResourceExtensionService(); - commonExtensionService = CommonExtensionServiceFactory.getCommonExtensionService(); - } - - @Override - public Result newLogStore(LogStoreParam cmd) { - if (null != cmd.getId()) { - return updateLogStore(cmd); - } else { - return createLogStore(cmd); - } - } - - private Result createLogStore(LogStoreParam cmd) { - String errorInfos = storeValidation.logStoreParamValid(cmd); - if (StringUtils.isNotEmpty(errorInfos)) { - return Result.failParam(errorInfos); - } - if (!cmd.getNameSameStatus() && logStoreDao.verifyExistByName(cmd.getLogstoreName(), null, cmd.getSpaceId())) { - return new Result<>(CommonError.UnknownError.getCode(), "There is a store name with the same name", ""); - } - List logStoreDOS = logStoreDao.queryBySpaceIdNamed(cmd.getSpaceId(), cmd.getLogstoreName()); - if (CollectionUtils.isNotEmpty(logStoreDOS)) { - return Result.failParam("The store name is duplicated, please fill in the name again"); - } - MilogLogStoreDO storeDO = MilogLogstoreConvert.INSTANCE.fromCommand(cmd); - wrapBaseCommon(storeDO, OperateEnum.ADD_OPERATE); - // Bind resources - storeExtensionService.storeResourceBinding(storeDO, cmd, OperateEnum.ADD_OPERATE); - checkRequiredFieldExist(storeDO, cmd); - // storage - boolean res = logStoreDao.newMilogLogStore(storeDO); - if (res == true) { - storeExtensionService.postProcessing(storeDO, cmd, OperateEnum.ADD_OPERATE); - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage()); - } else { - log.warn("[MilogLogstoreService.newMilogLogstore] creator MilogLogstore err,logstoreName:{}", cmd.getLogstoreName()); - return new Result<>(CommonError.UnknownError.getCode(), CommonError.UnknownError.getMessage()); - } - - } - - private void checkRequiredFieldExist(MilogLogStoreDO storeDO, LogStoreParam cmd) { - String keyList = storeDO.getKeyList(); - String columnTypeList = storeDO.getColumnTypeList(); - for (Pair> requiredField : requiredFields) { - if (!keyList.contains(requiredField.getKey())) { - keyList = String.format("%s,%s:%s", keyList, requiredField.getKey(), requiredField.getValue().getValue()); - columnTypeList = String.format("%s,%s", columnTypeList, requiredField.getValue().getKey()); - } - } - storeDO.setKeyList(keyList); - storeDO.setColumnTypeList(columnTypeList); - cmd.setKeyList(keyList); - cmd.setColumnTypeList(columnTypeList); - } - - @Override - public MilogLogStoreDO buildLogStoreEsInfo(LogStoreParam storeParam, String creator) { - // get esIndex - EsInfoDTO esInfo = esIndexTemplate.getEsInfo(storeParam.getMachineRoom(), storeParam.getLogType()); - storeParam.setEsIndex(esInfo.getIndex()); - MilogLogStoreDO ml = MilogLogstoreConvert.INSTANCE.fromCommand(storeParam); - ml.setEsClusterId(esInfo.getClusterId()); - wrapBaseCommon(ml, OperateEnum.ADD_OPERATE, creator); - return ml; - } - - @Override - public Result getLogStoreById(Long id) { - if (null == id) { - return Result.failParam("id can not be empty"); - } - MilogLogStoreDO milogLogStoreDO = logStoreDao.queryById(id); - LogStoreDTO logStoreDTO = new LogStoreDTO(); - if (null != milogLogStoreDO) { - BeanUtil.copyProperties(milogLogStoreDO, logStoreDTO); - if (MoneUserContext.getCurrentUser().getIsAdmin()) { - logStoreDTO.setSelectCustomIndex(Boolean.TRUE); - } - logStoreDTO.setEsResourceId(milogLogStoreDO.getEsClusterId()); - logStoreDTO.setLogTypeName(LogTypeEnum.queryNameByType(milogLogStoreDO.getLogType())); - logStoreDTO.setMachineRoomName(commonExtensionService.getMachineRoomName(milogLogStoreDO.getMachineRoom())); - } - return Result.success(logStoreDTO); - } - - public Result>> getLogStoreBySpaceId(Long spaceId) { - List stores = logStoreDao.getMilogLogstoreBySpaceId(spaceId); - ArrayList> ret = new ArrayList<>(); - for (MilogLogStoreDO s : stores) { - ret.add(new MapDTO<>(s.getLogstoreName(), s.getId())); - } - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), ret); - } - - public Result> getLogStoreByPage(String logstoreName, Long spaceId, int page, int pagesize) { - Map ret = logStoreDao.getMilogLogstoreByPage(logstoreName, spaceId, page, pagesize); - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), ret); - } - - public Result> getAllLogStore() { - Map ret = logStoreDao.getAllMilogLogstore(MoneUserContext.getCurrentUser().getZone()); - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), ret); - } - - public Result> getLogStoreByIds(List ids) { - List ret = logStoreDao.getMilogLogstore(ids); - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), ret); - } - - public Result updateLogStore(LogStoreParam param) { - MilogLogStoreDO milogLogstoreDO = logStoreDao.queryById(param.getId()); - if (null == milogLogstoreDO) { - return new Result<>(CommonError.ParamsError.getCode(), "logstore not found "); - } - if (!param.getLogType().equals(milogLogstoreDO.getLogType()) && 0 != milogLogtailDao.getTailCount(param.getId())) { - return new Result<>(CommonError.ParamsError.getCode(), "Logtail has been created under logstore and does not allow you to change the log type"); - } - String errorInfos = storeValidation.logStoreParamValid(param); - if (StringUtils.isNotEmpty(errorInfos)) { - return Result.failParam(errorInfos); - } - if (logStoreDao.verifyExistByName(param.getLogstoreName(), param.getId(), param.getSpaceId())) { - return new Result(CommonError.UnknownError.getCode(), "There is a store name with the same name", ""); - } - - MilogLogStoreDO ml = MilogLogstoreConvert.INSTANCE.fromCommand(param); - ml.setEsClusterId(milogLogstoreDO.getEsClusterId()); - ml.setEsIndex(milogLogstoreDO.getEsIndex()); - ml.setCtime(milogLogstoreDO.getCtime()); - ml.setCreator(milogLogstoreDO.getCreator()); - // Select the corresponding index - storeExtensionService.storeResourceBinding(ml, param, OperateEnum.UPDATE_OPERATE); - wrapBaseCommon(ml, OperateEnum.UPDATE_OPERATE); - checkRequiredFieldExist(ml, param); - boolean updateRes = storeExtensionService.updateLogStore(ml); - if (updateRes && storeExtensionService.sendConfigSwitch(param)) { - storeExtensionService.postProcessing(ml, param, OperateEnum.UPDATE_OPERATE); - //Check to see if there is a tail and if there is a resend configuration information (nacos and agent) - logTail.handleStoreTail(milogLogstoreDO.getId()); - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage()); - } - return Result.success(SUCCESS_MESSAGE); - } - - public Result deleteLogStore(Long id) { - MilogLogStoreDO logStore = logStoreDao.queryById(id); - if (null == logStore) { - return new Result<>(CommonError.ParamsError.getCode(), "logstore not found"); - } - List tails = milogLogtailDao.getMilogLogtailByStoreId(id); - if (tails != null && tails.size() != 0) { - return new Result<>(CommonError.ParamsError.getCode(), "There is a tail under the log store and cannot be deleted"); - } - storeExtensionService.deleteStorePostProcessing(logStore); - if (logStoreDao.deleteMilogSpace(id)) { - storeExtensionService.postProcessing(logStore, null, OperateEnum.DELETE_OPERATE); - //Delete the configuration in NACOS - logTailService.deleteConfigRemote(logStore.getSpaceId(), id, logStore.getMachineRoom(), LogStructureEnum.STORE); - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage()); - } else { - log.warn("[MilogLogstoreService.deleteMilogLogstore] delete Milogstore err,spaceId:{}", id); - return new Result<>(CommonError.UnknownError.getCode(), CommonError.UnknownError.getMessage()); - } - } - - - public Result>> getStoreIps(Long storeId) { - if (storeId == null) { - return Result.failParam("The parameter cannot be empty"); - } - final List logtailDoList = milogLogtailDao.getMilogLogtailByStoreId(storeId); - List> res = new ArrayList<>(); - Map ferry; - Set existIpSet = new HashSet<>(); - for (MilogLogTailDo tail : logtailDoList) { - List ipList = tail.getIps(); - if (CollectionUtils.isNotEmpty(ipList)) { - for (String ip : ipList) { - if (existIpSet.contains(ip) || StringUtils.isEmpty(ip)) { - continue; - } - ferry = new HashMap<>(2); - ferry.put("label", ip); - ferry.put("value", ip); - res.add(ferry); - existIpSet.add(ip); - } - } - } - return Result.success(res); - } - - public Result>> queryDeptExIndexList(String regionCode, Integer logTypeCode) { - if (StringUtils.isBlank(regionCode) || null == logTypeCode) { - return Result.failParam("regionCode or logTypeCode can not empty"); - } - //Query the ES information under the department to which the current user belongs - List middlewareConfigEs = milogMiddlewareConfigService.getESConfigs(regionCode); - middlewareConfigEs = resourceExtensionService.currentUserConfigFilter(middlewareConfigEs); - List> menuDTOS = middlewareConfigEs.stream().map(config -> { - MenuDTO menuDTO = new MenuDTO<>(); - menuDTO.setKey(config.getId()); - menuDTO.setLabel(config.getAlias()); - menuDTO.setChildren(getExIndexByLogType(config.getId(), logTypeCode)); - return menuDTO; - }).collect(Collectors.toList()); - return Result.success(menuDTOS); - } - - private List> getExIndexByLogType(Long clusterId, Integer logTypeCode) { - QueryWrapper queryWrapper = new QueryWrapper<>().eq("cluster_id", clusterId).eq("log_type", logTypeCode); - List esIndexDOS = milogEsIndexMapper.selectList(queryWrapper); - if (CollectionUtils.isNotEmpty(esIndexDOS)) { - return esIndexDOS.stream().map(indexDO -> { - MenuDTO menuDTO = new MenuDTO<>(); - menuDTO.setKey(indexDO.getId()); - menuDTO.setLabel(indexDO.getIndexName()); - return menuDTO; - }).collect(Collectors.toList()); - } - return Lists.newArrayList(); - } - - public Result redistributeStoreConfig(Long storeId) { - if (null == storeId) { - return Result.failParam("storeId can not be empty"); - } - logTail.handleStoreTail(storeId); - return Result.success(SUCCESS_MESSAGE); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogTailServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogTailServiceImpl.java deleted file mode 100644 index a7b6cd048..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogTailServiceImpl.java +++ /dev/null @@ -1,906 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import cn.hutool.core.lang.Assert; -import com.google.common.collect.Lists; -import org.apache.ozhera.app.api.response.AppBaseInfo; -import org.apache.ozhera.app.model.vo.HeraEnvIpVo; -import com.xiaomi.mone.log.api.enums.*; -import com.xiaomi.mone.log.api.model.meta.FilterDefine; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.exception.CommonError; -import com.xiaomi.mone.log.manager.common.Utils; -import com.xiaomi.mone.log.manager.common.context.MoneUserContext; -import com.xiaomi.mone.log.manager.common.utils.ManagerUtil; -import com.xiaomi.mone.log.manager.common.validation.HeraConfigValid; -import com.xiaomi.mone.log.manager.dao.*; -import com.xiaomi.mone.log.manager.mapper.MilogLogSearchSaveMapper; -import com.xiaomi.mone.log.manager.mapper.MilogLogTemplateMapper; -import com.xiaomi.mone.log.manager.model.bo.LogTailParam; -import com.xiaomi.mone.log.manager.model.bo.MlogParseParam; -import com.xiaomi.mone.log.manager.model.dto.*; -import com.xiaomi.mone.log.manager.model.pojo.*; -import com.xiaomi.mone.log.manager.model.vo.QuickQueryVO; -import com.xiaomi.mone.log.manager.service.BaseService; -import com.xiaomi.mone.log.manager.service.LogTailService; -import com.xiaomi.mone.log.manager.service.bind.LogTypeProcessor; -import com.xiaomi.mone.log.manager.service.bind.LogTypeProcessorFactory; -import com.xiaomi.mone.log.manager.service.env.HeraEnvIpServiceFactory; -import com.xiaomi.mone.log.manager.service.extension.agent.MilogAgentService; -import com.xiaomi.mone.log.manager.service.extension.agent.MilogAgentServiceFactory; -import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionServiceFactory; -import com.xiaomi.mone.log.manager.service.extension.resource.ResourceExtensionService; -import com.xiaomi.mone.log.manager.service.extension.resource.ResourceExtensionServiceFactory; -import com.xiaomi.mone.log.manager.service.extension.store.StoreExtensionService; -import com.xiaomi.mone.log.manager.service.extension.store.StoreExtensionServiceFactory; -import com.xiaomi.mone.log.manager.service.extension.tail.TailExtensionService; -import com.xiaomi.mone.log.manager.service.extension.tail.TailExtensionServiceFactory; -import com.xiaomi.mone.log.manager.service.nacos.impl.StreamConfigNacosProvider; -import com.xiaomi.mone.log.parse.LogParser; -import com.xiaomi.mone.log.parse.LogParserFactory; -import com.xiaomi.mone.log.utils.IndexUtils; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.plugin.config.anno.Value; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; -import java.time.Instant; -import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.*; -import static com.xiaomi.mone.log.manager.common.Utils.getKeyValueList; - -@Slf4j -@Service -public class LogTailServiceImpl extends BaseService implements LogTailService { - - @Resource - private HeraEnvIpServiceFactory heraEnvIpServiceFactory; - - @Resource - private MilogLogTailDao milogLogtailDao; - - @Resource - private MilogLogstoreDao logStoreDao; - - @Resource - private MilogSpaceDao milogSpaceDao; - - @Resource - private MilogConfigNacosServiceImpl milogConfigNacosServiceImpl; - - @Resource - private MilogAppTopicRelDao milogAppTopicRelDao; - - @Resource - private MilogStreamServiceImpl milogStreamService; - - @Resource - private MilogAppMiddlewareRelServiceImpl milogAppMiddlewareRelService; - @Resource - private MilogAppMiddlewareRelDao milogAppMiddlewareRelDao; - @Resource - private MilogMiddlewareConfigDao milogMiddlewareConfigDao; - @Resource - private StreamConfigNacosProvider streamConfigNacosProvider; - @Value("$log_type_mq_not_consume") - private String logTypeMqNotConsume; - @Value("$dubbo.miline.rpc.env") - private String dubboMilineRpcEnv; - @Value("$server.type") - private String serverType; - @Resource - private HeraAppServiceImpl heraAppService; - @Resource - private HeraAppEnvServiceImpl heraAppEnvService; - @Resource - private MilogLogTemplateMapper milogLogTemplateMapper; - @Resource - private MilogLogSearchSaveMapper searchSaveMapper; - @Resource - private HeraConfigValid heraConfigValid; - @Resource - private LogTailServiceImpl logTailService; - - @Resource - private LogTypeProcessorFactory logTypeProcessorFactory; - - private LogTypeProcessor logTypeProcessor; - - private TailExtensionService tailExtensionService; - - private MilogAgentService milogAgentService; - - private StoreExtensionService storeExtensionService; - - private ResourceExtensionService resourceExtensionService; - - public void init() { - logTypeProcessorFactory.setMilogLogTemplateMapper(milogLogTemplateMapper); - logTypeProcessor = logTypeProcessorFactory.getLogTypeProcessor(); - tailExtensionService = TailExtensionServiceFactory.getTailExtensionService(); - milogAgentService = MilogAgentServiceFactory.getAgentExtensionService(); - storeExtensionService = StoreExtensionServiceFactory.getStoreExtensionService(); - resourceExtensionService = ResourceExtensionServiceFactory.getResourceExtensionService(); - } - - private static boolean filterNameEmpty(MilogLogTailDo milogLogTailDo) { - return StringUtils.isNotBlank(milogLogTailDo.getTail()); - } - - private QuickQueryVO applyQueryVO(MilogLogTailDo logTailDo) { - Integer isFavourite = searchSaveMapper.isMyFavouriteTail(MoneUserContext.getCurrentUser().getUser(), logTailDo.getId()); - return applyQueryVO(logTailDo, isFavourite); - } - - private QuickQueryVO applyQueryVO(MilogLogTailDo logTailDo, Integer isFavourite) { - return QuickQueryVO.builder() - .spaceId(logTailDo.getSpaceId()) - .storeId(logTailDo.getStoreId()) - .tailId(logTailDo.getId()) - .tailName(logTailDo.getTail()) - .envId(logTailDo.getEnvId()) - .envName(logTailDo.getEnvName()) - .isFavourite(isFavourite == null || isFavourite < 1 ? 0 : 1) - .deploySpace(logTailDo.getDeploySpace()) - .collectionReady(logTailDo.getCollectionReady()) - .build(); - } - - - private void handleMqTailParam(MilogLogStoreDO logStoreDO, LogTailParam param) { - param.setMiddlewareConfig(param.getMiddlewareConfig().stream().filter(Objects::nonNull).collect(Collectors.toList())); - if (CollectionUtils.isNotEmpty(param.getMiddlewareConfig()) && param.getMiddlewareConfig().size() == 3) { - param.setMiddlewareConfigId(((Double) param.getMiddlewareConfig().get(1)).longValue()); - param.setTopicName((String) param.getMiddlewareConfig().get(2)); - } else if (CollectionUtils.isNotEmpty(param.getMiddlewareConfig()) && param.getMiddlewareConfig().size() == 2) { - param.setMiddlewareConfigId(((Double) param.getMiddlewareConfig().get(1)).longValue()); - deleteMqRel(param.getMilogAppId(), param.getId()); - } else { - if (tailExtensionService.tailHandlePreprocessingSwitch(logStoreDO, param)) { - deleteMqRel(param.getMilogAppId(), param.getId()); - MilogMiddlewareConfig config; - if (null != logStoreDO.getMqResourceId()) { - config = milogMiddlewareConfigDao.queryById(logStoreDO.getMqResourceId()); - } else { - // Take the default - config = milogMiddlewareConfigDao.queryDefaultMiddlewareConfig(); - } - param.setMiddlewareConfigId(config.getId()); - } - } - } - - private void deleteMqRel(Long milogAppId, Long tailId) { - if (null != tailId) { - milogAppMiddlewareRelDao.deleteRel(milogAppId, tailId); - } - } - - @Override - public Result newMilogLogTail(LogTailParam param) { - // Parameter validation - String errorMsg = heraConfigValid.verifyLogTailParam(param); - if (StringUtils.isNotEmpty(errorMsg)) { - return new Result<>(CommonError.ParamsError.getCode(), errorMsg); - } - - MilogLogStoreDO logStore = logStoreDao.queryById(param.getStoreId()); - if (logStore == null) { - return new Result<>(CommonError.ParamsError.getCode(), "logStore not found"); - } - - String machineRoom = logStore.getMachineRoom(); - String tail = param.getTail(); - if (heraConfigValid.checkTailNameSame(tail, null, machineRoom, param.getSpaceId())) { - return new Result<>(CommonError.ParamsError.getCode(), "The alias is duplicated, please confirm and submit"); - } - - param.setValueList(IndexUtils.getNumberValueList(logStore.getKeyList(), param.getValueList())); - - // Parameter handling - handleMqTailParam(logStore, param); - - AppBaseInfo appBaseInfo = getAppBaseInfo(param); - - MilogLogTailDo mt = buildLogTailDo(param, logStore, appBaseInfo, MoneUserContext.getCurrentUser().getUser()); - MilogLogTailDo milogLogtailDo = milogLogtailDao.add(mt); - boolean supportedConsume = logTypeProcessor.supportedConsume(logStore.getLogType()); - try { - if (null != milogLogtailDo) { - if (tailExtensionService.bindMqResourceSwitch(logStore, param.getAppType())) { - // tail creates a successful binding and MQ relationship - tailExtensionService.defaultBindingAppTailConfigRel(milogLogtailDo.getId(), param.getMilogAppId(), null == param.getMiddlewareConfigId() ? logStore.getMqResourceId() : param.getMiddlewareConfigId(), param.getTopicName(), param.getBatchSendSize()); - /** Synchronize the information after creating a tail**/ - tailExtensionService.sendMessageOnCreate(param, mt, param.getMilogAppId(), supportedConsume); - } else if (tailExtensionService.bindPostProcessSwitch(param.getStoreId())) { - tailExtensionService.defaultBindingAppTailConfigRelPostProcess(milogLogtailDo.getSpaceId(), milogLogtailDo.getStoreId(), milogLogtailDo.getId(), milogLogtailDo.getMilogAppId(), logStore.getMqResourceId()); - /** Synchronize the information after creating a tail**/ - tailExtensionService.sendMessageOnCreate(param, mt, param.getMilogAppId(), supportedConsume); - } - LogTailDTO ret = new LogTailDTO(); - ret.setId(mt.getId()); - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), ret); - } else { - log.warn("[MilogLogtailService.newMilogLogtail] creator MilogLogtail err,milogAppId:{},logpath:{}", param.getMilogAppId(), param.getLogPath()); - return new Result<>(CommonError.UnknownError.getCode(), CommonError.UnknownError.getMessage()); - } - } catch (Exception e) { - log.warn("[MilogLogtailService.newMilogLogtail] creator MilogLogtail err,milogAppId:{},logpath:{}", param.getMilogAppId(), param.getLogPath(), e); - return new Result<>(CommonError.UnknownError.getCode(), e.getMessage()); - } - } - - @Override - public void sengMessageNewTail(LogTailParam param, MilogLogTailDo milogLogtailDo, MilogLogStoreDO milogLogStore) { - // tail creates a successful binding and middleware relationship - milogAppMiddlewareRelService.bindingTailConfigRel(milogLogtailDo.getId(), param.getMilogAppId(), param.getMiddlewareConfigId(), param.getTopicName()); - /** Synchronize the information after creating a tail**/ - tailExtensionService.sendMessageOnCreate(param, milogLogtailDo, param.getMilogAppId(), logTypeProcessor.supportedConsume(milogLogStore.getLogType())); - } - - @Override - public MilogLogTailDo buildLogTailDo(LogTailParam param, MilogLogStoreDO milogLogStore, AppBaseInfo appBaseInfo, String creator) { - MilogLogTailDo mt = logTailParam2Do(param, milogLogStore, appBaseInfo); - wrapBaseCommon(mt, OperateEnum.ADD_OPERATE, creator); - return mt; - } - - private boolean checkTailNameSame(String tailName, Long id, String machineRoom, Long spaceId) { - // Verify the log file with the same name - List logtailDoList = milogLogtailDao.queryTailNameExists(tailName, machineRoom, spaceId); - if (null == id) { - return CollectionUtils.isNotEmpty(logtailDoList); - } else { - if (CollectionUtils.isEmpty(logtailDoList)) { - return false; - } - MilogLogTailDo milogLogtailDo = logtailDoList.get(logtailDoList.size() - 1); - return !milogLogtailDo.getId().equals(id); - } - } - - @Override - public void sengMessageToAgent(Long milogAppId, MilogLogTailDo logtailDo) { - milogAgentService.publishIncrementConfig(logtailDo.getId(), milogAppId, logtailDo.getIps()); - } - - @Override - public void sengMessageToStream(MilogLogTailDo mt, Integer type) { - handleNaocsConfigByMotorRoom(mt, MachineRegionEnum.CN_MACHINE.getEn(), type, ProjectTypeEnum.MIONE_TYPE.getCode()); - } - - @Override - public void handleNaocsConfigByMotorRoom(MilogLogTailDo mt, String motorRoomEn, Integer type, Integer projectType) { - milogConfigNacosServiceImpl.chooseCurrentEnvNacosSerevice(motorRoomEn); - if (OperateEnum.ADD_OPERATE.getCode().equals(type) || OperateEnum.UPDATE_OPERATE.getCode().equals(type)) { - milogConfigNacosServiceImpl.publishStreamConfig(mt.getSpaceId(), type, projectType, motorRoomEn); - } - milogConfigNacosServiceImpl.publishNameSpaceConfig(motorRoomEn, mt.getSpaceId(), mt.getStoreId(), mt.getId(), type, ""); - } - - @Override - public boolean deleteConfigRemote(Long spaceId, Long id, String motorRoomEn, LogStructureEnum logStructureEnum) { - Assert.notNull(spaceId, "deleteConfigRemote spaceId can not null"); - Assert.notNull(id, "deleteConfigRemote id can not null"); - milogConfigNacosServiceImpl.chooseCurrentEnvNacosSerevice(motorRoomEn); - if (LogStructureEnum.SPACE == logStructureEnum) { - milogConfigNacosServiceImpl.publishStreamConfig(spaceId, OperateEnum.DELETE_OPERATE.getCode(), null, motorRoomEn); - return true; - } - if (LogStructureEnum.STORE == logStructureEnum) { - milogConfigNacosServiceImpl.publishNameSpaceConfig(motorRoomEn, spaceId, id, null, OperateEnum.DELETE_OPERATE.getCode(), logStructureEnum.getCode()); - return true; - } - MilogLogTailDo tailDo = milogLogtailDao.queryById(id); - milogConfigNacosServiceImpl.publishNameSpaceConfig(motorRoomEn, spaceId, tailDo.getStoreId(), id, OperateEnum.DELETE_OPERATE.getCode(), logStructureEnum.getCode()); - return true; - } - - @Override - public Result getMilogLogtailById(Long id) { - MilogLogTailDo tail = milogLogtailDao.queryById(id); - if (null != tail) { - // handle value list - MilogLogStoreDO logStore = logStoreDao.queryById(tail.getStoreId()); - if (null != logStore && StringUtils.isNotEmpty(logStore.getKeyList())) { - String keyList = logStore.getKeyList(); - String valueList = getKeyValueList(keyList, tail.getValueList()); - tail.setValueList(valueList); - } - // Handle filterconf to rateLimit - LogTailDTO logTailDTO = milogLogtailDO2DTO(tail); - if (tailExtensionService.decorateTailDTOValId(logStore.getLogType(), logTailDTO.getAppType().intValue())) { - decorateMilogTailDTO(logTailDTO); - } - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), logTailDTO); - } else { - return new Result<>(CommonError.UnknownError.getCode(), "tail not found"); - } - } - - private void decorateMilogTailDTO(LogTailDTO logTailDTO) { - Optional optionalAppBaseInfo = Optional.ofNullable(heraAppService.queryById(logTailDTO.getMilogAppId())); - optionalAppBaseInfo.ifPresent(appBaseInfo -> { - logTailDTO.setSource(appBaseInfo.getPlatformType().toString()); - List milogAppMiddlewareRels = milogAppMiddlewareRelDao.queryByCondition(logTailDTO.getMilogAppId(), null, logTailDTO.getId()); - if (CollectionUtils.isNotEmpty(milogAppMiddlewareRels)) { - MilogAppMiddlewareRel milogAppMiddlewareRel = milogAppMiddlewareRels.get(0); - MilogMiddlewareConfig config = milogMiddlewareConfigDao.queryById(milogAppMiddlewareRel.getMiddlewareId()); - logTailDTO.setMiddlewareConfig(Arrays.asList(Long.valueOf(config.getType()), config.getId(), milogAppMiddlewareRel.getConfig().getTopic())); - logTailDTO.setBatchSendSize(milogAppMiddlewareRel.getConfig().getBatchSendSize()); - } - }); - } - - @Override - public Result> getMilogLogBypage(Long storeId, int page, int pagesize) { - List ret = milogLogtailDao.getMilogLogtailByPage(storeId, page, pagesize); - ArrayList res = Lists.newArrayList(); - ret.forEach(v -> { - res.add(milogLogtailDO2DTO(v)); - }); - Map result = new HashMap<>(); - result.put("list", res); - result.put("total", milogLogtailDao.getTailCount(storeId)); - result.put("page", page); - result.put("pageSize", pagesize); - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), result); - } - - @Override - public Result> getLogTailCountByStoreId(Long storeId) { - Map result = new HashMap<>(); - result.put("count", milogLogtailDao.getTailCount(storeId)); - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), result); - } - - @Override - public Result> getMilogLogtailByIds(List ids) { - List ret = milogLogtailDao.getMilogLogtail(ids); - ArrayList res = Lists.newArrayList(); - ret.forEach(v -> { - res.add(milogLogtailDO2DTO(v)); - }); - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), res); - } - - @Override - public Result updateMilogLogTail(LogTailParam param) { - MilogLogTailDo ret = milogLogtailDao.queryById(param.getId()); - if (ret == null) { - return new Result<>(CommonError.ParamsError.getCode(), "tail does not exist"); - } - // Parameter validation - String errorMsg = heraConfigValid.verifyLogTailParam(param); - if (StringUtils.isNotEmpty(errorMsg)) { - return new Result<>(CommonError.ParamsError.getCode(), errorMsg); - } - - MilogLogStoreDO logStoreDO = logStoreDao.queryById(param.getStoreId()); - if (logStoreDO == null) { - return new Result<>(CommonError.ParamsError.getCode(), "Logstore does not exist"); - } - - // Process the MqTailParam parameter - handleMqTailParam(logStoreDO, param); - - // Check for duplicate aliases - String tail = param.getTail(); - Long id = param.getId(); - String machineRoom = logStoreDO.getMachineRoom(); - if (checkTailNameSame(tail, id, machineRoom, param.getSpaceId())) { - return new Result<>(CommonError.ParamsError.getCode(), "The alias is duplicated, please confirm and submit"); - } - - // handle value list - param.setValueList(IndexUtils.getNumberValueList(logStoreDO.getKeyList(), param.getValueList())); - AppBaseInfo appBaseInfo = getAppBaseInfo(param); - // tailRate to filterConf - FilterDefine filterDefine = FilterDefine.consRateLimitFilterDefine(param.getTailRate()); - List defines = new ArrayList<>(); - if (filterDefine != null) { - defines.add(filterDefine); - } - - MilogLogTailDo milogLogtailDo = logTailParam2Do(param, logStoreDO, appBaseInfo); - wrapBaseCommon(milogLogtailDo, OperateEnum.UPDATE_OPERATE); - boolean isSucceed = milogLogtailDao.update(milogLogtailDo); - - if (isSucceed) { - Integer appType = param.getAppType(); - boolean processSwitch = tailExtensionService.bindPostProcessSwitch(param.getStoreId()); - if (tailExtensionService.bindMqResourceSwitch(logStoreDO, appType) || processSwitch) { - if (null != param.getMiddlewareConfigId()) { - tailExtensionService.defaultBindingAppTailConfigRel(param.getId(), param.getMilogAppId(), param.getMiddlewareConfigId(), param.getTopicName(), param.getBatchSendSize()); - } - try { - List oldIps = ret.getIps(); - boolean supportedConsume = logTypeProcessor.supportedConsume(logStoreDO.getLogType()); - tailExtensionService.updateSendMsg(milogLogtailDo, oldIps, supportedConsume); - } catch (Exception e) { - log.error("update tail error", e); - } - } - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage()); - } else { - log.warn("[MilogLogtailService.updateMilogLogtail] update MilogLogtail err,id:{}", param.getId()); - return new Result<>(CommonError.UnknownError.getCode(), CommonError.UnknownError.getMessage()); - } - } - - private AppBaseInfo getAppBaseInfo(LogTailParam param) { - return heraAppService.queryById(param.getMilogAppId()); - } - - public void compareChangeDelIps(Long tailId, String logPath, List newIps, List oldIps) { - if (CollectionUtils.isEmpty(oldIps)) { - return; - } - List delIps = oldIps.stream().filter(s -> !newIps.contains(s)).collect(Collectors.toList()); - if (CollectionUtils.isNotEmpty(delIps)) { - milogAgentService.delLogCollDirectoryByIp(tailId, ManagerUtil.getPhysicsDirectory(logPath), delIps); - } - } - - @Override - public Result deleteLogTail(Long id) { - MilogLogTailDo milogLogtailDo = milogLogtailDao.queryById(id); - if (null == milogLogtailDo) { - return new Result<>(CommonError.ParamsError.getCode(), "tail does not exist"); - } - //Precondition check before deletion. - String validMsg = tailExtensionService.deleteCheckProcessPre(id); - if (StringUtils.isNotEmpty(validMsg)) { - return new Result<>(CommonError.ParamsError.getCode(), validMsg); - } - - MilogLogStoreDO logStoreDO = logStoreDao.queryById(milogLogtailDo.getStoreId()); - deleteRemoteConfig(id, logStoreDO); - - if (milogLogtailDao.deleteMilogLogtail(id)) { - if (storeExtensionService.isNeedSendMsgType(logStoreDO.getLogType())) { - CompletableFuture.runAsync(() -> sendMessageOnDelete(milogLogtailDo, logStoreDO)); - tailExtensionService.logTailDelPostProcess(logStoreDO, milogLogtailDo); - } - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage()); - } else { - log.warn("[LogTailService.deleteLogTail] delete LogTail error,id:{}", id); - return new Result<>(CommonError.UnknownError.getCode(), CommonError.UnknownError.getMessage()); - } - } - - private void deleteRemoteConfig(Long id, MilogLogStoreDO storeDO) { - if (logTypeProcessor.supportedConsume(storeDO.getLogType())) { - logTailService.deleteConfigRemote(storeDO.getSpaceId(), id, storeDO.getMachineRoom(), LogStructureEnum.TAIL); - } - } - - @Override - public void sendMessageOnDelete(MilogLogTailDo mt, MilogLogStoreDO logStoreDO) { - log.info("Send delete configuration information，mt:{}", GSON.toJson(mt)); - resourceExtensionService.deleteMqResourceProcessing(mt, logStoreDO); - /** - * Send configuration information ---log-agent - */ - milogAgentService.publishIncrementDel(mt.getId(), mt.getMilogAppId(), mt.getIps()); - /** - * Delete the relationship - */ - milogAppMiddlewareRelDao.deleteRel(mt.getMilogAppId(), mt.getId()); - } - - @Override - public Result> getAppInfoByName(String appName, Integer type) { - return Result.success(queryAllApps(appName, type)); - } - - /** - * Only 200 are returned in full - * - * @param appName - * @param type - * @return - */ - private List queryAllApps(String appName, Integer type) { - List mapDTOS = queryAppInfo(appName, type); -// if (StringUtils.isEmpty(appName)) { -// return mapDTOS.stream().limit(200).collect(Collectors.toList()); -// } - return mapDTOS; - } - - private List queryAppInfo(String appName, Integer type) { - List apps = heraAppService.queryAppInfoWithLog(appName, type); - List mapDTOList = Lists.newArrayList(); - if (CollectionUtils.isNotEmpty(apps)) { - mapDTOList = apps.stream().map(response -> { - MapDTO mapDTO = new MapDTO(); - mapDTO.setLabel(String.format("%s_%s", response.getPlatformName(), response.getAppName())); - mapDTO.setValue(response.getId()); - mapDTO.setKey(response.getBindId()); - return mapDTO; - }).collect(Collectors.toList()); - } - return mapDTOList; - } - - /** - * If the application is milog-agent, get a list of all machines through an additional interface - * - * @param milogAppId - * @param deployWay - * @return - */ - @Override - public Result> getEnInfosByAppId(Long milogAppId, Integer deployWay, String machineRoom) { - if (null == milogAppId) { - return Result.failParam("The parameter cannot be empty"); - } - AppBaseInfo appBaseInfo = heraAppService.queryById(milogAppId); - if (null == appBaseInfo) { - return Result.failParam("The app does not exist"); - } - List appEnvDTOList = tailExtensionService.getEnInfosByAppId(appBaseInfo, milogAppId, deployWay, machineRoom); - return Result.success(appEnvDTOList); - } - - @Override - public Result> getTailNamesBystoreId(String tail, Integer appType, Long id) { - List milogLogTailDos = milogLogtailDao.getMilogLogtailByIdsAndName(new ArrayList() {{ - add(id); - }}, tail, appType); - List ret = Lists.newArrayList(); - if (CollectionUtils.isNotEmpty(milogLogTailDos)) { - ret = milogLogTailDos.stream().filter(LogTailServiceImpl::filterNameEmpty).map(MilogLogTailDo::getTail).distinct().collect(Collectors.toList()); - } - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), ret); - } - - @Override - public Result>> tailRatelimit() { - ArrayList> ret = new ArrayList<>(); - ret.add(MapDTO.Of("快速收集-尽快收集，占用一定资源", RateLimitEnum.RATE_LIMIT_FAST.getRateLimit())); - ret.add(MapDTO.Of("常规收集", RateLimitEnum.RATE_LIMIT_MEDIUM.getRateLimit())); - ret.add(MapDTO.Of("慢速收集", RateLimitEnum.RATE_LIMIT_SLOW.getRateLimit())); - ret.add(MapDTO.Of("停止收集", RateLimitEnum.RATE_LIMIT_NONE.getRateLimit())); - return Result.success(ret); - } - - /*** - * miline Dynamic scaling and contracting - * @param projectInfo - */ - @Override - public void dockerScaleDynamic(DockerScaleBo projectInfo) { - List milogLogtailDos = milogLogtailDao.queryByAppAndEnv(projectInfo.getProjectId(), projectInfo.getEnvId()); - if (CollectionUtils.isNotEmpty(projectInfo.getIps()) && CollectionUtils.isNotEmpty(milogLogtailDos)) { - log.info("Dynamically expand the configuration in the current environment,projectId:{},envId:{},config:{}", projectInfo.getProjectId(), projectInfo.getEnvId(), GSON.toJson(milogLogtailDos)); - for (MilogLogTailDo milogLogtailDo : milogLogtailDos) { - List exitIps = milogLogtailDo.getIps(); - List newIps = projectInfo.getIps(); - if (!CollectionUtils.isEqualCollection(exitIps, newIps)) { - //1.update config - milogLogtailDo.setIps(newIps); - milogLogtailDao.update(milogLogtailDo); - //2.send msg - compareIpToHandle(milogLogtailDo.getId(), milogLogtailDo.getLogPath(), exitIps, newIps); - } - } - } - } - - public void compareIpToHandle(Long tailId, String logPath, List exitIps, List newIps) { - if (null == milogAgentService) { - init(); - } - List expandIps = newIps.stream().filter(ip -> !exitIps.contains(ip)).collect(Collectors.toList()); - if (CollectionUtils.isNotEmpty(expandIps)) { - // Scaling --- synchronous configuration - expandIps.forEach(ip -> { - milogAgentService.configIssueAgent("", ip, ""); - milogStreamService.configIssueStream(ip); - }); - } - List stopFileCollIps = exitIps.stream().filter(ip -> !newIps.contains(ip)).collect(Collectors.toList()); - if (CollectionUtils.isNotEmpty(stopFileCollIps)) { - milogAgentService.delLogCollDirectoryByIp(tailId, ManagerUtil.getPhysicsDirectory(logPath), stopFileCollIps); - } - } - - private MilogLogTailDo logTailParam2Do(LogTailParam logTailParam, MilogLogStoreDO logStoreDO, AppBaseInfo appBaseInfo) { - MilogLogTailDo milogLogtailDo = new MilogLogTailDo(); - milogLogtailDo.setId(logTailParam.getId()); - milogLogtailDo.setTail(logTailParam.getTail()); - milogLogtailDo.setSpaceId(logTailParam.getSpaceId()); - milogLogtailDo.setStoreId(logTailParam.getStoreId()); - milogLogtailDo.setMilogAppId(logTailParam.getMilogAppId()); - milogLogtailDo.setAppId(logTailParam.getMilogAppId()); - milogLogtailDo.setAppName(logTailParam.getAppName()); - if (null != appBaseInfo) { - milogLogtailDo.setMilogAppId(logTailParam.getMilogAppId()); - milogLogtailDo.setAppId(Long.valueOf(appBaseInfo.getBindId())); - milogLogtailDo.setAppName(appBaseInfo.getAppName()); - } - milogLogtailDo.setEnvId(logTailParam.getEnvId()); - milogLogtailDo.setEnvName(logTailParam.getEnvName()); - milogLogtailDo.setMachineType(logTailParam.getMachineType()); - Integer appType = logTailParam.getAppType(); - milogLogtailDo.setAppType(appType); - milogLogtailDo.setParseType(logTailParam.getParseType()); - milogLogtailDo.setParseScript(StringUtils.isEmpty(logTailParam.getParseScript()) ? DEFAULT_TAIL_SEPARATOR : logTailParam.getParseScript()); - milogLogtailDo.setLogPath(logTailParam.getLogPath().trim()); - milogLogtailDo.setLogSplitExpress((StringUtils.isNotEmpty(logTailParam.getLogSplitExpress()) ? logTailParam.getLogSplitExpress().trim() : "")); - milogLogtailDo.setValueList(logTailParam.getValueList()); - FilterDefine filterDefine = FilterDefine.consRateLimitFilterDefine(logTailParam.getTailRate()); - if (filterDefine != null) { - milogLogtailDo.setFilter(Arrays.asList(filterDefine)); - } - tailExtensionService.logTailDoExtraFiled(milogLogtailDo, logStoreDO, logTailParam); - milogLogtailDo.setDeployWay(logTailParam.getDeployWay()); - if (logStoreDO.isPlatformResourceStore()) { - milogLogtailDo.setDeploySpace((StringUtils.isNotEmpty(logTailParam.getDeploySpace()) ? logTailParam.getDeploySpace().trim() : "")); - } - milogLogtailDo.setFirstLineReg((StringUtils.isNotEmpty(logTailParam.getFirstLineReg()) ? logTailParam.getFirstLineReg() : "")); - milogLogtailDo.setCollectionReady(logTailParam.getCollectionReady()); - return milogLogtailDo; - } - - @Override - public LogTailDTO milogLogtailDO2DTO(MilogLogTailDo milogLogtailDo) { - if (milogLogtailDo == null) { - return null; - } - LogTailDTO logTailDTO = new LogTailDTO(); - - logTailDTO.setId(milogLogtailDo.getId()); - logTailDTO.setCtime(milogLogtailDo.getCtime()); - logTailDTO.setUtime(milogLogtailDo.getUtime()); - logTailDTO.setSpaceId(milogLogtailDo.getSpaceId()); - logTailDTO.setStoreId(milogLogtailDo.getStoreId()); - logTailDTO.setMilogAppId(milogLogtailDo.getMilogAppId()); - logTailDTO.setAppId(milogLogtailDo.getAppId()); - logTailDTO.setAppName(milogLogtailDo.getAppName()); - logTailDTO.setEnvId(milogLogtailDo.getEnvId()); - logTailDTO.setEnvName(milogLogtailDo.getEnvName()); - List list = milogLogtailDo.getIps(); - if (CollectionUtils.isNotEmpty(list)) { - logTailDTO.setIps(list); - } - logTailDTO.setTail(milogLogtailDo.getTail()); - logTailDTO.setParseType(milogLogtailDo.getParseType()); - logTailDTO.setParseScript(milogLogtailDo.getParseScript()); - logTailDTO.setLogPath(milogLogtailDo.getLogPath()); - logTailDTO.setLogSplitExpress(milogLogtailDo.getLogSplitExpress()); - logTailDTO.setValueList(milogLogtailDo.getValueList()); - logTailDTO.setAppType(milogLogtailDo.getAppType()); - logTailDTO.setMachineType(milogLogtailDo.getMachineType()); - logTailDTO.setMotorRooms(milogLogtailDo.getMotorRooms()); - // filterConf to tailRate - logTailDTO.setTailRate(RateLimitEnum.consTailRate(milogLogtailDo.getFilter())); - logTailDTO.setDeployWay(milogLogtailDo.getDeployWay()); - logTailDTO.setDeploySpace(milogLogtailDo.getDeploySpace()); - logTailDTO.setFirstLineReg(milogLogtailDo.getFirstLineReg()); - logTailDTO.setCollectionReady(milogLogtailDo.getCollectionReady()); - return logTailDTO; - } - - @Override - public Result> queryAppByStoreId(Long storeId) { - List milogLogtailDos = milogLogtailDao.queryAppIdByStoreId(storeId); - if (CollectionUtils.isNotEmpty(milogLogtailDos)) { - return Result.success(milogLogtailDos.stream().map(milogLogtailDo -> MapDTO.Of(milogLogtailDo.getAppName(), milogLogtailDo.getMilogAppId())).distinct().collect(Collectors.toList())); - } - return Result.success(Lists.newArrayList()); - } - - @Override - public Result> queryAppTailByStoreId(Long storeId) { - if (null == storeId) { - return Result.failParam("The storeId cannot be empty"); - } - List appTypeTailDTOS = Lists.newArrayList(); - List milogLogtailDos = milogLogtailDao.queryAppIdByStoreId(storeId); - if (CollectionUtils.isNotEmpty(milogLogtailDos)) { - Map> appTypeMap = milogLogtailDos.stream().collect(Collectors.groupingBy(MilogLogTailDo::getAppType)); - MilogLogStoreDO milogLogstoreDO = logStoreDao.queryById(storeId); - String nameEn = milogLogstoreDO.getMachineRoom(); - - for (Map.Entry> listEntry : appTypeMap.entrySet()) { - AppTypeTailDTO appTypeTailDTO = new AppTypeTailDTO(); - appTypeTailDTO.setAppType(listEntry.getKey()); - appTypeTailDTO.setAppTypName(ProjectTypeEnum.queryTypeByCode(listEntry.getKey())); - List tailAppList = Lists.newArrayList(); - AppTypeTailDTO.TailApp tailApp = new AppTypeTailDTO.TailApp(); - tailApp.setNameEn(nameEn); - tailApp.setNameCn(CommonExtensionServiceFactory.getCommonExtensionService().getMachineRoomName(nameEn)); - List logtailDos = listEntry.getValue(); - if (CollectionUtils.isNotEmpty(logtailDos)) { - List tailInfos = logtailDos.stream().map(milogLogtailDo -> { - AppTypeTailDTO.TailInfo tailInfo = new AppTypeTailDTO.TailInfo(); - tailInfo.setId(milogLogtailDo.getId()); - tailInfo.setTailName(milogLogtailDo.getTail()); - return tailInfo; - }).collect(Collectors.toList()); - tailApp.setTailInfos(tailInfos); - } - tailAppList.add(tailApp); - appTypeTailDTO.setTailAppList(tailAppList); - appTypeTailDTOS.add(appTypeTailDTO); - } - } - return Result.success(appTypeTailDTOS); - } - - @Override - public Result> queryLogStoreByRegionEn(String nameEn) { - if (StringUtils.isEmpty(nameEn)) { - return Result.failParam("storeId Cannot be empty"); - } - List milogLogtailDos = milogLogtailDao.queryStoreIdByRegionNameEN(nameEn); - if (CollectionUtils.isNotEmpty(milogLogtailDos)) { - return Result.success(milogLogtailDos.stream().map(milogLogtailDo -> logStoreDao.queryById(milogLogtailDo.getStoreId())).collect(Collectors.collectingAndThen(Collectors.toCollection(() -> new TreeSet<>(Comparator.comparing(o -> o.getLogstoreName()))), ArrayList::new))); - } - return Result.success(Lists.newArrayList()); - } - - @Override - public Result> getTailByStoreId(Long storeId) { - if (storeId == null) { - return Result.failParam("storeId Cannot be empty"); - } - List tailList = milogLogtailDao.getMilogLogtailByStoreId(storeId); - List dtoList = new ArrayList<>(); - LogTailDTO logTailDTO; - for (MilogLogTailDo milogLogtailDo : tailList) { - logTailDTO = milogLogtailDO2DTO(milogLogtailDo); - if (milogLogtailDo != null) { - dtoList.add(logTailDTO); - } - } - return Result.success(dtoList); - } - - @Override - public Result parseScriptTest(MlogParseParam mlogParseParam) { - String checkMsg = checkParseParam(mlogParseParam); - if (StringUtils.isNotEmpty(checkMsg)) { - return Result.failParam(checkMsg); - } - MilogLogStoreDO logstoreDO = logStoreDao.queryById(mlogParseParam.getStoreId()); - if (null == logstoreDO) { - return Result.fail(CommonError.NOT_EXISTS_DATA.getCode(), "logStore does not exist"); - } - String keyList = Utils.parse2KeyAndTypeList(logstoreDO.getKeyList(), logstoreDO.getColumnTypeList()); - String valueList = IndexUtils.getNumberValueList(logstoreDO.getKeyList(), mlogParseParam.getValueList()); - Long currentStamp = Instant.now().toEpochMilli(); - try { - LogParser logParser = LogParserFactory.getLogParser(mlogParseParam.getParseType(), keyList, valueList, mlogParseParam.getParseScript()); - Map parseMsg = logParser.parseSimple(mlogParseParam.getMsg(), currentStamp); - return Result.success(parseMsg); - } catch (Exception e) { - log.info("Parsing configuration information errors: Configuration information:{}", GSON.toJson(mlogParseParam), e); - } - return Result.success("Resolution error Please check the configuration information"); - } - - public Result parseExample(MlogParseParam mlogParseParam) { - String checkMsg = heraConfigValid.checkParseExampleParam(mlogParseParam); - if (StringUtils.isNotEmpty(checkMsg)) { - return Result.failParam(checkMsg); - } - try { - LogParser logParser = LogParserFactory.getLogParser(mlogParseParam.getParseType(), "", "", mlogParseParam.getParseScript()); - List parsedLog = logParser.parseLogData(mlogParseParam.getMsg()); - return Result.success(parsedLog); - } catch (Exception e) { - log.info("Parsing configuration information errors: Configuration information:{}", GSON.toJson(mlogParseParam), e); - } - return Result.success("Resolution error Please check the configuration information"); - } - - private String checkParseParam(MlogParseParam mlogParseParam) { - StringBuilder sb = new StringBuilder(); - if (null == mlogParseParam.getStoreId()) { - sb.append("store cannot be empty;"); - } - if (null == mlogParseParam.getParseScript()) { - sb.append("The parsing script cannot be empty;"); - } - if (null == mlogParseParam.getValueList()) { - sb.append("Indexing rules cannot be empty;"); - } - if (null == mlogParseParam.getMsg()) { - sb.append("Log information cannot be empty;"); - } - return sb.toString(); - } - - @Override - public Result> quickQueryByApp(Long milogAppId) { - if (null == milogAppId) { - return Result.failParam("The milog App ID cannot be empty"); - } - List milogLogTailDos = milogLogtailDao.getLogTailByMilogAppId(milogAppId); - List quickQueryVOS = Lists.newArrayList(); - if (CollectionUtils.isNotEmpty(milogLogTailDos)) { - quickQueryVOS = milogLogTailDos.stream().map(this::applyQueryVO).collect(Collectors.toList()); - wrapStoreSpaceName(quickQueryVOS); - } - return Result.success(quickQueryVOS); - } - - private void wrapStoreSpaceName(List quickQueryVOS) { - List storeIds = quickQueryVOS.stream().map(QuickQueryVO::getStoreId).collect(Collectors.toList()); - List milogLogStoreDOS = logStoreDao.queryByIds(storeIds); - List spaceIds = quickQueryVOS.stream().map(QuickQueryVO::getSpaceId).collect(Collectors.toList()); - List milogSpaceDOS = milogSpaceDao.queryByIds(spaceIds); - quickQueryVOS.stream().map(quickQueryVO -> { - milogLogStoreDOS.stream().filter(milogLogsSoreDO -> Objects.equals(quickQueryVO.getStoreId(), milogLogsSoreDO.getId())).findFirst().map(milogLogStoreDO -> { - quickQueryVO.setStoreName(milogLogStoreDO.getLogstoreName()); - return true; - }); - milogSpaceDOS.stream().filter(milogSpaceDO -> Objects.equals(quickQueryVO.getSpaceId(), milogSpaceDO.getId())).findFirst().map(milogSpaceDO -> { - quickQueryVO.setSpaceName(milogSpaceDO.getSpaceName()); - return true; - }); - return quickQueryVO; - }).collect(Collectors.toList()); - } - - @Override - public void machineIpChange(HeraEnvIpVo heraEnvIpVo) { - List logTailDos = milogLogtailDao.queryByMilogAppAndEnvId(heraEnvIpVo.getHeraAppId(), heraEnvIpVo.getId()); - if (CollectionUtils.isNotEmpty(logTailDos)) { - log.info("Dynamically expand the configuration in the current environment，heraAppEnvVo:{}，logTailDos:{}", GSON.toJson(heraEnvIpVo), GSON.toJson(logTailDos)); - for (MilogLogTailDo milogLogtailDo : logTailDos) { - List exitIps = milogLogtailDo.getIps(); - List newIps = heraEnvIpVo.getIpList(); - if (!CollectionUtils.isEqualCollection(exitIps, newIps)) { - //1.Modify the configuration - milogLogtailDo.setIps(newIps); - milogLogtailDo.setUtime(Instant.now().toEpochMilli()); - milogLogtailDo.setUpdater(DEFAULT_JOB_OPERATOR); - milogLogtailDao.updateIps(milogLogtailDo); - //2.Send a message - compareIpToHandle(milogLogtailDo.getId(), milogLogtailDo.getLogPath(), exitIps, newIps); - } - } - } - } - - @Override - public Result queryAppStore(Long appId, Integer platFormCode) { - if (null == appId || null == platFormCode) { - return Result.failParam("The parameter cannot be empty"); - } - AppBaseInfo appBaseInfo = heraAppService.queryByAppIdPlatFormType(appId.toString(), platFormCode); - if (null == appBaseInfo) { - log.info("queryAppStore app not exist,milogAppId:{},platFormCode:{}", appId, platFormCode); - return Result.success(new QuickQueryVO()); - } - List logTailDos = milogLogtailDao.queryByAppId(appId, appBaseInfo.getId().longValue()); - if (CollectionUtils.isEmpty(logTailDos)) { - return Result.success(new QuickQueryVO()); - } - return Result.success(applyQueryVO(logTailDos.get(logTailDos.size() - 1), null)); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogTemplateServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogTemplateServiceImpl.java deleted file mode 100644 index bc6156a41..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogTemplateServiceImpl.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; -import com.xiaomi.mone.log.api.enums.LogTypeEnum; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.mapper.MilogLogTemplateDetailMapper; -import com.xiaomi.mone.log.manager.mapper.MilogLogTemplateMapper; -import com.xiaomi.mone.log.manager.model.convert.MilogLogTemplateConvert; -import com.xiaomi.mone.log.manager.model.convert.MilogLongTemplateDetailConvert; -import com.xiaomi.mone.log.manager.model.dto.LogTemplateDTO; -import com.xiaomi.mone.log.manager.model.dto.LogTemplateDetailDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTemplateDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTemplateDetailDO; -import com.xiaomi.mone.log.manager.service.LogTemplateService; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; -import java.util.List; - -@Service -@Slf4j -public class LogTemplateServiceImpl extends ServiceImpl implements LogTemplateService { - - @Resource - private MilogLogTemplateMapper milogLogTemplateMapper; - - @Resource - private MilogLogTemplateDetailMapper templateDetailMapper; - - /** - * A list of log templates - * - * @return - */ - @Override - public Result> getLogTemplateList(String area) { - List logTemplateDOList = milogLogTemplateMapper.selectSupportedTemplate(area); - List logTemplateDTOList = MilogLogTemplateConvert.INSTANCE.fromDOList(logTemplateDOList); - assembleLogTemplateDetail(logTemplateDTOList); - return Result.success(logTemplateDTOList); - } - - private void assembleLogTemplateDetail(List logTemplateDTOList) { - if (CollectionUtils.isNotEmpty(logTemplateDTOList)) { - logTemplateDTOList.forEach(logTemplateDTO -> { - logTemplateDTO.setLogTemplateDetailDTOList(getLogTemplateById(logTemplateDTO.getValue()).getData()); - LogTypeEnum logTypeEnum = LogTypeEnum.type2enum(logTemplateDTO.getType()); - logTemplateDTO.setDescribe(null != logTypeEnum ? logTypeEnum.getDescribe() : StringUtils.EMPTY); - }); - } - } - - - /** - * Get the log template - * - * @param logTemplateId - * @return - */ - @Override - public Result getLogTemplateById(long logTemplateId) { - MilogLogTemplateDO logTemplate = milogLogTemplateMapper.selectById(logTemplateId); - MilogLogTemplateDetailDO tmplateDetail = templateDetailMapper.getByTemplateId(logTemplateId); - if (tmplateDetail == null) { - return Result.success(null); - } - LogTemplateDetailDTO dto = MilogLongTemplateDetailConvert.INSTANCE.fromDO(logTemplate, tmplateDetail); - return Result.success(dto); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MiLogMetaManageServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MiLogMetaManageServiceImpl.java deleted file mode 100644 index 87ccbb0e0..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MiLogMetaManageServiceImpl.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.api.model.meta.AppLogMeta; -import com.xiaomi.mone.log.api.model.meta.LogCollectMeta; -import com.xiaomi.mone.log.api.model.meta.LogPattern; -import com.xiaomi.mone.log.manager.dao.MilogAppTopicRelDao; -import com.xiaomi.mone.log.manager.dao.MilogLogTailDao; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.mone.log.manager.service.MiLogMetaManageService; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Resource; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * @author shanwb - * @date 2021-07-09 - */ -@Slf4j -@Service -public class MiLogMetaManageServiceImpl implements MiLogMetaManageService { - - @Resource - private MilogLogTailDao milogLogtailDao; - - @Resource - private MilogAppTopicRelDao milogAppTopicRelDao; - - @Override - public LogCollectMeta queryLogCollectMeta(String agentId, String agentIp) { - List appIdList = Lists.newArrayList(); - Map> miLogTailMap = milogLogtailDao.getMilogLogtailByAppId(appIdList); - LogCollectMeta meta = new LogCollectMeta(); - meta.setAgentId(agentId); - meta.setAgentIp(agentIp); - meta.setAgentMachine(null); - List metaList = new ArrayList<>(); - for (Map.Entry> entry : miLogTailMap.entrySet()) { - AppLogMeta appLogMeta = new AppLogMeta(); - appLogMeta.setAppId(entry.getKey()); - appLogMeta.setAppName(null); -// MilogAppTopicRel topicRel = milogAppTopicRelDao.queryByAppId(entry.getKey(), MoneContext.getCurrentUser().getZone()); -// MQConfig mqConfig = new MQConfig(); -// try { -// BeanUtilsBean.getInstance().copyProperties(mqConfig, topicRel.getMq_config()); -// } catch (Exception e) { -// log.error("Data copy exception:{}", new Gson().toJson(topicRel)); -// } -// appLogMeta.setMQConfig(mqConfig); - List logPatternList = new ArrayList<>(); - for (MilogLogTailDo milogLogtail : entry.getValue()) { - LogPattern logPattern = new LogPattern(); - logPattern.setLogtailId(milogLogtail.getId()); - // TODO - logPattern.setLogType(milogLogtail.getParseType()); - logPatternList.add(logPattern); - } - appLogMeta.setLogPatternList(logPatternList); - metaList.add(appLogMeta); - } - meta.setAppLogMetaList(metaList); - return meta; - } - - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogAppMiddlewareRelServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogAppMiddlewareRelServiceImpl.java deleted file mode 100644 index e8f4ef7e1..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogAppMiddlewareRelServiceImpl.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import org.apache.ozhera.app.api.response.AppBaseInfo; -import com.xiaomi.mone.log.api.enums.MiddlewareEnum; -import com.xiaomi.mone.log.manager.common.Utils; -import com.xiaomi.mone.log.manager.common.context.MoneUserContext; -import com.xiaomi.mone.log.manager.common.exception.MilogManageException; -import com.xiaomi.mone.log.manager.dao.MilogAppMiddlewareRelDao; -import com.xiaomi.mone.log.manager.dao.MilogLogTailDao; -import com.xiaomi.mone.log.manager.dao.MilogMiddlewareConfigDao; -import com.xiaomi.mone.log.manager.model.pojo.MilogAppMiddlewareRel; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.mone.log.manager.model.pojo.MilogMiddlewareConfig; -import com.xiaomi.mone.log.manager.service.MilogAppMiddlewareRelService; -import com.xiaomi.mone.log.manager.service.MqConfigService; -import com.xiaomi.youpin.docean.Ioc; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; -import java.time.Instant; -import java.util.List; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.COMMON_MQ_PREFIX; -import static com.xiaomi.mone.log.common.Constant.DEFAULT_CONSUMER_GROUP; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/23 11:14 - */ -@Service -@Slf4j -public class MilogAppMiddlewareRelServiceImpl implements MilogAppMiddlewareRelService { - - @Resource - private MilogAppMiddlewareRelDao milogAppMiddlewareRelDao; - @Resource - private MilogMiddlewareConfigDao milogMiddlewareConfigDao; - @Resource - private HeraAppServiceImpl heraAppService; - @Resource - private MilogLogTailDao milogLogtailDao; - - private MqConfigService mqConfigService; - - @Resource - private MilogAppMiddlewareRelDao middlewareRelDao; - - @Override - public void bindingTailConfigRel(Long tailId, Long milogAppId, Long middlewareConfigId, String topicName) { - //1.Query the configuration configuration configuration - //2.Storage - MilogMiddlewareConfig wareConfig = queryMiddlewareConfig(middlewareConfigId); - if (null != wareConfig) { - middlewareConfigId = wareConfig.getId(); - instantiateMqConfigSubClass(MiddlewareEnum.queryByCode(wareConfig.getType())); - AppBaseInfo appBaseInfo = heraAppService.queryById(milogAppId); - MilogAppMiddlewareRel.Config config = new MilogAppMiddlewareRel.Config(); - if (StringUtils.isEmpty(topicName)) { - config = mqConfigService.generateConfig(wareConfig.getAk(), - wareConfig.getSk(), wareConfig.getNameServer(), wareConfig.getServiceUrl(), - wareConfig.getAuthorization(), wareConfig.getOrgId(), wareConfig.getTeamId(), Long.valueOf(appBaseInfo.getBindId()), appBaseInfo.getAppName(), appBaseInfo.getPlatformName(), tailId); - } else { - config.setTopic(topicName); - config.setPartitionCnt(1); - } - MilogLogTailDo milogLogtailDo = milogLogtailDao.queryById(tailId); - String tag = Utils.createTag(milogLogtailDo.getSpaceId(), milogLogtailDo.getStoreId(), milogLogtailDo.getId()); - config.setTag(tag); - config.setConsumerGroup(DEFAULT_CONSUMER_GROUP + tag); - milogAppMiddlewareRelDao.insertUpdate(generateMiddlewareRel(tailId, milogAppId, middlewareConfigId, config)); - } else { - log.error("The middleware configuration for the current environment is empty,tailId:{},milogAppId:{},middlewareConfigId:{}", tailId, milogAppId, middlewareConfigId); - } - } - - @Override - public void defaultBindingAppTailConfigRel(Long id, Long milogAppId, - Long middleWareId, String topicName, Integer batchSendSize) { - MilogLogTailDo logTailDo = milogLogtailDao.queryById(id); - MilogMiddlewareConfig wareConfig = milogMiddlewareConfigDao.queryById(middleWareId); - if (null != wareConfig) { - if (StringUtils.isEmpty(topicName)) { - List commonTagTopicNames = Utils.generateCommonTagTopicName(StringUtils.EMPTY); -// topicName = commonTagTopicNames.get(Utils.getRandomNum(commonTagTopicNames.size())); - //find the least used -// topicName = findLeastUsedCommonTopic(commonTagTopicNames); - instantiateMqConfigSubClass(MiddlewareEnum.queryByCode(wareConfig.getType())); - AppBaseInfo appBaseInfo = heraAppService.queryById(milogAppId); - topicName = mqConfigService.generateSimpleTopicName(id, appBaseInfo.getAppName()); - } - MilogAppMiddlewareRel.Config config = new MilogAppMiddlewareRel.Config(); - config.setTopic(topicName); - config.setPartitionCnt(1); - String tag = Utils.createTag(logTailDo.getSpaceId(), logTailDo.getStoreId(), id); - config.setConsumerGroup(String.format("%s_%s", "group", tag)); - config.setTag(tag); - if (null != batchSendSize) { - config.setBatchSendSize(batchSendSize); - } - handleTailMqRel(id, milogAppId, wareConfig, config); - } else { - log.error("If the current organization does not have MQ configuration information configured, configure the message configuration information of the current organization,tailId：{},storeId:{},middleWareId:{}", - id, milogAppId, middleWareId); - throw new MilogManageException("If the current organization does not configure MQ configuration information, configure the resource configuration information of the current department"); - } - } - - private String findLeastUsedCommonTopic(List existTopics) { - List commonTopics = existTopics.stream() - .filter(s -> s.startsWith(COMMON_MQ_PREFIX)) - .sorted((a, b) -> middlewareRelDao.queryCountByTopicName(a) - .compareTo(middlewareRelDao.queryCountByTopicName(b))) - .collect(Collectors.toList()); - if (CollectionUtils.isNotEmpty(commonTopics)) { - return commonTopics.get(0); - } - return StringUtils.EMPTY; - } - - - private void handleTailMqRel(Long tailId, Long milogAppId, MilogMiddlewareConfig wareConfig, MilogAppMiddlewareRel.Config config) { - MilogLogTailDo milogLogtailDo = milogLogtailDao.queryById(tailId); - String tag = Utils.createTag(milogLogtailDo.getSpaceId(), milogLogtailDo.getStoreId(), milogLogtailDo.getId()); - config.setTag(tag); - config.setConsumerGroup(DEFAULT_CONSUMER_GROUP + tag); - MilogAppMiddlewareRel milogAppMiddlewareRel = generateMiddlewareRel(tailId, milogAppId, wareConfig.getId(), config); - milogAppMiddlewareRelDao.insertUpdate(milogAppMiddlewareRel); - } - - private MilogAppMiddlewareRel generateMiddlewareRel(Long tailId, Long milogAppId, Long configId, MilogAppMiddlewareRel.Config config) { - MilogAppMiddlewareRel milogAppMiddlewareRel = new MilogAppMiddlewareRel(); - milogAppMiddlewareRel.setMilogAppId(milogAppId); - milogAppMiddlewareRel.setMiddlewareId(configId); - milogAppMiddlewareRel.setTailId(tailId); - milogAppMiddlewareRel.setConfig(config); - milogAppMiddlewareRel.setCtime(Instant.now().toEpochMilli()); - milogAppMiddlewareRel.setUtime(Instant.now().toEpochMilli()); - if (StringUtils.isEmpty(MoneUserContext.getCurrentUser().getUser())) { - MilogLogTailDo milogLogTailDo = milogLogtailDao.queryById(tailId); - milogAppMiddlewareRel.setCreator(milogLogTailDo.getCreator()); - milogAppMiddlewareRel.setUpdater(milogLogTailDo.getUpdater()); - } else { - milogAppMiddlewareRel.setCreator(MoneUserContext.getCurrentUser().getUser()); - milogAppMiddlewareRel.setUpdater(MoneUserContext.getCurrentUser().getUser()); - } - return milogAppMiddlewareRel; - } - - /** - * middlewareConfigId does not look for its own correspondence for empty Otherwise, select the default - * - * @param middlewareConfigId - * @return - */ - public MilogMiddlewareConfig queryMiddlewareConfig(Long middlewareConfigId) { - MilogMiddlewareConfig wareConfig; - if (null != middlewareConfigId) { - wareConfig = milogMiddlewareConfigDao.queryById(middlewareConfigId); - } else { - wareConfig = milogMiddlewareConfigDao.queryDefaultMiddlewareConfig(); - } - return wareConfig; - } - - - private void instantiateMqConfigSubClass(MiddlewareEnum middlewareEnum) { - switch (middlewareEnum) { - case ROCKETMQ: - mqConfigService = Ioc.ins().getBean(RocketMqConfigService.class); - break; - case KAFKA: - mqConfigService = Ioc.ins().getBean(KafkaMqConfigService.class); - } - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogConfigNacosServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogConfigNacosServiceImpl.java deleted file mode 100644 index 1221aad19..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogConfigNacosServiceImpl.java +++ /dev/null @@ -1,473 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import cn.hutool.core.bean.BeanUtil; -import cn.hutool.core.lang.Assert; -import com.alibaba.nacos.api.config.ConfigService; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.gson.Gson; -import com.xiaomi.data.push.nacos.NacosNaming; -import com.xiaomi.mone.log.api.enums.LogStorageTypeEnum; -import com.xiaomi.mone.log.api.enums.MQSourceEnum; -import com.xiaomi.mone.log.api.enums.OperateEnum; -import com.xiaomi.mone.log.manager.common.Utils; -import com.xiaomi.mone.log.manager.dao.MilogAppMiddlewareRelDao; -import com.xiaomi.mone.log.manager.dao.MilogLogTailDao; -import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; -import com.xiaomi.mone.log.manager.dao.MilogMiddlewareConfigDao; -import com.xiaomi.mone.log.manager.domain.EsCluster; -import com.xiaomi.mone.log.manager.mapper.MilogLogTemplateMapper; -import com.xiaomi.mone.log.manager.model.pojo.*; -import com.xiaomi.mone.log.manager.service.MilogConfigNacosService; -import com.xiaomi.mone.log.manager.service.bind.LogTypeProcessor; -import com.xiaomi.mone.log.manager.service.bind.LogTypeProcessorFactory; -import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionService; -import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionServiceFactory; -import com.xiaomi.mone.log.manager.service.extension.store.DorisLogStorageService; -import com.xiaomi.mone.log.manager.service.extension.tail.TailExtensionService; -import com.xiaomi.mone.log.manager.service.extension.tail.TailExtensionServiceFactory; -import com.xiaomi.mone.log.manager.service.nacos.DynamicConfigProvider; -import com.xiaomi.mone.log.manager.service.nacos.DynamicConfigPublisher; -import com.xiaomi.mone.log.manager.service.nacos.FetchStreamMachineService; -import com.xiaomi.mone.log.manager.service.nacos.MultipleNacosConfig; -import com.xiaomi.mone.log.manager.service.nacos.impl.*; -import com.xiaomi.mone.log.model.*; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.plugin.config.anno.Value; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.map.HashedMap; - -import javax.annotation.Resource; -import java.util.*; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.*; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/19 16:10 - */ -@Slf4j -@Service -public class MilogConfigNacosServiceImpl implements MilogConfigNacosService { - - private static Map configPublisherMap = new HashedMap(); - private static Map configProviderMap = new HashedMap(); - - private static Map streamServiceUniqueMap = new HashedMap(); - - @Resource - private MilogLogstoreDao logStoreDao; - - @Resource - private MilogLogTailDao milogLogtailDao; - - @Resource - private EsCluster esCluster; - - @Resource - private MilogAppMiddlewareRelDao milogAppMiddlewareRelDao; - @Resource - private MilogMiddlewareConfigDao milogMiddlewareConfigDao; - - @Resource - private DorisLogStorageService dorisLogStorageService; - - @Value(value = "$europe.ip.key") - private String europeIpKey; - - @Value(value = "$app.env") - private String appEnv; - - @Resource - private MilogLogTemplateMapper milogLogTemplateMapper; - - @Resource - private LogTypeProcessorFactory logTypeProcessorFactory; - - private TailExtensionService tailExtensionService; - - private LogTypeProcessor logTypeProcessor; - - private CommonExtensionService commonExtensionService; - - public void init() { - tailExtensionService = TailExtensionServiceFactory.getTailExtensionService(); - commonExtensionService = CommonExtensionServiceFactory.getCommonExtensionService(); - - logTypeProcessorFactory.setMilogLogTemplateMapper(milogLogTemplateMapper); - logTypeProcessor = logTypeProcessorFactory.getLogTypeProcessor(); - - initializeNacosConfigurations(); - } - - private void initializeNacosConfigurations() { - List regions = commonExtensionService.queryMachineRegions(); - for (String region : regions) { - chooseCurrentEnvNacosSerevice(region); - } - } - - - public StreamConfigNacosPublisher getStreamConfigNacosPublisher(String motorRoomEn) { - return (StreamConfigNacosPublisher) configPublisherMap.get(STREAM_PREFIX + motorRoomEn); - } - - public StreamConfigNacosProvider getStreamConfigNacosProvider(String motorRoomEn) { - return (StreamConfigNacosProvider) configProviderMap.get(STREAM_PREFIX + motorRoomEn); - } - - public SpaceConfigNacosPublisher getSpaceConfigNacosPublisher(String motorRoomEn) { - return (SpaceConfigNacosPublisher) configPublisherMap.get(SPACE_PREFIX + motorRoomEn); - } - - public SpaceConfigNacosProvider getSpaceConfigNacosProvider(String motorRoomEn) { - return (SpaceConfigNacosProvider) configProviderMap.get(SPACE_PREFIX + motorRoomEn); - } - - public FetchStreamMachineService getFetchStreamMachineService(String motorRoomEn) { - return streamServiceUniqueMap.get(STREAM_PREFIX + motorRoomEn); - } - - /** - * Push namespace configuration - * - * @param spaceId spaceId - */ - - @Override - public void publishStreamConfig(Long spaceId, Integer type, Integer projectTypeCode, String motorRoomEn) { - //1.Query all stream machine IPs - real-time query - List mioneStreamIpList = tailExtensionService.fetchStreamUniqueKeyList(getFetchStreamMachineService(motorRoomEn), spaceId, motorRoomEn); - log.info("Query the list of machines in log-stream：{}", new Gson().toJson(mioneStreamIpList)); - //2.send msg - getStreamConfigNacosPublisher(motorRoomEn).publish(spaceId, dealStreamConfigByRule(motorRoomEn, mioneStreamIpList, spaceId, type)); - tailExtensionService.publishStreamConfigPostProcess(getStreamConfigNacosPublisher(motorRoomEn), spaceId, motorRoomEn); - } - - private synchronized MiLogStreamConfig dealStreamConfigByRule(String motorRoomEn, List ipList, Long spaceId, Integer type) { - MiLogStreamConfig existConfig = getStreamConfigNacosProvider(motorRoomEn).getConfig(spaceId); - ipList = ensureDefaultCompatibility(existConfig, ipList); - // New configuration - String spaceKey = CommonExtensionServiceFactory.getCommonExtensionService().getLogManagePrefix() + TAIL_CONFIG_DATA_ID + spaceId; - if (null == existConfig || OperateEnum.ADD_OPERATE.getCode().equals(type) || OperateEnum.UPDATE_OPERATE.getCode().equals(type)) { - // The configuration is not configured yet, initialize the configuration - if (null == existConfig) { - existConfig = new MiLogStreamConfig(); - Map> config = new HashMap<>(); - boolean idAdd = false; - for (String ip : ipList) { - Map map = Maps.newHashMapWithExpectedSize(1); - if (!idAdd) { - map.put(spaceId, spaceKey); - idAdd = true; - } - config.put(ip, map); - } - existConfig.setConfig(config); - } else { - Map> config = existConfig.getConfig(); - if (config.values().stream() - .flatMap(longStringMap -> longStringMap.values().stream()) - .anyMatch(s -> s.equals(spaceKey))) { - return existConfig; - } - // 1.Average the quantity first - // 2.Added name Space - if (CollectionUtils.isNotEmpty(ipList)) { - for (String ip : ipList) { - if (!config.containsKey(ip)) { - config.put(ip, Maps.newHashMap()); - } - } - } - // The number of name spaces held per machine - Map ipSizeMap = config.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, stringMapEntry -> stringMapEntry.getValue().size())); - List finalIpList = ipList; - String key = ipSizeMap.entrySet().stream() - .filter(entry -> finalIpList.contains(entry.getKey())) - .min(Map.Entry.comparingByValue()).get().getKey(); - config.get(key).put(spaceId, spaceKey); - } - } - // Delete the configuration - if (OperateEnum.DELETE_OPERATE.getCode().equals(type)) { - if (null != existConfig) { - Map> config = existConfig.getConfig(); - config.values().forEach(longStringMap -> longStringMap.keySet().removeIf(key -> key.equals(spaceId))); - } - getSpaceConfigNacosPublisher(motorRoomEn).remove(spaceId.toString()); - } - return existConfig; - } - - /** - * compatible When the queried IP is different from the actual one, the actual one is returned - * - * @param existConfig - * @param ipList - * @return - */ - private List ensureDefaultCompatibility(MiLogStreamConfig existConfig, List ipList) { - Set keySet = existConfig.getConfig().keySet(); - if (!CollectionUtils.isEqualCollection(keySet, ipList)) { - log.info("ipList not belong to config,query list:{},actual list:{}", GSON.toJson(ipList), GSON.toJson(keySet)); - ipList = keySet.stream().toList(); - } - return ipList; - } - - @Override - public void publishNameSpaceConfig(String motorRoomEn, Long spaceId, Long storeId, Long tailId, Integer type, String changeType) { - Assert.notNull(spaceId, "logSpaceId not empty"); - Assert.notNull(storeId, "storeId not empty"); - //send msg - getSpaceConfigNacosPublisher(motorRoomEn).publish(spaceId, - dealSpaceConfigByRule(motorRoomEn, spaceId, storeId, tailId, type, changeType)); - } - - /** - * Select the appropriate NACOS environment address - * - * @param motorRoomEn - */ - public void chooseCurrentEnvNacosSerevice(String motorRoomEn) { - MilogMiddlewareConfig middlewareConfig = milogMiddlewareConfigDao.queryCurrentEnvNacos(motorRoomEn); - if (null != middlewareConfig) { - ConfigService configService = MultipleNacosConfig.getConfigService(middlewareConfig.getNameServer()); - - SpaceConfigNacosPublisher spaceConfigNacosPublisher = (SpaceConfigNacosPublisher) configPublisherMap.get(SPACE_PREFIX + motorRoomEn); - if (null == spaceConfigNacosPublisher) { - spaceConfigNacosPublisher = new SpaceConfigNacosPublisher(); - spaceConfigNacosPublisher.setConfigService(configService); - configPublisherMap.put(SPACE_PREFIX + motorRoomEn, spaceConfigNacosPublisher); - } - - StreamConfigNacosPublisher streamConfigNacosPublisher = (StreamConfigNacosPublisher) configPublisherMap.get(STREAM_PREFIX + motorRoomEn); - if (null == streamConfigNacosPublisher) { - streamConfigNacosPublisher = new StreamConfigNacosPublisher(); - streamConfigNacosPublisher.setConfigService(configService); - configPublisherMap.put(STREAM_PREFIX + motorRoomEn, streamConfigNacosPublisher); - } - - SpaceConfigNacosProvider spaceConfigNacosProvider = (SpaceConfigNacosProvider) configProviderMap.get(SPACE_PREFIX + motorRoomEn); - if (null == spaceConfigNacosProvider) { - spaceConfigNacosProvider = new SpaceConfigNacosProvider(); - spaceConfigNacosProvider.setConfigService(configService); - configProviderMap.put(SPACE_PREFIX + motorRoomEn, spaceConfigNacosProvider); - } - - StreamConfigNacosProvider streamConfigNacosProvider = (StreamConfigNacosProvider) configProviderMap.get(STREAM_PREFIX + motorRoomEn); - if (null == streamConfigNacosProvider) { - streamConfigNacosProvider = new StreamConfigNacosProvider(); - streamConfigNacosProvider.setConfigService(configService); - configProviderMap.put(STREAM_PREFIX + motorRoomEn, streamConfigNacosProvider); - } - NacosFetchStreamMachineService fetchStreamMachineService = (NacosFetchStreamMachineService) streamServiceUniqueMap.get(STREAM_PREFIX + motorRoomEn); - if (null == fetchStreamMachineService) { - NacosNaming nacosNaming = MultipleNacosConfig.getNacosNaming(middlewareConfig.getNameServer()); - fetchStreamMachineService = new NacosFetchStreamMachineService(nacosNaming); - streamServiceUniqueMap.put(STREAM_PREFIX + motorRoomEn, fetchStreamMachineService); - } - } else { - log.info("Current data center: {} does not have NACOS configuration information", motorRoomEn); - } - } - - @Override - public void removeStreamConfig(String motorRoomEn, Long id) { - getSpaceConfigNacosPublisher(motorRoomEn).remove(id + ""); - } - - private synchronized MilogSpaceData dealSpaceConfigByRule( - String motorRoomEn, Long spaceId, Long storeId, Long tailId, Integer type, String changeType) { - MilogSpaceData existConfig = getSpaceConfigNacosProvider(motorRoomEn).getConfig(spaceId); - // new configuration - if (null == existConfig || OperateEnum.ADD_OPERATE.getCode().equals(type)) { - // The configuration is not configured yet, initialize the configuration - if (null == existConfig || CollectionUtils.isEmpty(existConfig.getSpaceConfig())) { - existConfig = new MilogSpaceData(); - existConfig.setMilogSpaceId(spaceId); - List spaceConfigs = Lists.newArrayList(); - spaceConfigs.add(assembleSinkConfig(storeId, tailId, motorRoomEn)); - existConfig.setSpaceConfig(spaceConfigs); - } else { - List spaceConfig = existConfig.getSpaceConfig(); - SinkConfig currentStoreConfig = spaceConfig.stream() - .filter(sinkConfig -> sinkConfig.getLogstoreId().equals(storeId)) - .findFirst() - .orElse(null); - existConfig.setMilogSpaceId(spaceId); - if (null != currentStoreConfig) { - List logtailConfigs = currentStoreConfig.getLogtailConfigs(); - if (CollectionUtils.isEmpty(logtailConfigs)) { - logtailConfigs = Lists.newArrayList(); - } - logtailConfigs.add(assembleLogTailConfigs(tailId)); - currentStoreConfig.setLogtailConfigs(logtailConfigs); - } else { - //New addition to the log Store - spaceConfig.add(assembleSinkConfig(storeId, tailId, motorRoomEn)); - } - } - } - // Delete configuration -- Delete log-tail - if (OperateEnum.DELETE_OPERATE.getCode().equals(type) && !LOG_STORE.equalsIgnoreCase(changeType)) { - if (null != existConfig) { - List spaceConfig = existConfig.getSpaceConfig(); - SinkConfig currentStoreConfig = spaceConfig.stream() - .filter(sinkConfig -> sinkConfig.getLogstoreId().equals(storeId)) - .findFirst() - .orElse(null); - if (null != currentStoreConfig) { - List logTailConfigs = currentStoreConfig.getLogtailConfigs(); - List logtailConfigList = new ArrayList<>(logTailConfigs); - - if (null != tailId && CollectionUtils.isNotEmpty(logTailConfigs) && - logTailConfigs.stream().anyMatch(config -> config.getLogtailId().equals(tailId))) { - logtailConfigList.removeIf(logtailConfig -> logtailConfig.getLogtailId().equals(tailId)); - } - currentStoreConfig.setLogtailConfigs(logtailConfigList); - } - } - } - // Delete configuration -- Delete log-tail - if (OperateEnum.DELETE_OPERATE.getCode().equals(type) && LOG_STORE.equalsIgnoreCase(changeType)) { - if (null != existConfig) { - List sinkConfigListDelStore = existConfig.getSpaceConfig().stream() - .filter(sinkConfig -> !storeId.equals(sinkConfig.getLogstoreId())) - .collect(Collectors.toList()); - existConfig.setSpaceConfig(sinkConfigListDelStore); - } - } - // Modify the configuration -- find a specific tail under this store to make changes - if (OperateEnum.UPDATE_OPERATE.getCode().equals(type)) { - if (null != existConfig) { - List spaceConfig = existConfig.getSpaceConfig(); - //Compare whether the store has changed - SinkConfig newSinkConfig = assembleSinkConfig(storeId, tailId, motorRoomEn); - SinkConfig currentStoreConfig = spaceConfig.stream() - .filter(sinkConfig -> sinkConfig.getLogstoreId().equals(storeId)) - .findFirst() - .orElse(null); - if (null != currentStoreConfig) { - if (!newSinkConfig.equals(currentStoreConfig)) { - currentStoreConfig.updateStoreParam(newSinkConfig); - } - // Find the specific tail under the old store - LogtailConfig filterLogTailConfig = currentStoreConfig.getLogtailConfigs().stream() - .filter(logTailConfig -> Objects.equals(tailId, logTailConfig.getLogtailId())) - .findFirst() - .orElse(null); - if (null != filterLogTailConfig) { - BeanUtil.copyProperties(assembleLogTailConfigs(tailId), filterLogTailConfig); - } else { - log.info("query logtailConfig no designed config,tailId:{},insert", tailId); - currentStoreConfig.getLogtailConfigs().add(assembleLogTailConfigs(tailId)); - } - } else { - //Does not exist, new - //New addition to the logstore - spaceConfig.add(assembleSinkConfig(storeId, tailId, motorRoomEn)); - } - } - } - return existConfig; - } - - public SinkConfig assembleSinkConfig(Long storeId, Long tailId, String motorRoomEn) { - SinkConfig sinkConfig = new SinkConfig(); - sinkConfig.setLogstoreId(storeId); - // Query the log Store - MilogLogStoreDO logStoreDO = logStoreDao.queryById(storeId); - if (null != logStoreDO) { - boolean supportedConsume = logTypeProcessor.supportedConsume(logStoreDO.getLogType()); - - if (!supportedConsume) { - return sinkConfig; - } - - sinkConfig.setLogstoreName(logStoreDO.getLogstoreName()); - sinkConfig.setKeyList(Utils.parse2KeyAndTypeList(logStoreDO.getKeyList(), logStoreDO.getColumnTypeList())); - MilogEsClusterDO esInfo = esCluster.getById(logStoreDO.getEsClusterId()); - if (null != esInfo) { - sinkConfig.setEsIndex(logStoreDO.getEsIndex()); - sinkConfig.setEsInfo(buildEsInfo(esInfo)); - sinkConfig.setStorageType(esInfo.getLogStorageType()); - - LogStorageTypeEnum storageTypeEnum = LogStorageTypeEnum.queryByName(esInfo.getLogStorageType()); - if (LogStorageTypeEnum.DORIS == storageTypeEnum) { - sinkConfig.setColumnList(dorisLogStorageService.getColumnList(logStoreDO.getEsClusterId(), logStoreDO.getEsIndex())); - } - } else { - log.info("assembleSinkConfig esInfo is null,logStoreId:{}", logStoreDO.getId()); - } - } - sinkConfig.setLogtailConfigs(Arrays.asList(assembleLogTailConfigs(tailId))); - return sinkConfig; - } - - private StorageInfo buildEsInfo(MilogEsClusterDO clusterDO) { - if (Objects.equals(ES_CONWAY_PWD, clusterDO.getConWay())) { - return new StorageInfo(clusterDO.getId(), clusterDO.getAddr(), clusterDO.getUser(), clusterDO.getPwd()); - } - return new StorageInfo(clusterDO.getId(), clusterDO.getAddr(), clusterDO.getToken(), clusterDO.getDtCatalog(), clusterDO.getDtDatabase()); - } - - - public LogtailConfig assembleLogTailConfigs(Long tailId) { - LogtailConfig logtailConfig = new LogtailConfig(); - MilogLogTailDo milogLogTail = milogLogtailDao.queryById(tailId); - if (null != milogLogTail) { - logtailConfig.setLogtailId(tailId); - logtailConfig.setTail(milogLogTail.getTail()); - logtailConfig.setParseType(milogLogTail.getParseType()); - logtailConfig.setParseScript(milogLogTail.getParseScript()); - logtailConfig.setValueList(milogLogTail.getValueList()); - logtailConfig.setAppType(milogLogTail.getAppType()); - // Query MQ information - handleTailConfig(tailId, milogLogTail.getStoreId(), milogLogTail.getSpaceId(), - milogLogTail.getMilogAppId(), logtailConfig, milogLogTail.getAppType()); - } - return logtailConfig; - } - - private void handleTailConfig(Long tailId, Long storeId, Long spaceId, Long milogAppId, LogtailConfig logtailConfig, Integer appType) { - List milogAppMiddlewareRels = milogAppMiddlewareRelDao.queryByCondition(milogAppId, null, tailId); - if (CollectionUtils.isNotEmpty(milogAppMiddlewareRels)) { - MilogAppMiddlewareRel milogAppMiddlewareRel = milogAppMiddlewareRels.get(0); - MilogAppMiddlewareRel.Config config = milogAppMiddlewareRel.getConfig(); - MilogMiddlewareConfig middlewareConfig = milogMiddlewareConfigDao.queryById(milogAppMiddlewareRel.getMiddlewareId()); - - logtailConfig.setAk(middlewareConfig.getAk()); - logtailConfig.setSk(middlewareConfig.getSk()); - logtailConfig.setTopic(config.getTopic()); - String tag = Utils.createTag(spaceId, storeId, tailId); - - logtailConfig.setTag(tag); - logtailConfig.setConsumerGroup(config.getConsumerGroup()); - logtailConfig.setType(MQSourceEnum.queryName(middlewareConfig.getType())); - logtailConfig.setClusterInfo(middlewareConfig.getNameServer()); - TailExtensionServiceFactory.getTailExtensionService().logTailConfigExtraField(logtailConfig, middlewareConfig); - } - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogDictionaryServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogDictionaryServiceImpl.java deleted file mode 100644 index 34884a992..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogDictionaryServiceImpl.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import com.google.common.base.Stopwatch; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.gson.Gson; -import com.xiaomi.mone.log.api.enums.LogStorageTypeEnum; -import com.xiaomi.mone.log.api.enums.MachineRegionEnum; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.dao.MilogLogTailDao; -import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; -import com.xiaomi.mone.log.manager.dao.MilogMiddlewareConfigDao; -import com.xiaomi.mone.log.manager.model.bo.MilogDictionaryParam; -import com.xiaomi.mone.log.manager.model.dto.DictionaryDTO; -import com.xiaomi.mone.log.manager.model.dto.MotorRoomDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.mone.log.manager.model.pojo.MilogMiddlewareConfig; -import com.xiaomi.mone.log.manager.service.MilogDictionaryService; -import com.xiaomi.mone.log.manager.service.extension.dictionary.DictionaryExtensionService; -import com.xiaomi.mone.log.manager.service.extension.dictionary.DictionaryExtensionServiceFactory; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/26 15:34 - */ -@Slf4j -@Service -public class MilogDictionaryServiceImpl implements MilogDictionaryService { - - @Resource - private MilogLogTailDao milogLogtailDao; - - @Resource - private MilogLogstoreDao milogLogstoreDao; - - @Resource - private MilogMiddlewareConfigDao milogMiddlewareConfigDao; - - private DictionaryExtensionService dictionaryExtensionService; - - public void init() { - dictionaryExtensionService = DictionaryExtensionServiceFactory.getDictionaryExtensionService(); - } - - /** - * @param dictionaryParam code : - * 1001:When creating a tail, select the MQ configuration for collection - * 1002：mq type - * 1003: MQ type and the topic information below - * 1004：app type - * 1005: Store and applications under the computer room - * 1006:Computer room information - * 1007:Deployment method - * 1008:Resource tab page number - * 1009:LogStorageTypeEnum type - * 1010:mq type - * @return - */ - @Override - public Result>>> queryDictionaryList(MilogDictionaryParam dictionaryParam) { - if (null == dictionaryParam || CollectionUtils.isEmpty(dictionaryParam.getCodes())) { - return Result.failParam("code Cannot be empty"); - } - if (CollectionUtils.isNotEmpty(dictionaryParam.getCodes().stream().filter(code -> code.intValue() == 1003).collect(Collectors.toList())) && null == dictionaryParam.getMiddlewareId()) { - return Result.failParam("middlewareId Cannot be empty"); - } - if (CollectionUtils.isNotEmpty(dictionaryParam.getCodes().stream().filter(code -> code.intValue() == 1005).collect(Collectors.toList())) && StringUtils.isEmpty(dictionaryParam.getNameEn())) { - return Result.failParam("nameEn Cannot be empty"); - } - Map>> dictionaryDTO = Maps.newHashMap(); - dictionaryParam.getCodes().stream().forEach(code -> { - switch (code) { - case 1001: - dictionaryDTO.put(code, dictionaryExtensionService.queryMiddlewareConfigDictionary(StringUtils.isNotEmpty(dictionaryParam.getNameEn()) ? dictionaryParam.getNameEn() : MachineRegionEnum.CN_MACHINE.getEn())); - break; - case 1002: - dictionaryDTO.put(code, dictionaryExtensionService.queryResourceDictionary()); - break; - case 1003: - dictionaryDTO.put(code, queryAllRocketMqTopic(dictionaryParam.getMiddlewareId())); - break; - case 1004: - dictionaryDTO.put(code, dictionaryExtensionService.queryAppType()); - break; - case 1005: - dictionaryDTO.put(code, queryStoreTailByEnName(dictionaryParam.getNameEn())); - break; - case 1006: - dictionaryDTO.put(code, dictionaryExtensionService.queryMachineRegion()); - break; - case 1007: - dictionaryDTO.put(code, dictionaryExtensionService.queryDeployWay()); - break; - case 1008: - dictionaryDTO.put(code, dictionaryExtensionService.queryResourceTypeDictionary()); - break; - case 1009: - dictionaryDTO.put(code, queryLogStorageTypeDictionary()); - break; - case 1010: - dictionaryDTO.put(code, dictionaryExtensionService.queryMQDictionary()); - break; - } - }); - log.debug("return val：{}", new Gson().toJson(dictionaryDTO)); - return Result.success(dictionaryDTO); - } - - private List queryLogStorageTypeDictionary() { - return Arrays.stream(LogStorageTypeEnum.values()).map(data -> { - DictionaryDTO dictionaryDTO = new DictionaryDTO(); - dictionaryDTO.setLabel(data.name()); - dictionaryDTO.setValue(data.name().toLowerCase()); - return dictionaryDTO; - }).collect(Collectors.toList()); - } - - private List> queryStoreTailByEnName(String nameEn) { - List> dictionaryDTOS = Lists.newArrayList(); - List milogLogtailDos = dictionaryExtensionService.querySpecialTails(); - if (CollectionUtils.isNotEmpty(milogLogtailDos)) { - List storedIds = milogLogtailDos.stream() - .filter(milogLogtailDo -> milogLogtailDo.getMotorRooms().stream() - .map(MotorRoomDTO::getNameEn).collect(Collectors.toList()).contains(nameEn)) - .map(MilogLogTailDo::getStoreId).collect(Collectors.toList()); - storedIds.forEach(storeId -> { - MilogLogStoreDO milogLogstoreDO = milogLogstoreDao.queryById(storeId); - DictionaryDTO dictionaryDTO = new DictionaryDTO(); - dictionaryDTO.setLabel(milogLogstoreDO.getLogstoreName()); - dictionaryDTO.setValue(milogLogstoreDO.getId()); - dictionaryDTO.setChildren(queryTailByStore(milogLogtailDos, storeId, nameEn)); - dictionaryDTOS.add(dictionaryDTO); - }); - } - return dictionaryDTOS; - } - - private List> queryTailByStore(List milogLogtailDos, Long storeId, String nameEn) { - List> dictionaryDTOS = Lists.newArrayList(); - if (CollectionUtils.isNotEmpty(milogLogtailDos)) { - milogLogtailDos = milogLogtailDos.stream() - .filter(milogLogtailDo -> storeId.equals(milogLogtailDo.getStoreId())) - .filter(milogLogtailDo -> milogLogtailDo.getMotorRooms().stream() - .map(MotorRoomDTO::getNameEn).collect(Collectors.toList()).contains(nameEn)) - .collect(Collectors.toList()); - milogLogtailDos.forEach(milogLogtailDo -> { - DictionaryDTO dictionaryDTO = new DictionaryDTO(); - dictionaryDTO.setLabel(milogLogtailDo.getTail()); - dictionaryDTO.setValue(milogLogtailDo.getId()); - dictionaryDTOS.add(dictionaryDTO); - }); - } - return dictionaryDTOS; - } - - @Override - public Result downLoadFile() { - File file = new File("D:\\work\\rocketmq.log"); - String str = ""; - try { - FileOutputStream fileOutputStream = new FileOutputStream(file); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } - return null; - } - - @Override - public Result fixLogTailMilogAppId(String appName) { - List milogLogtailDos = milogLogtailDao.queryTailByAppName(appName); - log.info("Synchronously repair tail's milogAppId, with {} entries", milogLogtailDos.size()); - int count = 0; - Stopwatch stopwatch = Stopwatch.createStarted(); - for (MilogLogTailDo milogLogtailDo : milogLogtailDos) { - ++count; - log.info("Start synchronizing the {} article of the tail, and there are {} articles left", count, milogLogtailDos.size() - count); - if (null == milogLogtailDo.getMilogAppId()) { - } - } - stopwatch.stop(); - log.info("Synchronously repair tail's milogAppId, which takes time: {} s", stopwatch.elapsed().getSeconds()); - return Result.success(); - } - - private List> queryAllRocketMqTopic(Long middlewareId) { - MilogMiddlewareConfig middlewareConfig = milogMiddlewareConfigDao.queryById(middlewareId); - List dictionaryDTOS = dictionaryExtensionService.queryExistsTopic(middlewareConfig.getAk(), middlewareConfig.getSk(), middlewareConfig.getNameServer(), - middlewareConfig.getServiceUrl(), middlewareConfig.getAuthorization(), middlewareConfig.getOrgId(), middlewareConfig.getTeamId()); - Set existTopics = dictionaryDTOS.stream().map(dictionaryDTO -> dictionaryDTO.getValue().toString()).collect(Collectors.toSet()); - List> dictionaryDTOList = Lists.newArrayList(); - existTopics.stream().map(s -> dictionaryDTOList.add(DictionaryDTO.Of(s, s))); - return dictionaryDTOList; - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogEsClusterServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogEsClusterServiceImpl.java deleted file mode 100644 index e8caf434e..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogEsClusterServiceImpl.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; -import com.xiaomi.mone.log.manager.mapper.MilogEsClusterMapper; -import com.xiaomi.mone.log.manager.model.pojo.MilogEsClusterDO; -import com.xiaomi.mone.log.manager.service.IMilogEsClusterService; -import com.xiaomi.youpin.docean.anno.Service; - -/** - * @author wanghaoyang - * @since 2021-09-18 - */ -@Service -public class MilogEsClusterServiceImpl extends ServiceImpl implements IMilogEsClusterService { - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogEsIndexServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogEsIndexServiceImpl.java deleted file mode 100644 index 72a599885..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogEsIndexServiceImpl.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; -import com.xiaomi.mone.log.manager.mapper.MilogEsIndexMapper; -import com.xiaomi.mone.log.manager.model.pojo.MilogEsIndexDO; -import com.xiaomi.mone.log.manager.service.IMilogEsIndexService; -import com.xiaomi.youpin.docean.anno.Service; - -/** - *

- * Service implementation class - *

- * - * @author wanghaoyang - * @since 2021-09-18 - */ -@Service -public class MilogEsIndexServiceImpl extends ServiceImpl implements IMilogEsIndexService { - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogLogSearchSaveServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogLogSearchSaveServiceImpl.java deleted file mode 100644 index 98cad2ff1..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogLogSearchSaveServiceImpl.java +++ /dev/null @@ -1,282 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.api.enums.FavouriteSearchEnum; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.exception.CommonError; -import com.xiaomi.mone.log.manager.common.context.MoneUserContext; -import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; -import com.xiaomi.mone.log.manager.dao.MilogSpaceDao; -import com.xiaomi.mone.log.manager.dao.MilogStoreSpaceAuthDao; -import com.xiaomi.mone.log.manager.domain.Store; -import com.xiaomi.mone.log.manager.mapper.MilogLogSearchSaveMapper; -import com.xiaomi.mone.log.manager.model.convert.MilogSpaceConvert; -import com.xiaomi.mone.log.manager.model.convert.SearchSaveConvert; -import com.xiaomi.mone.log.manager.model.dto.MilogSpaceDTO; -import com.xiaomi.mone.log.manager.model.dto.SearchSaveDTO; -import com.xiaomi.mone.log.manager.model.dto.SpaceTreeFavouriteDTO; -import com.xiaomi.mone.log.manager.model.dto.StoreTreeDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogSearchSaveDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogSpaceDO; -import com.xiaomi.mone.log.manager.model.vo.KeywordPageParam; -import com.xiaomi.mone.log.manager.model.vo.SearchSaveInsertCmd; -import com.xiaomi.mone.log.manager.model.vo.SearchSaveUpdateCmd; -import com.xiaomi.mone.log.manager.service.IMilogLogSearchSaveService; -import com.xiaomi.mone.tpc.common.vo.NodeVo; -import com.xiaomi.mone.tpc.common.vo.PageDataVo; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; - -import javax.annotation.Resource; -import java.util.*; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.stream.Collectors; - -/** - * @author wanghaoyang - * @since 2022-03-29 - */ -@Service -@Slf4j -public class MilogLogSearchSaveServiceImpl implements IMilogLogSearchSaveService { - - @Resource - private MilogLogSearchSaveMapper logSearchSaveMapper; - - @Resource - private MilogLogstoreDao logStoreDao; - - @Resource - private MilogSpaceDao logSpaceDao; - - @Resource - private MilogStoreSpaceAuthDao storeSpaceAuthDao; - - @Resource - private MilogLogSearchSaveMapper searchSaveMapper; - - @Resource - private TpcSpaceAuthService spaceAuthService; - - @Resource - private Store store; - - public Result> list(Long storeId, Integer sort) { - List list = logSearchSaveMapper.selectByCreator(MoneUserContext.getCurrentUser().getUser(), sort); - List result = new ArrayList<>(); - for (SearchSaveDTO searchSaveDTO : list) { - Long storeIdInner = searchSaveDTO.getStoreId(); - MilogLogStoreDO logStoreDO = logStoreDao.queryById(storeIdInner); - if (null != logStoreDO) { - searchSaveDTO.setSpaceId(logStoreDO.getSpaceId()); - MilogSpaceDO spaceDO = logSpaceDao.queryById(logStoreDO.getSpaceId()); - searchSaveDTO.setSpaceName(spaceDO.getSpaceName()); - result.add(searchSaveDTO); - } - } - if (storeId != null) { - result = result.stream() - .filter(searchSaveDTO -> Objects.equals(storeId, searchSaveDTO.getStoreId())) - .collect(Collectors.toList()); - } - return Result.success(result); - } - - public SearchSaveDTO getById(Long id) { - return SearchSaveConvert.INSTANCE.fromDO(logSearchSaveMapper.selectById(id)); - } - - public Result save(SearchSaveInsertCmd cmd) { - if (cmd.getSort() == null) { - return Result.failParam("The classification field sort cannot be empty"); - } - switch (FavouriteSearchEnum.queryByCode(cmd.getSort())) { - case TEXT: - if (isRepeatName(cmd.getName())) { - return Result.failParam("Names cannot be duplicated"); - } - break; - case STORE: - Integer isMyFavouriteStore = logSearchSaveMapper.isMyFavouriteStore(MoneUserContext.getCurrentUser().getUser(), cmd.getStoreId()); - if (isMyFavouriteStore >= 1) { - return Result.failParam("Bookmarked"); - } - case TAIL: - Integer isMyFavouriteTail = logSearchSaveMapper.isMyFavouriteTail(MoneUserContext.getCurrentUser().getUser(), cmd.getTailId()); - if (isMyFavouriteTail >= 1) { - return Result.failParam("Bookmarked"); - } - break; - } - - MilogLogSearchSaveDO logSearchSaveDO = SearchSaveConvert.INSTANCE.toDO(cmd); - long current = System.currentTimeMillis(); - logSearchSaveDO.setCreateTime(current); - logSearchSaveDO.setUpdateTime(current); - String user = MoneUserContext.getCurrentUser().getUser(); - logSearchSaveDO.setCreator(user); - logSearchSaveDO.setUpdater(user); - Integer maxOrder = logSearchSaveMapper.getMaxOrder(user, cmd.getSort()); - logSearchSaveDO.setOrderNum(maxOrder == null ? 100 : maxOrder + 100); - int insert = logSearchSaveMapper.insert(logSearchSaveDO); - return Result.success(insert); - } - - public Result update(SearchSaveUpdateCmd cmd) { - MilogLogSearchSaveDO milogLogSearchSaveDO = logSearchSaveMapper.selectById(cmd.getId()); - if (milogLogSearchSaveDO == null) { - return Result.failParam("Data not found"); - } - if (!cmd.getName().equals(milogLogSearchSaveDO.getName()) && isRepeatName(cmd.getName())) { - return Result.failParam("Names cannot be duplicated"); - } - milogLogSearchSaveDO.setName(cmd.getName()); - milogLogSearchSaveDO.setQueryText(cmd.getQueryText()); - milogLogSearchSaveDO.setIsFixTime(cmd.getIsFixTime()); - milogLogSearchSaveDO.setStartTime(cmd.getStartTime()); - milogLogSearchSaveDO.setEndTime(cmd.getEndTime()); - milogLogSearchSaveDO.setCommon(cmd.getCommon()); - milogLogSearchSaveDO.setUpdateTime(System.currentTimeMillis()); - milogLogSearchSaveDO.setUpdater(MoneUserContext.getCurrentUser().getUser()); - int i = logSearchSaveMapper.updateById(milogLogSearchSaveDO); - return Result.success(i); - } - - public Result removeById(Long id) { - int i = logSearchSaveMapper.removeById(id); - return Result.success(i); - } - - private boolean isRepeatName(String name) { - Long count = logSearchSaveMapper.countByStoreAndName(name, MoneUserContext.getCurrentUser().getUser()); - return count >= 1; - } - - public Result swapOrder(Long idFrom, Long idTo) { - MilogLogSearchSaveDO from = logSearchSaveMapper.selectById(idFrom); - MilogLogSearchSaveDO to = logSearchSaveMapper.selectById(idTo); - int t = from.getOrderNum(); - from.setOrderNum(to.getOrderNum()); - to.setOrderNum(t); - int fromRes = logSearchSaveMapper.updateById(from); - int toRes = logSearchSaveMapper.updateById(to); - return Result.success(fromRes + toRes == 2); - } - - public Result deFavourite(Integer sort, Long id) { - if (sort == null || id == null) { - Result.failParam("The parameter cannot be empty"); - } - Map paramMap = new HashMap<>(); - switch (FavouriteSearchEnum.queryByCode(sort)) { - case STORE: - paramMap.put("store_id", id); - break; - case TAIL: - paramMap.put("tail_id", id); - break; - default: - return Result.failParam("Invalid sort field"); - } - paramMap.put("creator", MoneUserContext.getCurrentUser().getUser()); - paramMap.put("sort", sort); - int res = logSearchSaveMapper.deleteByMap(paramMap); - return res == 1 ? Result.success() : Result.fail(CommonError.ParamsError); - } - - public Result> storeTree(KeywordPageParam keywordPageParam) { - List dtoList = new CopyOnWriteArrayList<>(); - List spaceDTOList = new ArrayList<>(); - - com.xiaomi.youpin.infra.rpc.Result> userPermSpace = spaceAuthService.getUserPermSpace(keywordPageParam.getKeyword(), keywordPageParam.getPageNum(), keywordPageParam.getPageSize()); - - if (userPermSpace.getCode() != 0) { - return Result.fail(CommonError.UNAUTHORIZED); - } - if (CollectionUtils.isEmpty(userPermSpace.getData().getList())) { - return Result.success(dtoList); - } - List spaceDTOListPage = MilogSpaceConvert.INSTANCE.fromTpcPage(userPermSpace.getData()).getList(); - spaceDTOList.addAll(spaceDTOListPage); - if (CollectionUtils.isEmpty(spaceDTOList)) { - return Result.success(dtoList); - } - List spaceIdList = spaceDTOList.stream().map(MilogSpaceDTO::getId).collect(Collectors.toList()); - List storeList = store.getStoreList(spaceIdList, keywordPageParam.getKeyword()); - Map> spaceStoreMap = new HashMap<>(); - if (storeList != null && !storeList.isEmpty()) { - for (MilogLogStoreDO store : storeList) { - if (spaceStoreMap.containsKey(store.getSpaceId())) { - spaceStoreMap.get(store.getSpaceId()).add(store); - } else { - spaceStoreMap.put(store.getSpaceId(), Lists.newArrayList(store)); - } - } - } - List favouriteList = searchSaveMapper.selectByCreator(MoneUserContext.getCurrentUser().getUser(), FavouriteSearchEnum.STORE.getCode()); - Set favouriteStoreIdSet; - if (favouriteList != null && !favouriteList.isEmpty()) { - favouriteStoreIdSet = favouriteList.stream().map(SearchSaveDTO::getStoreId).collect(Collectors.toSet()); - } else { - favouriteStoreIdSet = new HashSet<>(); - } - spaceDTOList.parallelStream().forEach(space -> { - List children; - SpaceTreeFavouriteDTO dto = new SpaceTreeFavouriteDTO(); - dto.setValue(space.getId()); - dto.setLabel(space.getSpaceName()); - - children = new ArrayList<>(); - List storeFerryList = spaceStoreMap.get(space.getId()); - if (storeFerryList != null && !storeFerryList.isEmpty()) { - for (MilogLogStoreDO storeDO : storeFerryList) { - children.add(StoreTreeDTO.Of(storeDO.getId(), storeDO.getLogstoreName(), favouriteStoreIdSet.contains(storeDO.getId()) ? 1 : 0)); - } - } - dto.setChildren(children); - dtoList.add(dto); - }); - return Result.success(dtoList); - } - - public Result initOrder(String key) { - if (!"384384".equals(key)) { - return null; - } - List saveList = logSearchSaveMapper.getAll(); - Map orderMap = new HashMap<>(); - if (saveList != null && !saveList.isEmpty()) { - log.info("save List size is #{}", saveList.size()); - for (MilogLogSearchSaveDO save : saveList) { - if (orderMap.containsKey(save.getCreator())) { - Integer order = orderMap.get(save.getCreator()); - save.setOrderNum(order + 100); - orderMap.put(save.getCreator(), order + 100); - } else { - save.setOrderNum(100); - orderMap.put(save.getCreator(), 100); - } - logSearchSaveMapper.updateById(save); - } - } - - return null; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogMachineServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogMachineServiceImpl.java deleted file mode 100644 index 467735bca..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogMachineServiceImpl.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.exception.CommonError; -import com.xiaomi.mone.log.manager.dao.MilogMachineDao; -import com.xiaomi.mone.log.manager.model.bo.MachineParamParam; -import com.xiaomi.mone.log.manager.model.bo.MachineQueryParam; -import com.xiaomi.mone.log.manager.model.pojo.MiLogMachine; -import com.xiaomi.mone.log.manager.service.MilogMachineService; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; -import java.util.HashMap; -import java.util.Map; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/16 11:32 - */ -@Slf4j -@Service -public class MilogMachineServiceImpl implements MilogMachineService { - - @Resource - MilogMachineDao milogMachineDao; - - @Override - public Result addMachineInfo(MachineParamParam param) { - if (StringUtils.isEmpty(param.getIp())) { - return Result.fail(CommonError.ParamsError.getCode(), "The IP address cannot be empty"); - } - if (null == param.getType()) { - return Result.fail(CommonError.ParamsError.getCode(), "Machine type cannot be empty"); - } - milogMachineDao.insert(param); - return Result.success(); - } - - @Override - public Result deleteMachineInfo(Long id) { - if (null == id) { - return Result.fail(CommonError.ParamsError.getCode(), "ID cannot be empty"); - } - milogMachineDao.deleteMachineInfo(id); - return Result.success(); - } - - @Override - public Result> queryMachineByPage(MachineQueryParam param) { - Map result = new HashMap<>(); - result.put("milogSpaceList", milogMachineDao.queryMachinePage(param)); - result.put("total", milogMachineDao.queryMachinePageCount(param)); - result.put("page", param.getPageNum()); - result.put("pageSize", param.getPageSize()); - return Result.success(result); - } - - @Override - public Result queryMachineInfoById(Long id) { - if (null == id) { - return Result.fail(CommonError.ParamsError.getCode(), "ID cannot be empty"); - } - MiLogMachine miLogMachine = milogMachineDao.queryById(id); - return Result.success((MachineParamParam) miLogMachine); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogMiddlewareConfigServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogMiddlewareConfigServiceImpl.java deleted file mode 100644 index 167ef627a..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogMiddlewareConfigServiceImpl.java +++ /dev/null @@ -1,887 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import cn.hutool.core.bean.BeanUtil; -import cn.hutool.core.collection.CollUtil; -import cn.hutool.core.collection.CollectionUtil; -import com.baomidou.mybatisplus.core.conditions.Wrapper; -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.baomidou.mybatisplus.core.toolkit.Wrappers; -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.api.enums.*; -import com.xiaomi.mone.log.api.model.bo.MiLogResource; -import com.xiaomi.mone.log.api.model.bo.ResourcePage; -import com.xiaomi.mone.log.api.model.vo.EsIndexVo; -import com.xiaomi.mone.log.api.model.vo.ResourceInfo; -import com.xiaomi.mone.log.api.model.vo.ResourceUserSimple; -import com.xiaomi.mone.log.api.model.vo.ValueKeyObj; -import com.xiaomi.mone.log.common.Constant; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.exception.CommonError; -import com.xiaomi.mone.log.manager.bootstrap.LogStoragePlugin; -import com.xiaomi.mone.log.manager.common.context.MoneUserContext; -import com.xiaomi.mone.log.manager.common.exception.MilogManageException; -import com.xiaomi.mone.log.manager.common.validation.ResourceValidation; -import com.xiaomi.mone.log.manager.dao.MilogAppMiddlewareRelDao; -import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; -import com.xiaomi.mone.log.manager.dao.MilogMiddlewareConfigDao; -import com.xiaomi.mone.log.manager.domain.LogTail; -import com.xiaomi.mone.log.manager.mapper.MilogEsClusterMapper; -import com.xiaomi.mone.log.manager.mapper.MilogEsIndexMapper; -import com.xiaomi.mone.log.manager.model.bo.MiddlewareAddParam; -import com.xiaomi.mone.log.manager.model.bo.MiddlewareQueryParam; -import com.xiaomi.mone.log.manager.model.bo.MiddlewareUpdateParam; -import com.xiaomi.mone.log.manager.model.dto.MiddlewareConfigDTO; -import com.xiaomi.mone.log.manager.model.page.PageInfo; -import com.xiaomi.mone.log.manager.model.pojo.*; -import com.xiaomi.mone.log.manager.service.BaseService; -import com.xiaomi.mone.log.manager.service.MilogMiddlewareConfigService; -import com.xiaomi.mone.log.manager.service.extension.resource.ResourceExtensionService; -import com.xiaomi.mone.log.manager.service.extension.resource.ResourceExtensionServiceFactory; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; -import org.nutz.dao.Cnd; -import org.nutz.dao.Condition; -import org.nutz.dao.pager.Pager; -import org.nutz.dao.util.cri.IsNull; - -import javax.annotation.Resource; -import java.time.Instant; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.*; -import static com.xiaomi.mone.log.manager.common.ManagerConstant.RESOURCE_NOT_INITIALIZED_MESSAGE; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/22 11:42 - */ -@Service -@Slf4j -public class MilogMiddlewareConfigServiceImpl extends BaseService implements MilogMiddlewareConfigService { - - @Resource - private MilogMiddlewareConfigDao milogMiddlewareConfigDao; - @Resource - private MilogAppMiddlewareRelDao milogAppMiddlewareRelDao; - @Resource - private ResourceValidation resourceValidation; - @Resource - private MilogEsClusterMapper milogEsClusterMapper; - @Resource - private MilogEsIndexMapper milogEsIndexMapper; - @Resource - private MilogLogstoreDao logstoreDao; - @Resource - private LogStoragePlugin esPlugin; - @Resource - private LogTail logTail; - - private ResourceExtensionService resourceExtensionService; - - private List accurateTypes = Arrays.stream(MQSourceEnum.values()).map(MQSourceEnum::getCode).toList(); - - public void init() { - resourceExtensionService = ResourceExtensionServiceFactory.getResourceExtensionService(); - } - - @Override - public PageInfo queryMiddlewareConfigPage(MiddlewareQueryParam middlewareQueryParam) { - Condition cnd = handleParamToCondition(middlewareQueryParam); - List milogMiddlewareConfigs = milogMiddlewareConfigDao.queryMiddlewareConfigByCondition(cnd, new Pager(middlewareQueryParam.getPage(), middlewareQueryParam.getPageSize())); - Integer count = milogMiddlewareConfigDao.queryMiddlewareConfigCountByCondition(cnd); - return new PageInfo<>(middlewareQueryParam.getPage(), middlewareQueryParam.getPageSize(), count, milogMiddlewareConfigs); - } - - @Override - public Result addMiddlewareConfig(MiddlewareAddParam middlewareAddParam) { - List checkMsg = preCheckMiddlewareConfig(middlewareAddParam); - if (CollectionUtils.isNotEmpty(checkMsg)) { - return Result.failParam(checkMsg.stream().collect(Collectors.joining(","))); - } - List milogMiddlewareConfigs = milogMiddlewareConfigDao.queryMiddlewareConfigByCondition(Cnd.where("alias", "=", middlewareAddParam.getAlias()), null); - if (CollectionUtils.isNotEmpty(milogMiddlewareConfigs)) { - return Result.failParam("The alias already exists, please modify it and save it"); - } - //There will only be one default value - if (accurateTypes.contains(middlewareAddParam.getType())) { - MilogMiddlewareConfig config = milogMiddlewareConfigDao.queryDefaultMiddlewareConfig(); - if (null != config && middlewareAddParam.getIsDefault() == 1) { - return Result.failParam("There will only be one default value, please turn off the other default defaults"); - } - } - milogMiddlewareConfigDao.addMiddlewareConfig(AddParamToAddEntity(middlewareAddParam)); - return Result.success(); - } - - @Override - public Result updateMiddlewareConfig(MiddlewareUpdateParam middlewareUpdateParam) { - List checkMsg = preCheckUpdateMiddlewareConfig(middlewareUpdateParam); - if (CollectionUtils.isNotEmpty(checkMsg)) { - return Result.failParam(checkMsg.stream().collect(Collectors.joining(","))); - } - MilogMiddlewareConfig milogMiddlewareConfig = milogMiddlewareConfigDao.queryById(middlewareUpdateParam.getId()); - if (null == milogMiddlewareConfig) { - return Result.failParam("The modified object does not exist, make sure the id is correct"); - } - List milogMiddlewareConfigs = milogMiddlewareConfigDao.queryMiddlewareConfigByCondition(Cnd.where("alias", "=", middlewareUpdateParam.getAlias()), null); - if (CollectionUtils.isNotEmpty(milogMiddlewareConfigs)) { - List middlewareConfigs = milogMiddlewareConfigs.stream().filter(config -> config.getId().longValue() != middlewareUpdateParam.getId().longValue()).collect(Collectors.toList()); - if (CollectionUtils.isNotEmpty(middlewareConfigs)) { - return Result.failParam("The alias already exists, please modify it and save it"); - } - } - //There will only be one default value - if (accurateTypes.contains(middlewareUpdateParam.getType())) { - MilogMiddlewareConfig config = milogMiddlewareConfigDao.queryDefaultMiddlewareConfig(); - if (null != config && middlewareUpdateParam.getIsDefault() == 1 && !config.getId().equals(middlewareUpdateParam.getId())) { - return Result.failParam("There will only be one default value, please turn off the other default values and modify it"); - } - } - milogMiddlewareConfigDao.updateMiddlewareConfig(updateParamToUpdateEntity(middlewareUpdateParam, milogMiddlewareConfig)); - return Result.success(); - } - - @Override - public Result deleteMiddlewareConfig(Long id) { - if (null == id) { - return Result.failParam("id can not null"); - } - MilogMiddlewareConfig milogMiddlewareConfig = milogMiddlewareConfigDao.queryById(id); - if (null == milogMiddlewareConfig) { - return Result.failParam("The deleted object does not exist, make sure the Id is correct"); - } - List milogAppMiddlewareRels = milogAppMiddlewareRelDao.queryByCondition(null, id, null); - if (CollectionUtils.isNotEmpty(milogAppMiddlewareRels)) { - return Result.failParam("The associated tail exists, make sure the tail has changed"); - } - milogMiddlewareConfigDao.deleteMiddlewareConfig(id); - return Result.success(); - } - - @Override - public List queryMiddlewareConfigList() { - return milogMiddlewareConfigDao.queryAll(); - } - - @Override - public Result queryMiddlewareConfigById(Long id) { - if (null == id) { - return Result.failParam("id can not null"); - } - MilogMiddlewareConfig config = milogMiddlewareConfigDao.queryById(id); - MiddlewareConfigDTO middlewareConfigDTO = new MiddlewareConfigDTO(); - BeanUtil.copyProperties(config, middlewareConfigDTO); - middlewareConfigDTO.setTypes(Arrays.asList(config.getType(), config.getRegionEn())); - return Result.success(middlewareConfigDTO); - } - - /** - * @param resourcePage - * @return - */ - @Override - public PageInfo queryResourceWithTab(ResourcePage resourcePage) { - if (null == resourcePage.getResourceCode()) { - return PageInfo.emptyPageInfo(); - } - Cnd mqCnd = buildMqQueryCnd(resourcePage); - - List milogMiddlewareConfigs = milogMiddlewareConfigDao.queryMiddlewareConfigByCondition(mqCnd, null); - - List userShowAuthorityConfigs = resourceExtensionService.userShowAuthority(milogMiddlewareConfigs); - - List pageList = CollectionUtil.page(resourcePage.getPage() - 1, resourcePage.getPageSize(), userShowAuthorityConfigs); - - Integer mqResourceTotal = userShowAuthorityConfigs.size(); - - Long esResourceTotal = queryStorageResource(pageList, resourcePage); - - List resourceInfos = pageList.stream().map(MilogMiddlewareConfig::configToResourceVO).collect(Collectors.toList()); - - return new PageInfo(resourcePage.getPage(), resourcePage.getPageSize(), mqResourceTotal + esResourceTotal.intValue(), resourceInfos); - } - - private Cnd buildMqQueryCnd(ResourcePage resourcePage) { - Cnd cnd = Cnd.NEW(); - if (Objects.equals(ResourceEnum.MQ.getCode(), resourcePage.getResourceCode())) { - cnd.andEX("type", "in", resourceExtensionService.getMqResourceCodeList()); - } - if (null != resourcePage.getRegionEnCode()) { - cnd.andEX("region_en", EQUAL_OPERATE, resourcePage.getRegionEnCode()); - } - if (StringUtils.isNotBlank(resourcePage.getAliasName())) { - cnd.andEX("alias", LIKE_OPERATE, String.format("%s%s%s", "%", resourcePage.getAliasName(), "%")); - } - if (Objects.equals(ResourceEnum.STORAGE.getCode(), resourcePage.getResourceCode())) { - cnd.and(new IsNull("id")); - } - return cnd; - } - - /** - * Dealing with ES resources, due to historical principles, ES resources are in separate tables and we need to combine data - */ - private Long queryStorageResource(List milogMiddlewareConfigs, ResourcePage resourcePage) { - Integer count = 0; - if (Objects.equals(ResourceEnum.STORAGE.getCode(), resourcePage.getResourceCode())) { - - List clusterDOList = milogEsClusterMapper.selectList(generateEsQueryWrapper(resourcePage)); - List configEsList = clusterDOList.stream().map(MilogEsClusterDO::miLogEsResourceToConfig).toList(); - configEsList = resourceExtensionService.userShowAuthority(configEsList); - List pageList = CollectionUtil.page(resourcePage.getPage() - 1, resourcePage.getPageSize(), configEsList); - milogMiddlewareConfigs.addAll(pageList); - - count = configEsList.size(); - } - return count.longValue(); - } - - private Wrapper generateEsQueryWrapper(ResourcePage resourcePage) { - QueryWrapper queryWrapper = new QueryWrapper(); - if (StringUtils.isNotBlank(resourcePage.getRegionEnCode())) { - queryWrapper.eq("area", resourcePage.getRegionEnCode()); - } - if (StringUtils.isNotBlank(resourcePage.getAliasName())) { - queryWrapper.like("name", resourcePage.getAliasName()); - } - resourceExtensionService.filterEsQueryWrapper(queryWrapper); - return queryWrapper; - } - - @Override - public Result resourceOperate(MiLogResource miLogResource) { - String errInfos = resourceValidation.resourceOperateValid(miLogResource); - if (StringUtils.isNotBlank(errInfos)) { - return Result.failParam(errInfos); - } - OperateEnum operateEnum = OperateEnum.queryByCode(miLogResource.getOperateCode()); - checkAlias(miLogResource, operateEnum); - - switch (operateEnum) { - case ADD_OPERATE: - addResource(miLogResource); - break; - - case UPDATE_OPERATE: - updateResource(miLogResource); - break; - - case DELETE_OPERATE: - deleteResource(miLogResource.getResourceCode(), miLogResource.getId()); - break; - - default: - break; - } - return Result.success(SUCCESS_MESSAGE); - } - - private void checkEsAddressPortOperate(MiLogResource miLogResource) { - String serviceUrl = miLogResource.getServiceUrl(); - if (StringUtils.isBlank(serviceUrl)) { - return; - } - - int index = serviceUrl.lastIndexOf(":"); - if (index == -1) { - serviceUrl = String.format("%s:%s", serviceUrl, "80"); - } else { - String portStr = serviceUrl.substring(index + 1); - if (StringUtils.isBlank(portStr)) { - serviceUrl = String.format("%s%s", serviceUrl, "80"); - } - } - miLogResource.setServiceUrl(serviceUrl); - } - - private void deleteResource(Integer resourceCode, Long id) { - if (Objects.equals(ResourceEnum.STORAGE.getCode(), resourceCode)) { - milogEsClusterMapper.deleteById(id); - deleteEsIndex(id); - return; - } - Result result = deleteMiddlewareConfig(id); - if (result.getCode() != CommonError.Success.getCode()) { - throw new MilogManageException(result.getMessage()); - } - } - - @Override - public String synchronousResourceLabel(Long id) { - synchronousMqResourceLabel(id); - synchronousEsResourceLabel(id); - return SUCCESS_MESSAGE; - } - - private void synchronousMqResourceLabel(Long id) { - List milogMiddlewareConfigs = Lists.newArrayList(); - if (null != id) { - milogMiddlewareConfigs.add(milogMiddlewareConfigDao.queryById(id)); - } - if (CollectionUtils.isEmpty(milogMiddlewareConfigs)) { - milogMiddlewareConfigs = milogMiddlewareConfigDao.queryAll(); - } - milogMiddlewareConfigs.forEach(this::updateMqResourceLabel); - } - - private void synchronousEsResourceLabel(Long id) { - List esClusterDOS = Lists.newArrayList(); - if (null != id) { - esClusterDOS.add(milogEsClusterMapper.selectById(id)); - } - if (CollectionUtils.isEmpty(esClusterDOS)) { - esClusterDOS = milogEsClusterMapper.selectList(new QueryWrapper<>()); - } - esClusterDOS.forEach(this::updateEsResourceLabel); - } - - /** - * Resource information of the current user - * 1. Whether to initialize - * 2. Whether to display - * 3. Resource list - * - * @return - */ - @Override - public ResourceUserSimple userResourceList(String regionCode, Integer logTypeCode) { - if (StringUtils.isBlank(regionCode)) { - throw new MilogManageException("Region Code cannot be empty"); - } - if (null == logTypeCode) { - throw new MilogManageException("Log Type Code cannot be empty"); - } - ResourceUserSimple resourceUserSimple = new ResourceUserSimple(); - - if (resourceExtensionService.userResourceListPre(logTypeCode)) { - resourceUserSimple.setInitializedFlag(Boolean.TRUE); - resourceUserSimple.setShowFlag(Boolean.FALSE); - return resourceUserSimple; - } - - final Boolean initializedStatus = queryResourceInitialized(resourceUserSimple, regionCode, logTypeCode); - final boolean showStatus = resourceExtensionService.resourceShowStatusFlag(resourceUserSimple); - - if (initializedStatus && showStatus) { - - List milogMiddlewareConfigs = milogMiddlewareConfigDao.queryByResourceCodes(resourceExtensionService.getMqResourceCodeList(), regionCode); - queryCurrentUserResourceList(resourceUserSimple, milogMiddlewareConfigs, resourceExtensionService.getResourceCode()); - - List middlewareConfigEss = getESConfigs(regionCode); - queryCurrentUserResourceList(resourceUserSimple, middlewareConfigEss, MiddlewareEnum.ELASTICSEARCH.getCode()); - } - return resourceUserSimple; - } - - @NotNull - public List getESConfigs(String regionCode) { - QueryWrapper queryWrapper = new QueryWrapper<>().eq("area", regionCode); - List esClusterDOS = milogEsClusterMapper.selectList(queryWrapper); - List middlewareConfigEss = esClusterDOS.stream().map(MilogEsClusterDO::miLogEsResourceToConfig).collect(Collectors.toList()); - return middlewareConfigEss; - } - - /** - * Query whether the resources under the current user have been initialized - */ - private Boolean queryResourceInitialized(ResourceUserSimple configResource, String regionCode, Integer logTypeCode) { - List milogMiddlewareConfigs = milogMiddlewareConfigDao.queryByResourceCodes(resourceExtensionService.getMqResourceCodeList(), regionCode); - List middlewareMqConfigs = resourceExtensionService.currentUserConfigFilter(milogMiddlewareConfigs); - - List esClusterDOS = milogEsClusterMapper.selectList(Wrappers.lambdaQuery()); - List middlewareConfigEss = esClusterDOS.stream().map(logEsClusterDO -> { - MilogMiddlewareConfig milogMiddlewareConfig = MilogEsClusterDO.miLogEsResourceToConfig(logEsClusterDO); - return milogMiddlewareConfig; - }).collect(Collectors.toList()); - - List middlewareEsConfigs = resourceExtensionService.currentUserConfigFilter(middlewareConfigEss); - - List milogEsIndexDOS = Lists.newArrayList(); - if (CollectionUtils.isNotEmpty(middlewareEsConfigs)) { - milogEsIndexDOS = queryMilogEsIndex(middlewareEsConfigs.stream().map(MilogMiddlewareConfig::getId).collect(Collectors.toList()), logTypeCode); - } - - if (resourceExtensionService.resourceNotRequiredInit(logTypeCode, middlewareMqConfigs, middlewareEsConfigs, milogEsIndexDOS)) { - configResource.setInitializedFlag(Boolean.TRUE); - return Boolean.TRUE; - } - String msg = ""; - if (CollectionUtils.isEmpty(middlewareMqConfigs)) { - msg = "MQ resource is not initialized"; - } - if (CollectionUtils.isEmpty(middlewareEsConfigs) || CollectionUtils.isEmpty(milogEsIndexDOS)) { - msg = "The ES resource is not initialized or the log index of the current type is not initialized"; - } - configResource.setInitializedFlag(Boolean.FALSE); - configResource.setNotInitializedMsg(String.format("%s%s", msg, RESOURCE_NOT_INITIALIZED_MESSAGE)); - return Boolean.FALSE; - } - - private List queryMilogEsIndex(List clusterIds, Integer logTyCode) { - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.in("cluster_id", clusterIds); - queryWrapper.eq("log_type", logTyCode); - return milogEsIndexMapper.selectList(queryWrapper); - } - - /** - * Query the list of resources for the current user - * - * @param configResource - */ - private void queryCurrentUserResourceList(ResourceUserSimple configResource, List milogMiddlewareConfigs, Integer code) { - milogMiddlewareConfigs = resourceExtensionService.currentUserConfigFilter(milogMiddlewareConfigs); - List valueKeyObjs = milogMiddlewareConfigs.stream().map(milogMiddlewareConfig -> new ValueKeyObj(milogMiddlewareConfig.getId(), milogMiddlewareConfig.getAlias())).collect(Collectors.toList()); - if (Objects.equals(MiddlewareEnum.ELASTICSEARCH.getCode(), code)) { - configResource.setStorageResourceList(valueKeyObjs); - return; - } - configResource.setMqResourceList(valueKeyObjs); - } - - @Override - public ResourceInfo resourceDetail(Integer resourceCode, Long id) { - if (Objects.equals(ResourceEnum.STORAGE.getCode(), resourceCode)) { - return wrapEsResourceInfo(id); - } - MilogMiddlewareConfig milogMiddlewareConfig = milogMiddlewareConfigDao.queryById(id); - if (null != milogMiddlewareConfig) { - return milogMiddlewareConfig.configToResourceVO(); - } - return ResourceInfo.builder().build(); - } - - @Override - public MilogMiddlewareConfig queryMiddlewareConfigDefault(String regionCode) { - List milogMiddlewareConfigs = milogMiddlewareConfigDao.queryByResourceCodes(resourceExtensionService.getMqResourceCodeList(), regionCode); - List middlewareMqConfigs = resourceExtensionService.currentUserConfigFilter(milogMiddlewareConfigs); - - if (CollectionUtils.isNotEmpty(middlewareMqConfigs)) { - return middlewareMqConfigs.get(middlewareMqConfigs.size() - 1); - } - return null; - } - - private ResourceInfo wrapEsResourceInfo(Long id) { - MilogEsClusterDO esClusterDO = milogEsClusterMapper.selectById(id); - if (null != esClusterDO) { - List milogEsIndexDOS = milogEsIndexMapper.selectList(new QueryWrapper().eq("cluster_id", esClusterDO.getId())); - List multipleEsIndex = milogEsIndexDOS.stream().map(MilogEsIndexDO::getLogType) - .distinct().sorted((val1, val2) -> null != val1 && null != val2 ? val1.compareTo(val2) : 0).map(getIntegerEsIndexVoFunction(milogEsIndexDOS)).collect(Collectors.toList()); - return esClusterDO.configToResourceVO(multipleEsIndex); - } - return ResourceInfo.builder().build(); - } - - @NotNull - private Function getIntegerEsIndexVoFunction(List milogEsIndexDOS) { - return logTypeCode -> { - EsIndexVo esIndexVo = new EsIndexVo(); - esIndexVo.setLogTypeCode(logTypeCode); - esIndexVo.setLogTypeName(LogTypeEnum.queryNameByType(logTypeCode)); - List esIndexList = milogEsIndexDOS.stream().filter(milogEsIndexDO -> Objects.equals(logTypeCode, milogEsIndexDO.getLogType())).map(MilogEsIndexDO::getIndexName).collect(Collectors.toList()); - esIndexVo.setEsIndexList(esIndexList); - return esIndexVo; - }; - } - - /** - * Update the label for each column of resources - * - * @param milogMiddlewareConfig - */ - private void updateMqResourceLabel(MilogMiddlewareConfig milogMiddlewareConfig) { - List labels = milogMiddlewareConfig.getLabels(); - if (Constant.DEFAULT_OPERATOR.equals(milogMiddlewareConfig.getUpdater())) { - return; - } - List newLabels = mergeUserAndResourceLabels(milogMiddlewareConfig.getCreator(), milogMiddlewareConfig.getUpdater(), labels); - milogMiddlewareConfig.setLabels(newLabels); - milogMiddlewareConfigDao.updateMiddlewareConfig(milogMiddlewareConfig); - } - - private void updateEsResourceLabel(MilogEsClusterDO clusterDO) { - List labels = clusterDO.getLabels(); - if (StringUtils.isBlank(clusterDO.getUpdater()) && StringUtils.isBlank(clusterDO.getCreator())) { - return; - } - List newLabels = mergeUserAndResourceLabels(clusterDO.getCreator(), StringUtils.isBlank(clusterDO.getUpdater()) ? clusterDO.getCreator() : clusterDO.getUpdater(), labels); - clusterDO.setLabels(newLabels); - milogEsClusterMapper.updateById(clusterDO); - } - - private List mergeUserAndResourceLabels(String creatorUId, String updaterUId, List existLabels) { - List resourceDeptLabels = resourceExtensionService.generateResourceLabels(updaterUId); - if (!Objects.equals(creatorUId, updaterUId) && - CollectionUtils.isNotEmpty(existLabels) && !CollUtil.containsAll(existLabels, resourceDeptLabels)) { - existLabels.addAll(resourceDeptLabels); - } - return existLabels; - } - - /** - * Add resources - * 1.If MQ is MQ, create 3 public topics and start tag filtering - * 2.Label processing - * 2.Information is stored - * - * @param miLogResource - */ - private void addResource(MiLogResource miLogResource) { - List resourceDeptLabels = resourceExtensionService.generateResourceLabels(MoneUserContext.getCurrentUser().getUser()); - resourceDeptLabels.addAll(miLogResource.getLabels()); - miLogResource.setLabels(resourceDeptLabels); - resourceExtensionService.addResourcePreProcessing(resourceDeptLabels, miLogResource); - - if (Objects.equals(ResourceEnum.STORAGE.getCode(), miLogResource.getResourceCode())) { - if (StringUtils.equalsIgnoreCase(LogStorageTypeEnum.ELASTICSEARCH.name(), miLogResource.getStorageType())) { - checkEsAddressPortOperate(miLogResource); - addEsResourceInfo(miLogResource); - return; - } - addOtherResourceInfo(miLogResource); - return; - } - - resourceExtensionService.addResourceMiddleProcessing(miLogResource); - - MilogMiddlewareConfig milogMiddlewareConfig = MilogMiddlewareConfig.miLogMqResourceToConfig(miLogResource); - - wrapBaseCommon(milogMiddlewareConfig, OperateEnum.ADD_OPERATE); - - resourceExtensionService.addResourcePostProcessing(milogMiddlewareConfig); - milogMiddlewareConfigDao.addMiddlewareConfig(milogMiddlewareConfig); - } - - private void addOtherResourceInfo(MiLogResource miLogResource) { - MilogEsClusterDO esClusterDO = MilogEsClusterDO.miLogEsResourceToConfig(miLogResource); - wrapBaseCommon(esClusterDO, OperateEnum.ADD_OPERATE); - resourceExtensionService.addEsResourcePreProcessing(esClusterDO); - milogEsClusterMapper.insert(esClusterDO); - - //Add the ES client - esPlugin.initializeLogStorage(esClusterDO); - } - - private void addEsResourceInfo(MiLogResource miLogResource) { - MilogEsClusterDO esClusterDO = MilogEsClusterDO.miLogEsResourceToConfig(miLogResource); - wrapBaseCommon(esClusterDO, OperateEnum.ADD_OPERATE); - resourceExtensionService.addEsResourcePreProcessing(esClusterDO); - milogEsClusterMapper.insert(esClusterDO); - - addEsIndex(esClusterDO.getId(), miLogResource.getMultipleEsIndex()); - //Add the ES client - esPlugin.initializeLogStorage(esClusterDO); - } - - private void checkAlias(MiLogResource resource, OperateEnum operateEnum) { - if (Objects.equals(ResourceEnum.STORAGE.getCode(), resource.getResourceCode())) { - List logEsClusterDOS = milogEsClusterMapper.selectByAlias(resource.getAlias()); - if (operateEnum == OperateEnum.ADD_OPERATE && CollectionUtils.isNotEmpty(logEsClusterDOS)) { - throw new MilogManageException("alias has exists,please refill"); - } - if (operateEnum == OperateEnum.UPDATE_OPERATE && CollectionUtils.isNotEmpty(logEsClusterDOS)) { - MilogEsClusterDO logEsClusterDO = logEsClusterDOS.get(logEsClusterDOS.size() - 1); - if (!Objects.equals(resource.getId(), logEsClusterDO.getId())) { - throw new MilogManageException("alias has exists,please refill"); - } - } - } - - if (MiddlewareEnum.ROCKETMQ.getCode().equals(resource.getResourceCode())) { - List middlewareConfigs = milogMiddlewareConfigDao.selectByAlias(resource.getAlias()); - if (operateEnum == OperateEnum.ADD_OPERATE && CollectionUtils.isNotEmpty(middlewareConfigs)) { - throw new MilogManageException("alias has exists,please refill"); - } - if (operateEnum == OperateEnum.UPDATE_OPERATE && CollectionUtils.isNotEmpty(middlewareConfigs)) { - MilogMiddlewareConfig middlewareConfig = middlewareConfigs.get(middlewareConfigs.size() - 1); - if (!Objects.equals(resource.getId(), middlewareConfig.getId())) { - throw new MilogManageException("alias has exists,please refill"); - } - } - } - } - - private void addEsIndex(Long esClusterId, List multipleEsIndex) { - List milogEsIndexDOS = Lists.newArrayList(); - if (CollectionUtils.isNotEmpty(multipleEsIndex)) { - for (EsIndexVo esIndexVo : multipleEsIndex) { - milogEsIndexDOS.addAll(MilogEsIndexDO.essIndexVoToIndexDO(esClusterId, esIndexVo)); - } - } - for (MilogEsIndexDO milogEsIndexDO : milogEsIndexDOS) { - milogEsIndexMapper.insert(milogEsIndexDO); - } - } - - /** - * Modify the resource - * 1.To compare whether the information in MQ, if so, you need to create 3 public topics and start tag filtering - * 2.Information modification into storage - * - * @param miLogResource - */ - private void updateResource(MiLogResource miLogResource) { - - if (Objects.equals(ResourceEnum.STORAGE.getCode(), miLogResource.getResourceCode())) { - if (StringUtils.equalsIgnoreCase(LogStorageTypeEnum.ELASTICSEARCH.name(), miLogResource.getStorageType())) { - boolean changed = authenticationInChanged(milogEsClusterMapper.selectById(miLogResource.getId()), miLogResource); - checkEsAddressPortOperate(miLogResource); - MilogEsClusterDO esClusterDO = MilogEsClusterDO.miLogEsResourceToConfig(miLogResource); - esClusterDO.setId(miLogResource.getId()); - resourceExtensionService.addEsResourcePreProcessing(esClusterDO); - milogEsClusterMapper.updateById(esClusterDO); - handleEsIndexStore(miLogResource, esClusterDO.getId(), changed); - // Modify the ES client information - esPlugin.initializeLogStorage(esClusterDO); - return; - } - MilogEsClusterDO esClusterDO = MilogEsClusterDO.miLogEsResourceToConfig(miLogResource); - esClusterDO.setId(miLogResource.getId()); - resourceExtensionService.addEsResourcePreProcessing(esClusterDO); - milogEsClusterMapper.updateById(esClusterDO); - // Modify the storage client information - esPlugin.initializeLogStorage(esClusterDO); - return; - - } - - updateCompareMqInfo(miLogResource); - - MilogMiddlewareConfig milogMiddlewareConfig = MilogMiddlewareConfig.miLogMqResourceToConfig(miLogResource); - - buildCommonAttr(miLogResource.getId(), milogMiddlewareConfig); - - milogMiddlewareConfigDao.updateMiddlewareConfig(milogMiddlewareConfig); - } - - private boolean authenticationInChanged(MilogEsClusterDO clusterDO, MiLogResource miLogResource) { - if (!Objects.equals(clusterDO.getConWay(), miLogResource.getConWay())) { - return true; - } - if (Objects.equals(ES_CONWAY_PWD, miLogResource.getConWay())) { - return !Objects.equals(clusterDO.getUser(), miLogResource.getAk()) || !Objects.equals(clusterDO.getPwd(), miLogResource.getSk()); - } - if (Objects.equals(ES_CONWAY_TOKEN, miLogResource.getConWay())) { - return !Objects.equals(clusterDO.getToken(), miLogResource.getEsToken()) || !Objects.equals(clusterDO.getDtCatalog(), miLogResource.getCatalog()) || !Objects.equals(clusterDO.getDtDatabase(), miLogResource.getDatabase()); - } - return false; - } - - /** - * 1.Check if the ES has already been used - * 2.If so, see if the index used has changed - * 3.If it changes, modify - * 4.Otherwise, delete the new one - * - * @param miLogResource - */ - private void handleEsIndexStore(MiLogResource miLogResource, Long esClusterId, boolean changed) { - //Delete the old ES index, add a new ES index (compare if there is already one, there is no incoming one, delete, there is no existing one, and there is a new one that comes in) - handleEsIndex(esClusterId, miLogResource.getMultipleEsIndex()); - if (changed) { - List storeDOS = logstoreDao.queryByEsInfo(miLogResource.getRegionEn(), esClusterId); - if (changed && CollectionUtils.isNotEmpty(storeDOS)) { - //Sync to stream - for (MilogLogStoreDO storeDO : storeDOS) { - logTail.handleStoreTail(storeDO.getId()); - } - } - } - } - - private void handleEsIndex(Long esClusterId, List multipleEsIndex) { - for (EsIndexVo esIndex : multipleEsIndex) { - doHandleEsIndexByLogType(esClusterId, esIndex.getLogTypeCode(), esIndex.getEsIndexList()); - } - } - - private void doHandleEsIndexByLogType(Long esClusterId, Integer logTypeCode, List esIndexList) { - List addExIndexDo; - List delExIndexDo = Lists.newArrayList(); - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("cluster_id", esClusterId); - queryWrapper.eq("log_type", logTypeCode); - List existEsIndexDO = milogEsIndexMapper.selectList(queryWrapper); - if (CollectionUtils.isEmpty(existEsIndexDO)) { - addExIndexDo = esIndexList.stream().map(indexName -> getMilogEsIndexDO(esClusterId, logTypeCode, indexName)).collect(Collectors.toList()); - } else { - for (MilogEsIndexDO milogEsIndexDO : existEsIndexDO) { - if (!esIndexList.contains(milogEsIndexDO.getIndexName())) { - delExIndexDo.add(milogEsIndexDO); - } - } - List existIndexes = existEsIndexDO.stream().map(MilogEsIndexDO::getIndexName).collect(Collectors.toList()); - addExIndexDo = esIndexList.stream().filter(indexName -> !existIndexes.contains(indexName)).map(indexName -> getMilogEsIndexDO(esClusterId, logTypeCode, indexName)).collect(Collectors.toList()); - } - if (CollectionUtils.isNotEmpty(delExIndexDo)) { - milogEsIndexMapper.deleteBatchIds(delExIndexDo.stream().map(MilogEsIndexDO::getId).collect(Collectors.toList())); - } - addExIndex(addExIndexDo); - } - - private void addExIndex(List addExIndexDo) { - for (MilogEsIndexDO milogEsIndexDO : addExIndexDo) { - milogEsIndexMapper.insert(milogEsIndexDO); - } - } - - @NotNull - private MilogEsIndexDO getMilogEsIndexDO(Long esClusterId, Integer logTypeCode, String indexName) { - MilogEsIndexDO milogEsIndexDO = new MilogEsIndexDO(); - milogEsIndexDO.setClusterId(esClusterId); - milogEsIndexDO.setLogType(logTypeCode); - milogEsIndexDO.setIndexName(indexName); - return milogEsIndexDO; - } - - private void buildCommonAttr(Long resourceId, MilogMiddlewareConfig milogMiddlewareConfig) { - MilogMiddlewareConfig existConfig = milogMiddlewareConfigDao.queryById(resourceId); - milogMiddlewareConfig.setCtime(existConfig.getCtime()); - milogMiddlewareConfig.setCreator(existConfig.getCreator()); - wrapBaseCommon(milogMiddlewareConfig, OperateEnum.UPDATE_OPERATE); - resourceExtensionService.addResourcePostProcessing(milogMiddlewareConfig); - milogMiddlewareConfig.setId(resourceId); - } - - private void deleteEsIndex(Long esClusterId) { - QueryWrapper deleteWrapper = new QueryWrapper<>(); - deleteWrapper.eq("cluster_id", esClusterId); - milogEsIndexMapper.delete(deleteWrapper); - } - - /** - * Compare whether the information in the MQ changes - * After the change, you need to modify the information of the public topic - * - * @param miLogResource - */ - private void updateCompareMqInfo(MiLogResource miLogResource) { - MilogMiddlewareConfig config = milogMiddlewareConfigDao.queryById(miLogResource.getId()); - if (!Objects.equals(miLogResource.getAk(), config.getAk()) || !Objects.equals(miLogResource.getSk(), config.getSk()) || !Objects.equals(miLogResource.getServiceUrl(), config.getServiceUrl()) || !Objects.equals(miLogResource.getOrgId(), config.getOrgId()) || !Objects.equals(miLogResource.getTeamId(), config.getTeamId())) { - resourceExtensionService.addResourceMiddleProcessing(miLogResource); - } - } - - private void baseMiddlewareConfigAssemble(MiddlewareAddParam param, MilogMiddlewareConfig middlewareConfig) { - middlewareConfig.setType(param.getType()); - middlewareConfig.setRegionEn(param.getRegionEn()); - middlewareConfig.setAlias(param.getAlias()); - middlewareConfig.setNameServer(param.getNameServer()); - middlewareConfig.setServiceUrl(param.getServiceUrl()); - middlewareConfig.setAk(param.getAk()); - middlewareConfig.setSk(param.getSk()); - middlewareConfig.setAuthorization(param.getAuthorization()); - middlewareConfig.setOrgId(param.getOrgId()); - middlewareConfig.setTeamId(param.getTeamId()); - middlewareConfig.setIsDefault(param.getIsDefault()); - } - - private MilogMiddlewareConfig AddParamToAddEntity(MiddlewareAddParam addParam) { - MilogMiddlewareConfig middlewareConfig = new MilogMiddlewareConfig(); - baseMiddlewareConfigAssemble(addParam, middlewareConfig); - middlewareConfig.setCtime(Instant.now().toEpochMilli()); - middlewareConfig.setUtime(Instant.now().toEpochMilli()); - middlewareConfig.setCreator(MoneUserContext.getCurrentUser().getUser()); - middlewareConfig.setUpdater(MoneUserContext.getCurrentUser().getUser()); - return middlewareConfig; - } - - private MilogMiddlewareConfig updateParamToUpdateEntity(MiddlewareUpdateParam updateParam, MilogMiddlewareConfig middlewareConfig) { - baseMiddlewareConfigAssemble(updateParam, middlewareConfig); - middlewareConfig.setUtime(Instant.now().toEpochMilli()); - middlewareConfig.setUpdater(MoneUserContext.getCurrentUser().getUser()); - return middlewareConfig; - } - - - private List preCheckMiddlewareConfig(MiddlewareAddParam param) { - List msg = Lists.newArrayList(); - if (null == param) { - msg.add("addParam can not be null"); - return msg; - } - List types = param.getTypes(); - if (CollectionUtils.isEmpty(types) || types.size() != 2) { - msg.add("types can not be null"); - return msg; - } - param.setType(((Double) types.get(0)).intValue()); - param.setRegionEn((String) types.get(1)); - if (null == param.getType()) { - msg.add("region can not be null"); - } - if (StringUtils.isEmpty(param.getNameServer())) { - msg.add("nameServer can not be null"); - } - if (StringUtils.isEmpty(param.getServiceUrl()) && accurateTypes.contains(param.getType())) { - msg.add("serviceUrl can not be null"); - } - if (StringUtils.isEmpty(param.getAk()) && accurateTypes.contains(param.getType())) { - msg.add("ak can not be null"); - } - if (StringUtils.isEmpty(param.getSk()) && accurateTypes.contains(param.getType())) { - msg.add("sk can not be null"); - } - if (MiddlewareEnum.ROCKETMQ.getCode().equals(param.getType()) && StringUtils.isEmpty(param.getAuthorization())) { - msg.add("authorization can not be null"); - } - if (StringUtils.isEmpty(param.getOrgId()) && accurateTypes.contains(param.getType())) { - msg.add("orgId can not be null"); - } - if (MiddlewareEnum.ROCKETMQ.getCode().equals(param.getType()) && StringUtils.isEmpty(param.getTeamId())) { - msg.add("teamId can not be null"); - } - return msg; - } - - private List preCheckUpdateMiddlewareConfig(MiddlewareUpdateParam updateParam) { - List checkMsg = preCheckMiddlewareConfig(updateParam); - if (null == updateParam.getId()) { - checkMsg.add("ID can not null"); - } - return checkMsg; - } - - - private Condition handleParamToCondition(MiddlewareQueryParam param) { - Cnd cnd = Cnd.NEW(); - int size = 2; - List types = param.getTypes(); - if (CollectionUtils.isNotEmpty(param.getTypes()) && null != types.get(0)) { - cnd.and("type", EQUAL_OPERATE, ((Double) types.get(0)).intValue()); - } - if (CollectionUtils.isNotEmpty(param.getTypes()) && types.size() == size && null != types.get(1)) { - cnd.and("region_en", EQUAL_OPERATE, types.get(1)); - } - if (StringUtils.isNotEmpty(param.getAlias())) { - cnd.and("alias", "like", "%" + param.getAlias() + "%"); - } - cnd.orderBy("ctime", "desc"); - return cnd; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogOpenServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogOpenServiceImpl.java deleted file mode 100644 index a712b24d7..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogOpenServiceImpl.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import com.google.gson.Gson; -import org.apache.ozhera.app.api.response.AppBaseInfo; -import com.xiaomi.mone.log.api.enums.DeployWayEnum; -import com.xiaomi.mone.log.api.enums.ProjectTypeEnum; -import com.xiaomi.mone.log.api.model.bo.MiLogMoneTransfer; -import com.xiaomi.mone.log.api.model.dto.MontorAppDTO; -import com.xiaomi.mone.log.api.model.vo.MiLogMoneEnv; -import com.xiaomi.mone.log.api.service.MilogOpenService; -import com.xiaomi.mone.log.manager.dao.MilogLogTailDao; -import com.xiaomi.mone.log.manager.common.exception.MilogManageException; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.mone.log.manager.common.validation.OpenSourceValid; -import com.xiaomi.youpin.docean.plugin.dubbo.anno.Service; -import joptsimple.internal.Strings; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/12/8 14:54 - */ -@Slf4j -@Service(interfaceClass = MilogOpenService.class, group = "$dubbo.group") -public class MilogOpenServiceImpl implements MilogOpenService { - - @Resource - private MilogLogTailDao milogLogtailDao; - - @Resource - private OpenSourceValid openSourceValid; - - @Resource - private HeraAppServiceImpl heraAppService; - - @Resource - private Gson gson; - - - @Override - public MontorAppDTO queryHaveAccessMilog(Long iamTreeId, String bingId, Integer platformType) { - MontorAppDTO montorAppDTO = new MontorAppDTO(); - if (null == iamTreeId) { - return montorAppDTO; - } - AppBaseInfo appBaseInfo = heraAppService.queryByIamTreeId(iamTreeId, bingId, platformType); - if (null == appBaseInfo) { - return montorAppDTO; - } - List logTailDos = milogLogtailDao.getLogTailByMilogAppId(appBaseInfo.getId().longValue()); - if (CollectionUtils.isNotEmpty(logTailDos)) { - montorAppDTO.setAppId(Long.valueOf(appBaseInfo.getBindId())); - montorAppDTO.setAppName(appBaseInfo.getAppName()); - montorAppDTO.setSource(appBaseInfo.getPlatformName()); - montorAppDTO.setIsAccess(true); - } - return montorAppDTO; - } - - @Override - public Long querySpaceIdByIamTreeId(Long iamTreeId) { - AppBaseInfo appBaseInfo = heraAppService.queryByIamTreeId(iamTreeId, Strings.EMPTY, null); - if (null == appBaseInfo) { - return null; - } - List milogLogtailDos = milogLogtailDao.getLogTailByMilogAppId(appBaseInfo.getId().longValue()); - if (CollectionUtils.isEmpty(milogLogtailDos)) { - return null; - } - List spaceIds = milogLogtailDos.stream().map(MilogLogTailDo::getSpaceId).distinct().collect(Collectors.toList()); - return spaceIds.get(spaceIds.size() - 1); - } - - @Override - public MiLogMoneTransfer ypMoneEnvTransfer(MiLogMoneEnv logMoneEnv) { - String errors = openSourceValid.validMiLogMoneEnv(logMoneEnv); - if (StringUtils.isNotBlank(errors)) { - throw new MilogManageException(errors); - } - log.info("youpin mione transfer milie,data:{}", gson.toJson(logMoneEnv)); - MiLogMoneTransfer miLogMoneTransfer = new MiLogMoneTransfer(); - //1.Find apps - handleMilogAppInfo(logMoneEnv, miLogMoneTransfer); - //2.Modify tail - handleMilogAppTail(logMoneEnv, miLogMoneTransfer); - //3.Modify the source of the app - not required after migration -// handleAppSource(logMoneEnv, miLogMoneTransfer); - return miLogMoneTransfer; - } - - private void handleMilogAppTail(MiLogMoneEnv logMoneEnv, MiLogMoneTransfer miLogMoneTransfer) { - // Query the old tail - List milogLogtailDos = milogLogtailDao.queryByMilogAppAndEnv(miLogMoneTransfer.getMilogAppId(), logMoneEnv.getOldEnvId()); - if (CollectionUtils.isEmpty(milogLogtailDos)) { - return; - } - for (MilogLogTailDo milogLogtailDo : milogLogtailDos) { - milogLogtailDo.setAppName(logMoneEnv.getNewAppName()); - milogLogtailDo.setEnvId(logMoneEnv.getNewEnvId()); - milogLogtailDo.setEnvName(logMoneEnv.getNewEnvName()); - if (Objects.equals(1, logMoneEnv.getRollback())) { - milogLogtailDo.setDeployWay(DeployWayEnum.MIONE.getCode()); - } - milogLogtailDao.update(milogLogtailDo); - } - miLogMoneTransfer.setEnvId(logMoneEnv.getNewEnvId()); - miLogMoneTransfer.setEnvName(logMoneEnv.getNewEnvName()); - miLogMoneTransfer.setTailNames(milogLogtailDos - .stream() - .map(MilogLogTailDo::getTail) - .collect(Collectors.toList())); - } - - private void handleMilogAppInfo(MiLogMoneEnv logMoneEnv, MiLogMoneTransfer miLogMoneTransfer) { - //1.Find apps based on old IDs - AppBaseInfo appBaseInfo = heraAppService.queryByAppId(logMoneEnv.getNewAppId(), ProjectTypeEnum.MIONE_TYPE.getCode()); - if (null == appBaseInfo) { - appBaseInfo = heraAppService.queryByAppId(logMoneEnv.getNewAppId(), 20); - if (null == appBaseInfo) { - throw new MilogManageException("The app does not exist"); - } - } - miLogMoneTransfer.setMilogAppId(appBaseInfo.getId().longValue()); - miLogMoneTransfer.setAppId(logMoneEnv.getNewAppId()); - miLogMoneTransfer.setAppName(logMoneEnv.getNewAppName()); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogStoreSpaceAuthServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogStoreSpaceAuthServiceImpl.java deleted file mode 100644 index d4899cacb..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogStoreSpaceAuthServiceImpl.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import com.xiaomi.mone.log.api.enums.OperateEnum; -import com.xiaomi.mone.log.common.Constant; -import com.xiaomi.mone.log.manager.common.exception.MilogManageException; -import com.xiaomi.mone.log.manager.common.validation.StoreSpaceAuthValid; -import com.xiaomi.mone.log.manager.dao.MilogStoreSpaceAuthDao; -import com.xiaomi.mone.log.manager.model.bo.StoreSpaceAuth; -import com.xiaomi.mone.log.manager.model.pojo.MilogStoreSpaceAuth; -import com.xiaomi.mone.log.manager.service.BaseService; -import com.xiaomi.mone.log.manager.service.MilogStoreSpaceAuthService; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/7/14 16:13 - */ -@Slf4j -@Service -public class MilogStoreSpaceAuthServiceImpl extends BaseService implements MilogStoreSpaceAuthService { - - @Resource - private MilogStoreSpaceAuthDao milogStoreSpaceAuthDao; - - @Resource - private StoreSpaceAuthValid storeSpaceAuthValid; - - @Override - public String storeSpaceAuth(StoreSpaceAuth storeSpaceAuth) { - // Verify parameters - String paramsErrorInfos = storeSpaceAuthValid.validParam(storeSpaceAuth); - if (StringUtils.isNotBlank(paramsErrorInfos)) { - throw new MilogManageException(paramsErrorInfos); - } - // Verify that the data is real - String dataCollectInfos = storeSpaceAuthValid.validStoreAuthData(storeSpaceAuth); - if (StringUtils.isNotBlank(dataCollectInfos)) { - throw new MilogManageException(dataCollectInfos); - } - // Whether there is, there are modifications, there are no additions - MilogStoreSpaceAuth milogStoreSpaceAuth = milogStoreSpaceAuthDao.queryByStoreSpace(storeSpaceAuth.getStoreId(), storeSpaceAuth.getSpaceId()); - if (null == milogStoreSpaceAuth) { - MilogStoreSpaceAuth auth = buildStoreSpaceAuth(storeSpaceAuth.getStoreId(), storeSpaceAuth.getSpaceId()); - milogStoreSpaceAuthDao.add(auth); - return Constant.SUCCESS_MESSAGE; - } - wrapBaseCommon(milogStoreSpaceAuth, OperateEnum.UPDATE_OPERATE); - milogStoreSpaceAuthDao.update(milogStoreSpaceAuth); - return Constant.SUCCESS_MESSAGE; - } - - private MilogStoreSpaceAuth buildStoreSpaceAuth(Long storeId, Long spaceId) { - MilogStoreSpaceAuth storeSpaceAuth = new MilogStoreSpaceAuth(); - storeSpaceAuth.setStoreId(storeId); - storeSpaceAuth.setSpaceId(spaceId); - wrapBaseCommon(storeSpaceAuth, OperateEnum.ADD_OPERATE); - return storeSpaceAuth; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogStreamServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogStreamServiceImpl.java deleted file mode 100644 index ddd15650f..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogStreamServiceImpl.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import com.alibaba.nacos.api.config.ConfigService; -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.api.enums.MachineRegionEnum; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.manager.dao.MilogLogTailDao; -import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; -import com.xiaomi.mone.log.manager.dao.MilogMachineDao; -import com.xiaomi.mone.log.manager.dao.MilogMiddlewareConfigDao; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.mone.log.manager.model.pojo.MilogMiddlewareConfig; -import com.xiaomi.mone.log.manager.service.MilogStreamService; -import com.xiaomi.mone.log.manager.service.nacos.MultipleNacosConfig; -import com.xiaomi.mone.log.manager.service.nacos.impl.SpaceConfigNacosProvider; -import com.xiaomi.mone.log.manager.service.nacos.impl.SpaceConfigNacosPublisher; -import com.xiaomi.mone.log.manager.service.nacos.impl.StreamConfigNacosProvider; -import com.xiaomi.mone.log.manager.service.nacos.impl.StreamConfigNacosPublisher; -import com.xiaomi.mone.log.model.LogtailConfig; -import com.xiaomi.mone.log.model.MiLogStreamConfig; -import com.xiaomi.mone.log.model.MilogSpaceData; -import com.xiaomi.mone.log.model.SinkConfig; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.plugin.config.anno.Value; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/27 17:25 - */ -@Service -public class MilogStreamServiceImpl implements MilogStreamService { - - private StreamConfigNacosProvider streamConfigNacosProvider; - - private StreamConfigNacosPublisher streamConfigNacosPublisher; - - private SpaceConfigNacosProvider spaceConfigNacosProvider; - - private SpaceConfigNacosPublisher spaceConfigNacosPublisher; - @Resource - private MilogMachineDao milogMachineDao; - @Resource - private MilogLogstoreDao milogLogstoreDao; - @Resource - private MilogLogTailDao milogLogtailDao; - @Resource - private MilogConfigNacosServiceImpl milogConfigNacosService; - - @Resource - private MilogMiddlewareConfigDao milogMiddlewareConfigDao; - - @Value("$log_type_mq_not_consume") - private String logTypeMqNotConsume; - - /** - * @param ip - * @return - */ - @Override - public Result configIssueStream(String ip) { - if (StringUtils.isEmpty(ip)) { - return Result.failParam("IP cannot be empty"); - } - streamConfigNacosProvider = new StreamConfigNacosProvider(); - spaceConfigNacosProvider = new SpaceConfigNacosProvider(); - spaceConfigNacosPublisher = new SpaceConfigNacosPublisher(); - for (String address : MultipleNacosConfig.getAllNachosAdders()) { - ConfigService configService = MultipleNacosConfig.getConfigService(address); - streamConfigNacosProvider.setConfigService(configService); - spaceConfigNacosProvider.setConfigService(configService); - spaceConfigNacosPublisher.setConfigService(configService); - MilogMiddlewareConfig milogMiddlewareConfig = milogMiddlewareConfigDao.queryNacosRegionByNameServer(address.trim()); - if (null != milogMiddlewareConfig) { - - MiLogStreamConfig existConfig = streamConfigNacosProvider.getConfig(null); - Optional.ofNullable(existConfig).map(miLogStreamConfig -> { - Map> config = existConfig.getConfig(); - config.entrySet().stream().filter(entry -> entry.getKey().equals(ip)).forEach(entry -> { - Map streamMap = entry.getValue(); - streamMap.keySet().stream().forEach(spaceKey -> { - MilogSpaceData milogSpaceData = spaceConfigNacosProvider.getConfig(spaceKey); - if (null == milogSpaceData) { - milogSpaceData = new MilogSpaceData(); - } - //Find all configurations under this space - List sinkConfigs = generateSinkConfig(spaceKey, milogMiddlewareConfig.getRegionEn()); - milogSpaceData.setSpaceConfig(sinkConfigs); - spaceConfigNacosPublisher.publish(spaceKey, milogSpaceData); - }); - }); - return null; - }).orElse(false); - } - } - return Result.success("success"); - } - - private List generateSinkConfig(Long spaceKey, String region) { - List sinkConfigs = Lists.newArrayList(); - List logstoreDOS = milogLogstoreDao.getMilogLogstoreBySpaceIdRegion(spaceKey, region.trim()); - logstoreDOS = logstoreDOS.stream().filter(milogLogstoreDO -> !milogLogstoreDO.getLogType().toString().equals(logTypeMqNotConsume)).collect(Collectors.toList()); - logstoreDOS.forEach(milogLogstoreDO -> { - List sameStoreSinkConfigs = Lists.newArrayList(); - List logtailConfigs = Lists.newArrayList(); - List logtailDos = milogLogtailDao.getMilogLogtailByStoreId(milogLogstoreDO.getId()); - for (MilogLogTailDo logtailDo : logtailDos) { - SinkConfig sinkConfig = milogConfigNacosService.assembleSinkConfig(milogLogstoreDO.getId(), logtailDo.getId(), MachineRegionEnum.CN_MACHINE.getEn()); - sameStoreSinkConfigs.add(sinkConfig); - } - // merge - sameStoreSinkConfigs.stream().forEach(sinkConfig -> { - logtailConfigs.addAll(sinkConfig.getLogtailConfigs()); - }); - if (CollectionUtils.isNotEmpty(sameStoreSinkConfigs)) { - SinkConfig sameStoresinkConfig = sameStoreSinkConfigs.get(0); - sameStoresinkConfig.setLogtailConfigs(logtailConfigs); - sinkConfigs.add(sameStoresinkConfig); - } - }); - return sinkConfigs; - } - - @Override - public void executeSql(String sql) { - milogMachineDao.executeSql(sql); - } - - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/RocketMqConfigService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/RocketMqConfigService.java deleted file mode 100644 index da6505ae5..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/RocketMqConfigService.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.manager.common.exception.MilogManageException; -import com.xiaomi.mone.log.manager.model.dto.DictionaryDTO; -import com.xiaomi.mone.log.manager.model.dto.RocketMqResponseDTO; -import com.xiaomi.mone.log.manager.model.pojo.MilogAppMiddlewareRel; -import com.xiaomi.mone.log.manager.service.CommonRocketMqService; -import com.xiaomi.mone.log.manager.service.MqConfigService; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.producer.DefaultMQProducer; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/23 11:31 - */ -@Service -@Slf4j -public class RocketMqConfigService implements MqConfigService, CommonRocketMqService { - - private Map mqMap = new HashMap<>(); - - @Override - public MilogAppMiddlewareRel.Config generateConfig(String ak, String sk, String nameServer, String serviceUrl, - String authorization, String orgId, String teamId, Long exceedId, - String name, String source, Long id) { - MilogAppMiddlewareRel.Config config = new MilogAppMiddlewareRel.Config(); - DefaultMQProducer defaultMQProducer = mqMap.get(nameServer); - if (null == defaultMQProducer) { - defaultMQProducer = new DefaultMQProducer("hera_log"); - defaultMQProducer.setNamesrvAddr(nameServer); - try { - defaultMQProducer.start(); - } catch (MQClientException e) { - log.info("create mq producer error,nameServer:{}", nameServer, e); - throw new MilogManageException("create mq producer error", e); - } - mqMap.put(nameServer, defaultMQProducer); - } - String topicName = generateSimpleTopicName(id, name); - try { - String brokerName = "tj1-b2c-systech-infra03.kscn"; - defaultMQProducer.createTopic(brokerName, topicName, 1); - } catch (MQClientException e) { - log.info("create mq common topic error,nameServer:{}", nameServer, e); - throw new MilogManageException("create mq common topic error", e); - } - config.setTopic(topicName); - config.setPartitionCnt(1); - return config; - } - - @Override - public List queryExistsTopic(String ak, String sk, String nameServer, String serviceUrl, String authorization, String orgId, String teamId) { - List dictionaryDTOS = Lists.newArrayList(); - return dictionaryDTOS; - } - - @Override - public List createCommonTagTopic(String ak, String sk, String nameServer, String serviceUrl, String authorization, String orgId, String brokerName) { - DefaultMQProducer defaultMQProducer = mqMap.get(nameServer); - if (null == defaultMQProducer) { - defaultMQProducer = new DefaultMQProducer("hera_log"); - defaultMQProducer.setNamesrvAddr(nameServer); - try { - defaultMQProducer.start(); - } catch (MQClientException e) { - log.info("create mq producer error,nameServer:{}", nameServer, e); - throw new MilogManageException("create mq producer error", e); - } - mqMap.put(nameServer, defaultMQProducer); - } - List commonTagTopicNames = generateCommonTagTopicName(orgId); - try { - for (String commonTagTopicName : commonTagTopicNames) { -// String brokerName = "tj1-b2c-systech-infra03.kscn"; - defaultMQProducer.createTopic(brokerName, commonTagTopicName, 1); - } - } catch (MQClientException e) { - log.info("create mq common topic error,nameServer:{}", nameServer, e); - throw new MilogManageException("create mq common topic error", e); - } - return commonTagTopicNames; - } - - @Override - public boolean CreateGroup(String ak, String sk, String nameServer) { - return false; - } - - public boolean createSubscribeGroup(String serviceUrl, String authorization, String orgId, - Long spaceId, Long storeId, Long tailId, Long milogAppId) { - return false; - } - - public List querySubGroupList(String serviceUrl, String authorization, String orgId) { - return Lists.newArrayList(); - } - - public boolean deleteSubscribeGroup(String serviceUrl, String authorization, String orgId, Long spaceId, Long storeId, Long tailId) { - return false; - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/StatisticsServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/StatisticsServiceImpl.java deleted file mode 100644 index 207212c91..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/StatisticsServiceImpl.java +++ /dev/null @@ -1,276 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import cn.hutool.core.util.NumberUtil; -import com.google.common.collect.Lists; -import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.exception.CommonError; -import com.xiaomi.mone.log.manager.common.Utils; -import com.xiaomi.mone.log.manager.common.utils.ManagerUtil; -import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; -import com.xiaomi.mone.log.manager.domain.EsCluster; -import com.xiaomi.mone.log.manager.domain.SearchLog; -import com.xiaomi.mone.log.manager.model.StatisticsQuery; -import com.xiaomi.mone.log.manager.model.dto.EsStatisticsKeyWord; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.vo.LogQuery; -import com.xiaomi.mone.log.manager.service.StatisticsService; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.plugin.es.EsService; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.BucketOrder; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; -import org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram; -import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.Sum; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.FieldSortBuilder; -import org.elasticsearch.search.sort.SortOrder; - -import javax.annotation.Resource; -import java.io.IOException; -import java.time.ZoneId; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import static com.xiaomi.mone.log.common.Constant.GSON; -import static com.xiaomi.mone.log.manager.common.utils.ManagerUtil.getKeyColonPrefix; - -@Service -@Slf4j -public class StatisticsServiceImpl implements StatisticsService { - - - @Resource - private MilogLogstoreDao logstoreDao; - - @Resource - private EsCluster esCluster; - - - @Resource - private SearchLog searchLog; - - - /** - * Hourly data volume statistics for a single tail - */ - @Override - public Result> queryTailStatisticsByHour(StatisticsQuery statisticsQuery) throws IOException { - - Map result = new LinkedHashMap<>(); - if (statisticsQuery.getStartTime() == null || statisticsQuery.getStartTime() == 0) { - statisticsQuery.setStartTime(Utils.getTodayTime().get("start")); - statisticsQuery.setEndTime(Utils.getTodayTime().get("end")); - } - MilogLogStoreDO milogLogstoreDO = logstoreDao.queryById(statisticsQuery.getLogstoreId()); - if (milogLogstoreDO == null) { - return Result.success(); - } - EsService esService = esCluster.getEsService(milogLogstoreDO.getEsClusterId()); - String esIndexName = milogLogstoreDO.getEsIndex(); - SearchSourceBuilder builder = new SearchSourceBuilder(); - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.filter(QueryBuilders - .rangeQuery("timestamp") - .gte(statisticsQuery.getStartTime()) - .lte(statisticsQuery.getEndTime())); - boolQueryBuilder.filter(QueryBuilders.termQuery("logstore", milogLogstoreDO.getLogstoreName())); - boolQueryBuilder.filter(QueryBuilders.termQuery("tail", statisticsQuery.getTail())); - AbstractAggregationBuilder aggregation = AggregationBuilders - .dateHistogram("agg") - .field("timestamp") - .calendarInterval(DateHistogramInterval.hours(1)) - .format("yyyy-MM-dd HH:mm:ss") - .timeZone(ZoneId.of("+08:00")) - .minDocCount(0L); - builder.query(boolQueryBuilder); - builder.aggregation(aggregation); - SearchRequest searchRequest = new SearchRequest(esIndexName); - searchRequest.source(builder); - SearchResponse searchResponse = esService.search(searchRequest); - ParsedDateHistogram parsedDateHistogram = (ParsedDateHistogram) searchResponse.getAggregations().asMap().get("agg"); - for (Histogram.Bucket bucket : parsedDateHistogram.getBuckets()) { - result.put(bucket.getKeyAsString(), bucket.getDocCount()); - } - return Result.success(result); - } - - - /** - * Top 5 in the data volume of all tails in a single store on the day - */ - @Override - public Result> queryStoreTopTailStatisticsByDay(StatisticsQuery statisticsQuery) throws IOException { - - if (statisticsQuery.getStartTime() == null || statisticsQuery.getStartTime() == 0) { - statisticsQuery.setStartTime(Utils.getTodayTime().get("start")); - statisticsQuery.setEndTime(Utils.getTodayTime().get("end")); - } - MilogLogStoreDO logstoreDO = logstoreDao.queryById(statisticsQuery.getLogstoreId()); - if (logstoreDO == null) { - return Result.success(); - } - SearchRequest searchRequest = new SearchRequest(); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.filter(QueryBuilders - .rangeQuery("timestamp") - .from(statisticsQuery.getStartTime()) - .to(statisticsQuery.getEndTime())); - boolQueryBuilder.filter(QueryBuilders - .termQuery("logstore", logstoreDO.getLogstoreName())); - TermsAggregationBuilder tailAggr = AggregationBuilders.terms("tail"); - tailAggr.field("tail"); - tailAggr.order(BucketOrder.count(false)); - searchSourceBuilder.query(boolQueryBuilder).aggregation(tailAggr).size(0); - searchRequest.source(searchSourceBuilder); - searchRequest.indices(logstoreDO.getEsIndex()); - EsService esService = esCluster.getEsService(logstoreDO.getEsClusterId()); - SearchResponse searchResponse = esService.search(searchRequest); - ParsedStringTerms aggregations = searchResponse.getAggregations().get("tail"); - Map result = new LinkedHashMap<>(); - for (Terms.Bucket bucket : aggregations.getBuckets()) { - result.put(String.valueOf(bucket.getKey()), bucket.getDocCount()); - } - return Result.success(result); - } - - /** - * Top 5 in the volume of data of all stores in a single space - */ - @Override - public Result> querySpaceTopStoreByDay(StatisticsQuery statisticsQuery) throws IOException { - - Map result = new LinkedHashMap<>(); - if (statisticsQuery.getStartTime() == null || statisticsQuery.getStartTime() == 0) { - statisticsQuery.setStartTime(Utils.getTodayTime().get("start")); - statisticsQuery.setEndTime(Utils.getTodayTime().get("end")); - } - List logstoreList = logstoreDao.getMilogLogstoreBySpaceId(statisticsQuery.getSpaceId()); - if (null != logstoreList && logstoreList.size() > 0) { - for (MilogLogStoreDO logstoreDO : logstoreList) { - SearchRequest searchRequest = new SearchRequest(); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.filter(QueryBuilders - .rangeQuery("timestamp") - .from(statisticsQuery.getStartTime()) - .to(statisticsQuery.getEndTime())); - boolQueryBuilder.filter(QueryBuilders - .termQuery("logstore", logstoreDO.getLogstoreName())); - TermsAggregationBuilder tailAggr = AggregationBuilders.terms("logstore"); - tailAggr.field("logstore"); - tailAggr.order(BucketOrder.count(false)); - searchSourceBuilder.query(boolQueryBuilder).aggregation(tailAggr).size(0); - searchRequest.source(searchSourceBuilder); - searchRequest.indices(logstoreDO.getEsIndex()); - EsService esService = esCluster.getEsService(logstoreDO.getEsClusterId()); - SearchResponse searchResponse = esService.search(searchRequest); - ParsedStringTerms aggregations = searchResponse.getAggregations().get("logstore"); - for (Terms.Bucket bucket : aggregations.getBuckets()) { - result.put(String.valueOf(bucket.getKey()), bucket.getDocCount()); - } - } - } - return Result.success(result); - } - - @Override - public Result> queryEsStatisticsRation(LogQuery logQuery) { - List results = Lists.newArrayList(); - if (null == logQuery.getStoreId()) { - return Result.failParam("The storeId cannot be empty"); - } - MilogLogStoreDO logStoreDO = logstoreDao.queryById(logQuery.getStoreId()); - if (null == logStoreDO) { - return Result.fail(CommonError.NOT_EXISTS_DATA.getCode(), "The store does not exist"); - } - if (null == logStoreDO.getEsClusterId() || StringUtils.isEmpty(logStoreDO.getEsIndex())) { - return Result.fail(CommonError.NOT_EXISTS_DATA.getCode(), "ES index-related information does not exist"); - } - EsService esService = esCluster.getEsService(logStoreDO.getEsClusterId()); - String esIndexName = logStoreDO.getEsIndex(); - - List keyColons = ManagerUtil.getKeyColonPrefix(logStoreDO.getKeyList()); - for (String field : keyColons) { - EsStatisticsKeyWord esStatisticsKeyWord = new EsStatisticsKeyWord(); - esStatisticsKeyWord.setKey(field); - try { - String aggregationName = String.format("%s-%s", field, "static-name"); - BoolQueryBuilder boolQueryBuilder = searchLog.getQueryBuilder(logQuery, getKeyColonPrefix(logStoreDO.getKeyList())); - - // Create a search request - SearchRequest searchRequest = new SearchRequest(esIndexName); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(boolQueryBuilder); - searchSourceBuilder.size(5); - searchRequest.source(searchSourceBuilder); - - // Perform a search request - SearchResponse searchResponse = esService.search(searchRequest); - - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); - sourceBuilder.query(boolQueryBuilder); - sourceBuilder.aggregation(AggregationBuilders.terms(aggregationName) - .field(field) - .size(5) - .subAggregation(AggregationBuilders.sum("total") - .field(field))); - sourceBuilder.sort(new FieldSortBuilder(field).order(SortOrder.DESC)); - - SearchRequest aggregationRequest = new SearchRequest(esIndexName); - aggregationRequest.source(sourceBuilder); - - SearchResponse aggregationResponse = esService.search(aggregationRequest); - if (null != aggregationResponse.getAggregations()) { - Terms terms = aggregationResponse.getAggregations().get(aggregationName); - List staticsKeyWords = Lists.newArrayList(); - for (Terms.Bucket bucket : terms.getBuckets()) { - String fieldValue = bucket.getKeyAsString(); - Sum sum = bucket.getAggregations().get("total"); - double total = sum.getValue(); - double percentage = NumberUtil.div(total, searchResponse.getHits().getTotalHits().value, 4); - EsStatisticsKeyWord.StatisticsRation statisticsKeyWord = new EsStatisticsKeyWord.StatisticsRation(); - statisticsKeyWord.setValue(fieldValue); - statisticsKeyWord.setRation(percentage + ""); - staticsKeyWords.add(statisticsKeyWord); - } - esStatisticsKeyWord.setStatisticsRation(staticsKeyWords); - results.add(esStatisticsKeyWord); - } - } catch (Exception e) { - log.error("query es index exception,field:{},logQuery:{}", field, GSON.toJson(logQuery), e); - } - } - - return Result.success(results); - } - - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/StreamPartitionServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/StreamPartitionServiceImpl.java deleted file mode 100644 index 15c59b1f4..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/StreamPartitionServiceImpl.java +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import cn.hutool.core.collection.CollectionUtil; -import com.xiaomi.mone.log.manager.common.exception.MilogManageException; -import com.xiaomi.mone.log.manager.dao.MilogSpaceDao; -import com.xiaomi.mone.log.manager.model.Pair; -import com.xiaomi.mone.log.manager.model.bo.SpacePartitionBalance; -import com.xiaomi.mone.log.manager.model.page.PageInfo; -import com.xiaomi.mone.log.manager.model.pojo.MilogSpaceDO; -import com.xiaomi.mone.log.manager.model.vo.MachinePartitionParam; -import com.xiaomi.mone.log.manager.model.vo.SpaceIpParam; -import com.xiaomi.mone.log.manager.service.StreamPartitionService; -import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionServiceFactory; -import com.xiaomi.mone.log.manager.service.extension.resource.ResourceExtensionService; -import com.xiaomi.mone.log.manager.service.extension.resource.ResourceExtensionServiceFactory; -import com.xiaomi.mone.log.model.MiLogStreamConfig; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.plugin.config.anno.Value; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; -import java.util.*; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.TAIL_CONFIG_DATA_ID; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/9/19 15:06 - */ -@Service -@Slf4j -public class StreamPartitionServiceImpl implements StreamPartitionService { - - @Resource - private MilogConfigNacosServiceImpl logConfigNacosService; - @Resource - private MilogSpaceDao logSpaceDao; - @Value("$log_stream_name") - private String log_stream_name; - - private ResourceExtensionService resourceExtensionService; - - public void init() { - resourceExtensionService = ResourceExtensionServiceFactory.getResourceExtensionService(); - } - - @Override - public PageInfo querySpacePartitionBalance(MachinePartitionParam partitionParam) { - MiLogStreamConfig config = buildMiLogStreamConfig(partitionParam.getMachineRoom()); - - Map, List> spaceIps = buildSpaceIpsMap(config.getConfig()); - List spaceIds = resourceExtensionService.getSpaceIdsByNameExcluded(partitionParam.getSpaceName()); - - List spacePartitionBalanceList = buildSpacePartitionBalanceList(spaceIps); - - if (CollectionUtils.isNotEmpty(spaceIds)) { - spacePartitionBalanceList = spacePartitionBalanceList.stream() - .filter(data -> spaceIds.contains(data.getSpaceId())) - .collect(Collectors.toList()); - } - - List pageList = CollectionUtil.page(partitionParam.getPageNum() - 1, partitionParam.getPageSize(), spacePartitionBalanceList); - - updateSpaceNames(pageList); - - return buildPageInfo(partitionParam, spacePartitionBalanceList, pageList); - } - - private void updateSpaceNames(List spacePartitionBalanceList) { - spacePartitionBalanceList.forEach(data -> { - MilogSpaceDO milogSpaceDO = logSpaceDao.queryById(data.getSpaceId()); - data.setSpaceName(milogSpaceDO.getSpaceName()); - }); - } - - @Override - public PageInfo> queryIpPartitionBalance(MachinePartitionParam partitionParam) { - MiLogStreamConfig config = buildMiLogStreamConfig(partitionParam.getMachineRoom()); - - List excludingMifeSpaceIds = resourceExtensionService.getSpaceIdsByNameExcluded(partitionParam.getSpaceName()); - - Map spaceKeys = config.getConfig().get(partitionParam.getUniqueKey()); - List> pairList = spaceKeys.entrySet().stream() - .filter(data -> excludingMifeSpaceIds.contains(data.getKey())) - .sorted(Map.Entry.comparingByKey()) - .map(data -> Pair.of(data.getKey(), data.getValue())) - .collect(Collectors.toList()); - - List> pageList = CollectionUtil.page(partitionParam.getPageNum() - 1, partitionParam.getPageSize(), pairList); - pageList = pageList.stream().map(data -> { - MilogSpaceDO milogSpaceDO = logSpaceDao.queryById(data.getKey()); - return Pair.of(data.getKey(), milogSpaceDO.getSpaceName()); - }).collect(Collectors.toList()); - - return buildPageInfo(partitionParam, pairList, pageList); - - } - - @Override - public PageInfo> queryStreamList(MachinePartitionParam partitionParam) { - MiLogStreamConfig config = buildMiLogStreamConfig(partitionParam.getMachineRoom()); - List> dataList = config.getConfig().keySet() - .stream() - .filter(data -> { - if (StringUtils.isNotEmpty(partitionParam.getUniqueKey())) { - return Objects.equals(partitionParam.getUniqueKey(), data); - } - return true; - }) - .map(ip -> Pair.of(ip, queryStreamHostname(ip))) - .collect(Collectors.toList()); - - return buildPageInfo(partitionParam, dataList, dataList); - } - - private MiLogStreamConfig buildMiLogStreamConfig(String machineRoom) { - logConfigNacosService.chooseCurrentEnvNacosSerevice(machineRoom); - MiLogStreamConfig config = logConfigNacosService.getStreamConfigNacosProvider(machineRoom).getConfig(null); - if (config == null) { - throw new MilogManageException("当前机房nacos配置不存在"); - } - return config; - } - - @Override - public Boolean addSpaceToIp(SpaceIpParam param) { - MiLogStreamConfig config = buildMiLogStreamConfig(param.getMachineRoom()); - - for (String uniqueKey : param.getUniqueKeys()) { - config.getConfig().putIfAbsent(uniqueKey, new HashMap<>()); - for (Long spaceId : param.getSpaceIds()) { - String spaceKey = String.format("%s%s%s", CommonExtensionServiceFactory.getCommonExtensionService().getLogManagePrefix(), TAIL_CONFIG_DATA_ID, spaceId); - config.getConfig().get(uniqueKey).putIfAbsent(spaceId, spaceKey); - } - } - - logConfigNacosService.getStreamConfigNacosPublisher(param.getMachineRoom()).publish(param.getSpaceId(), config); - return true; - } - - @Override - public Boolean delSpaceToIp(SpaceIpParam param) { - MiLogStreamConfig config = buildMiLogStreamConfig(param.getMachineRoom()); - Map spaceMap = config.getConfig().get(param.getUniqueKey()); - if (null != spaceMap) { - spaceMap.remove(param.getSpaceId()); - if (spaceMap.isEmpty()) { - config.getConfig().remove(param.getUniqueKey()); - } - logConfigNacosService.getStreamConfigNacosPublisher(param.getMachineRoom()).publish(param.getSpaceId(), config); - } - return true; - } - - @Override - public boolean streamReBalance() { - return false; - } - - @Override - public String queryStreamHostname(String ip) { - return resourceExtensionService.queryHostName(ip); - } - - @Override - public List> findUnIncludedSpaceList(SpaceIpParam param) { - MiLogStreamConfig config = buildMiLogStreamConfig(param.getMachineRoom()); - - Map spaceKeys = config.getConfig().get(param.getUniqueKey()); - List includedSpaceList = new ArrayList<>(spaceKeys.keySet()); - - List allSpaces = logSpaceDao.queryByName(param.getSpaceName()); - - List> pairList = allSpaces.parallelStream() - .filter(space -> !includedSpaceList.contains(space.getId())) - .map(space -> Pair.of(space.getSpaceName(), space.getId())) - .collect(Collectors.toList()); - return CollectionUtil.page(param.getPageNum() - 1, param.getPageSize(), pairList); - } - - @Override - public List> queryAllUniqueKeyList(SpaceIpParam param) { - MiLogStreamConfig config = buildMiLogStreamConfig(param.getMachineRoom()); - - return config.getConfig().entrySet().stream() - .filter(data -> !data.getValue().containsKey(param.getSpaceId())) - .map(data -> Pair.of(data.getKey(), data.getKey())).collect(Collectors.toList()); - } - - private Map, List> buildSpaceIpsMap(Map> configConfig) { - Map, List> spaceIps = new HashMap<>(); - for (Map.Entry> ipEntry : configConfig.entrySet()) { - for (Map.Entry spaceEntry : ipEntry.getValue().entrySet()) { - Pair spacePair = new Pair<>(spaceEntry.getKey(), spaceEntry.getValue()); - spaceIps.computeIfAbsent(spacePair, k -> new ArrayList<>()).add(ipEntry.getKey()); - } - } - return spaceIps; - } - - private List buildSpacePartitionBalanceList(Map, List> spaceIps) { - List balanceList = spaceIps.entrySet().stream().map(entry -> { - SpacePartitionBalance spacePartitionBalance = new SpacePartitionBalance(); - Pair spacePair = entry.getKey(); - spacePartitionBalance.setSpaceId(spacePair.getKey()); - spacePartitionBalance.setSpaceIdentifiers(spacePair.getValue()); - spacePartitionBalance.setMachineUniques(entry.getValue()); - return spacePartitionBalance; - }).sorted(Comparator.comparing(SpacePartitionBalance::getSpaceId)) - .collect(Collectors.toList()); - return balanceList; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/TpcSpaceAuthService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/TpcSpaceAuthService.java deleted file mode 100644 index 516ee2962..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/TpcSpaceAuthService.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.impl; - -import com.xiaomi.mone.log.manager.domain.Tpc; -import com.xiaomi.mone.log.manager.model.MilogSpaceParam; -import com.xiaomi.mone.log.manager.model.pojo.MilogSpaceDO; -import com.xiaomi.mone.log.manager.service.SpaceAuthService; -import com.xiaomi.mone.tpc.common.vo.NodeVo; -import com.xiaomi.mone.tpc.common.vo.PageDataVo; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.infra.rpc.Result; - -import javax.annotation.Resource; -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/4/10 15:00 - */ -@Service -public class TpcSpaceAuthService implements SpaceAuthService { - - @Resource - private Tpc tpc; - - @Override - public Result saveSpacePerm(MilogSpaceDO spaceDO, String account) { - Result tpcResult = tpc.saveSpacePerm(spaceDO, account); - return tpcResult; - } - - @Override - public Result> getUserPermSpace(String spaceName, Integer page, Integer pageSize) { - Result> tpcRes = tpc.getUserPermSpace(spaceName, page, pageSize); - return tpcRes; - } - - @Override - public Result deleteSpaceTpc(Long spaceId, String account, Integer userType) { - Result tpcResult = tpc.deleteSpaceTpc(spaceId, account, userType); - return tpcResult; - } - - @Override - public Result updateSpaceTpc(MilogSpaceParam param, String account) { - Result tpcResult = this.tpc.updateSpaceTpc(param, account); - return tpcResult; - } - - @Override - public void addSpaceMember(Long spaceId, String userAccount, Integer userType, Integer memberCode) { - tpc.addSpaceMember(spaceId, userAccount, userType, memberCode); - } - - @Override - public Result> queryUserListNode(String spaceName, String userName) { - return Result.success(tpc.queryUserListNode(spaceName, userName)); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/init_sql/AnalyseGraphTypeService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/init_sql/AnalyseGraphTypeService.java deleted file mode 100644 index 04bb33b58..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/init_sql/AnalyseGraphTypeService.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.init_sql; - -import com.google.common.collect.Maps; -import com.xiaomi.mone.log.manager.mapper.MilogAnalyseGraphTypeMapper; -import com.xiaomi.mone.log.manager.model.pojo.MilogAnalyseGraphTypeDO; -import com.xiaomi.youpin.docean.anno.Service; -import org.apache.commons.collections.CollectionUtils; - -import javax.annotation.Resource; -import java.util.List; -import java.util.Map; - -import static com.xiaomi.mone.log.common.Constant.SYMBOL_COLON; - -/** - * @author wtt - * @version 1.0 - * @description Log analysis graph type initialization - * @date 2023/3/31 10:32 - */ -@Service -public class AnalyseGraphTypeService { - - @Resource - private MilogAnalyseGraphTypeMapper graphTypeMapper; - - private static Map namesMap = Maps.newHashMap(); - - static { - namesMap.put(1, String.format("%s%s%s", "饼图", SYMBOL_COLON, "比例")); - namesMap.put(2, String.format("%s%s%s", "折线图", SYMBOL_COLON, "折线图和面积图")); - namesMap.put(3, String.format("%s%s%s", "垂直条形图", SYMBOL_COLON, "条形图")); - - namesMap.put(4, String.format("%s%s%s", "水平条形图", SYMBOL_COLON, "条形图")); - namesMap.put(5, String.format("%s%s%s", "垂直条形图", SYMBOL_COLON, "折线图和面积图")); - namesMap.put(8, String.format("%s%s%s", "圆环图", SYMBOL_COLON, "比例")); - - namesMap.put(9, String.format("%s%s%s", "南丁格尔玫瑰图", SYMBOL_COLON, "比例")); - //没上线 -// namesMap.put(10, String.format("%s%s%s", "分时柱状图", SYMBOL_COLON, "分时柱状图")); - } - - public void init() { - List graphTypeDOList = graphTypeMapper.selectList(null); - if (CollectionUtils.isEmpty(graphTypeDOList)) { - for (Map.Entry entry : namesMap.entrySet()) { - MilogAnalyseGraphTypeDO logAnalyseGraphType = new MilogAnalyseGraphTypeDO(); - logAnalyseGraphType.setType(entry.getKey()); - String[] nameClassifiers = entry.getValue().split(SYMBOL_COLON); - logAnalyseGraphType.setName(nameClassifiers[0]); - logAnalyseGraphType.setCalculate(nameClassifiers[1]); - graphTypeMapper.insert(logAnalyseGraphType); - } - } - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/init_sql/HeraLogTemplateService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/init_sql/HeraLogTemplateService.java deleted file mode 100644 index bdc1dc8c1..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/init_sql/HeraLogTemplateService.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.init_sql; - -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.xiaomi.mone.log.api.enums.LogTypeEnum; -import com.xiaomi.mone.log.api.enums.MachineRegionEnum; -import com.xiaomi.mone.log.manager.mapper.MilogLogTemplateDetailMapper; -import com.xiaomi.mone.log.manager.mapper.MilogLogTemplateMapper; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTemplateDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTemplateDetailDO; -import com.xiaomi.mone.log.manager.service.BaseService; -import com.xiaomi.youpin.docean.anno.Service; -import org.apache.commons.collections.CollectionUtils; -import org.jetbrains.annotations.NotNull; - -import javax.annotation.Resource; -import java.time.Instant; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.SYMBOL_COMMA; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/3/3 10:46 - */ -@Service -public class HeraLogTemplateService extends BaseService { - - @Resource - private MilogLogTemplateMapper milogLogTemplateMapper; - @Resource - private MilogLogTemplateDetailMapper milogLogTemplateDetailMapper; - - public void init() { - for (LogTypeEnum typeEnum : LogTypeEnum.values()) { - //Query the log type - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("type", typeEnum.getType()); - List templateDOS = milogLogTemplateMapper.selectList(queryWrapper); - if (CollectionUtils.isEmpty(templateDOS)) { - MilogLogTemplateDO logTemplateDO = getInsertLogTemplateDO(typeEnum, - Arrays.stream(MachineRegionEnum.values()).map(MachineRegionEnum::getEn) - .collect(Collectors.joining(SYMBOL_COMMA))); - //The query type template exists - MilogLogTemplateDetailDO detailDO = milogLogTemplateDetailMapper.getByTemplateId(logTemplateDO.getId()); - if (null == detailDO) { - insertTemplateDetail(typeEnum, logTemplateDO); - } - } - } - } - - private void insertTemplateDetail(LogTypeEnum typeEnum, MilogLogTemplateDO logTemplateDO) { - MilogLogTemplateDetailDO detailDO; - detailDO = new MilogLogTemplateDetailDO(); - detailDO.setTemplateId(logTemplateDO.getId().toString()); - if (typeEnum == LogTypeEnum.APP_LOG_MULTI) { - detailDO.setPropertiesKey("timestamp:1,level:1,traceId:1,threadName:1,className:1,line:1,methodName:1,message:1,logstore:3,logsource:3,mqtopic:3,mqtag:3,logip:3,tail:3,linenumber:3,podName:1"); - detailDO.setPropertiesType("date,keyword,keyword,text,text,keyword,keyword,text,keyword,keyword,keyword,keyword,keyword,keyword,long,keyword"); - } - if (typeEnum == LogTypeEnum.NGINX) { - detailDO.setPropertiesKey("message:1,hostname:1,http_code:1,method:1,protocol:1,referer:1,timestamp:1,ua:1,url:1,linenumber:3,logip:3"); - detailDO.setPropertiesType("text,text,keyword,keyword,keyword,text,timestamp,text,text,long,keyword"); - } - if (typeEnum == LogTypeEnum.OPENTELEMETRY) { - detailDO.setPropertiesKey("logstore:3,logsource:3,mqtopic:3,mqtag:3,logip:3,tail:3,linenumber:3"); - detailDO.setPropertiesType("keyword,keyword,keyword,keyword,keyword,keyword,long"); - } - if (typeEnum == LogTypeEnum.DOCKER) { - detailDO.setPropertiesKey("logstore:3,logsource:3,mqtopic:3,mqtag:3,logip:3,tail:3,linenumber:3"); - detailDO.setPropertiesType("keyword,keyword,keyword,keyword,keyword,keyword,long"); - } - if (typeEnum == LogTypeEnum.APP_LOG_SIGNAL) { - detailDO.setPropertiesKey("timestamp:1,level:1,traceId:1,threadName:1,className:1,line:1,methodName:1,message:1,logstore:3,logsource:3,mqtopic:3,mqtag:3,logip:3,tail:3,linenumber:3,podName:1"); - detailDO.setPropertiesType("date,keyword,keyword,text,text,keyword,keyword,text,keyword,keyword,keyword,keyword,keyword,keyword,long,keyword"); - } - if (typeEnum == LogTypeEnum.ORIGIN_LOG) { - detailDO.setPropertiesKey("timestamp:1,level:1,traceId:1,threadName:1,className:1,line:1,methodName:1,message:1,logstore:3,logsource:3,mqtopic:3,mqtag:3,logip:3,tail:3,linenumber:3,podName:1"); - detailDO.setPropertiesType("date,keyword,keyword,text,text,keyword,keyword,text,keyword,keyword,keyword,keyword,keyword,keyword,long,keyword"); - } - if (typeEnum == LogTypeEnum.FREE) { - detailDO.setPropertiesKey("logstore:3,logsource:3,mqtopic:3,mqtag:3,logip:3,tail:3,linenumber:3"); - detailDO.setPropertiesType("keyword,keyword,keyword,keyword,keyword,keyword,long"); - } - detailDO.setCtime(Instant.now().toEpochMilli()); - detailDO.setUtime(Instant.now().toEpochMilli()); - milogLogTemplateDetailMapper.insert(detailDO); - } - - @NotNull - private MilogLogTemplateDO getInsertLogTemplateDO(LogTypeEnum typeEnum, String supportArea) { - MilogLogTemplateDO logTemplateDO = new MilogLogTemplateDO(); - logTemplateDO.setTemplateName(typeEnum.getTypeName()); - logTemplateDO.setType(typeEnum.getType()); - logTemplateDO.setOrderCol(typeEnum.getSort()); - logTemplateDO.setSupportArea(supportArea); - logTemplateDO.setSupportedConsume(typeEnum.getSupportedConsume()); - logTemplateDO.setCtime(Instant.now().toEpochMilli()); - logTemplateDO.setUtime(Instant.now().toEpochMilli()); - milogLogTemplateMapper.insert(logTemplateDO); - return logTemplateDO; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/init_sql/NcosConfigSqlService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/init_sql/NcosConfigSqlService.java deleted file mode 100644 index 563badd17..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/init_sql/NcosConfigSqlService.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.init_sql; - -import com.xiaomi.mone.log.api.enums.MachineRegionEnum; -import com.xiaomi.mone.log.api.enums.MiddlewareEnum; -import com.xiaomi.mone.log.api.enums.OperateEnum; -import com.xiaomi.mone.log.common.Constant; -import com.xiaomi.mone.log.manager.dao.MilogMiddlewareConfigDao; -import com.xiaomi.mone.log.manager.model.pojo.MilogMiddlewareConfig; -import com.xiaomi.mone.log.manager.service.BaseService; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.plugin.config.anno.Value; - -import javax.annotation.Resource; -import java.util.Objects; - -import static com.xiaomi.mone.log.common.Constant.DEFAULT_OPERATOR; - -/** - * @author wtt - * @version 1.0 - * @description Initialize the default NACOS configuration information into the table - * @date 2023/3/3 10:45 - */ -@Service -public class NcosConfigSqlService extends BaseService { - private static final String DEFAULT_NCOS_ALIAS = "system nacos"; - - @Resource - private MilogMiddlewareConfigDao milogMiddlewareConfigDao; - - @Value(value = "$defaultNacosAddres") - private String defaultNacosAddress; - - public void init() { - String defaultRegionCode = MachineRegionEnum.CN_MACHINE.getEn(); - MilogMiddlewareConfig middlewareConfig = milogMiddlewareConfigDao.queryCurrentEnvNacos(defaultRegionCode); - if (null == middlewareConfig) { - addNcosConfig(defaultRegionCode); - return; - } - if (Objects.equals(middlewareConfig.getNameServer(), defaultNacosAddress)) { - updateNcosAddress(middlewareConfig); - } - } - - private void addNcosConfig(String defaultRegionCode) { - MilogMiddlewareConfig middlewareConfig; - middlewareConfig = new MilogMiddlewareConfig(); - middlewareConfig.setType(MiddlewareEnum.NCOS.getCode()); - middlewareConfig.setRegionEn(defaultRegionCode); - middlewareConfig.setAlias(DEFAULT_NCOS_ALIAS); - middlewareConfig.setNameServer(defaultNacosAddress); - middlewareConfig.setIsDefault(Constant.YES.intValue()); - wrapBaseCommon(middlewareConfig, OperateEnum.ADD_OPERATE, DEFAULT_OPERATOR); - milogMiddlewareConfigDao.addMiddlewareConfig(middlewareConfig); - } - - private void updateNcosAddress(MilogMiddlewareConfig middlewareConfig) { - middlewareConfig.setNameServer(defaultNacosAddress); - wrapBaseCommon(middlewareConfig, OperateEnum.UPDATE_OPERATE, DEFAULT_OPERATOR); - milogMiddlewareConfigDao.updateMiddlewareConfig(middlewareConfig); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/init_sql/package-info.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/init_sql/package-info.java deleted file mode 100644 index c516c16e8..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/init_sql/package-info.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.init_sql; -/** - * Initialize SQL information into MySQL - */ \ No newline at end of file diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/listener/AutoLogAccessListener.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/listener/AutoLogAccessListener.java deleted file mode 100644 index 9fd5ff004..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/listener/AutoLogAccessListener.java +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.listener; - -import com.alibaba.nacos.api.NacosFactory; -import com.alibaba.nacos.api.config.ConfigService; -import com.alibaba.nacos.api.config.listener.Listener; -import com.alibaba.nacos.api.exception.NacosException; -import com.google.gson.reflect.TypeToken; -import org.apache.ozhera.app.api.response.AppBaseInfo; -import com.xiaomi.mone.log.api.enums.OperateEnum; -import com.xiaomi.mone.log.api.enums.ProjectTypeEnum; -import com.xiaomi.mone.log.manager.dao.MilogLogTailDao; -import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; -import com.xiaomi.mone.log.manager.dao.MilogSpaceDao; -import com.xiaomi.mone.log.manager.model.bo.AutoAccessLogParam; -import com.xiaomi.mone.log.manager.model.bo.LogTailParam; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.mone.log.manager.service.HeraAppService; -import com.xiaomi.mone.log.manager.service.extension.tail.TailExtensionService; -import com.xiaomi.mone.log.manager.service.extension.tail.TailExtensionServiceFactory; -import com.xiaomi.mone.log.manager.service.impl.LogTailServiceImpl; -import com.xiaomi.mone.log.parse.LogParserFactory; -import com.xiaomi.youpin.docean.anno.Component; -import com.xiaomi.youpin.docean.plugin.config.anno.Value; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; - -import javax.annotation.Resource; -import java.util.List; -import java.util.Properties; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executor; - -import static com.xiaomi.mone.log.common.Constant.*; - -/** - * - * @description - * @version 1.0 - * @author wtt - * @date 2024/7/10 10:45 - * - */ -@Component -@Slf4j -public class AutoLogAccessListener { - @Resource - private MilogSpaceDao spaceDao; - - @Resource - private MilogLogstoreDao logStoreDao; - - @Resource - private MilogLogTailDao logTailDao; - - @Resource - private LogTailServiceImpl logTailService; - - private TailExtensionService tailExtensionService; - - @Resource - private HeraAppService heraAppService; - - @Value("$defaultNacosAddres") - private String nacosAddress; - - private final String ACCESS_LOG_DATA_ID_KEY = "auto_access_log_key"; - - private ConfigService configService; - - public void init() throws NacosException { - freshAccessLogList(); - } - - private void freshAccessLogList() throws NacosException { - if (null == configService) { - tailExtensionService = TailExtensionServiceFactory.getTailExtensionService(); - this.configService = initConfigService(); - startListening(); - } - } - - private ConfigService initConfigService() throws NacosException { - Properties properties = new Properties(); - properties.setProperty("serverAddr", nacosAddress); - return NacosFactory.createConfigService(properties); - } - - private void startListening() throws NacosException { - configService.addListener(ACCESS_LOG_DATA_ID_KEY, DEFAULT_GROUP_ID, new Listener() { - @Override - public void receiveConfigInfo(String configInfo) { - autoLogAccess(configInfo); - } - - @Override - public Executor getExecutor() { - return null; - } - }); - } - - private void autoLogAccess(String dataContent) { - if (StringUtils.isNotEmpty(dataContent)) { - List accessLogParamList = GSON.fromJson(dataContent, new TypeToken>() { - }.getType()); - if (CollectionUtils.isNotEmpty(accessLogParamList)) { - accessLogParamList.forEach(this::processLogParam); - } - } - } - - private void processLogParam(AutoAccessLogParam logParam) { - if (!isValidLogParam(logParam)) { - log.warn("AutoLogAccess, data param valid error, data:{}", GSON.toJson(logParam)); - return; - } - - if (!isValidSpace(logParam.getSpaceId())) { - log.warn("AutoLogAccess, space not exist:{}", logParam.getSpaceId()); - return; - } - - MilogLogStoreDO logStoreDO = logStoreDao.queryById(logParam.getStoreId()); - if (logStoreDO == null) { - log.warn("AutoLogAccess, store not exist:{}", logParam.getStoreId()); - return; - } - - AppBaseInfo appBaseInfo = heraAppService.queryByAppId(logParam.getAppId(), ProjectTypeEnum.MIONE_TYPE.getCode()); - if (appBaseInfo == null) { - log.warn("AppBaseInfo not exist, appId:{}, appName:{}", logParam.getAppId(), logParam.getAppName()); - appBaseInfo = buildAppBaseInfo(logParam); - } - - String tailName = generateTailName(logParam.getAppName(), logParam.getEnvName(), logParam.getEnvId()); - if (!logTailExists(logParam, Long.valueOf(appBaseInfo.getId()), tailName)) { - addLogTail(logParam, logStoreDO, appBaseInfo, tailName); - } else { - log.info("AutoLogAccess, tail already exist, tailName:{}", tailName); - } - } - - @NotNull - private static AppBaseInfo buildAppBaseInfo(AutoAccessLogParam logParam) { - AppBaseInfo appBaseInfo; - appBaseInfo = new AppBaseInfo(); - appBaseInfo.setId((int) (logParam.getAppId() * 100)); - appBaseInfo.setAppName(logParam.getAppName()); - appBaseInfo.setBindId(logParam.getAppId().toString()); - return appBaseInfo; - } - - private boolean isValidLogParam(AutoAccessLogParam logParam) { - return logParam.getSpaceId() != null && logParam.getAppId() != null && - logParam.getStoreId() != null && logParam.getLogPath() != null && - logParam.getEnvId() != null && CollectionUtils.isNotEmpty(logParam.getIps()); - } - - private boolean isValidSpace(Long spaceId) { - return spaceDao.queryById(spaceId) != null; - } - - private boolean logTailExists(AutoAccessLogParam logParam, Long heraAppId, String tailName) { - List logTailDos = logTailDao.queryByCondition(logParam.getSpaceId(), logParam.getStoreId(), tailName, - heraAppId, logParam.getEnvId(), logParam.getLogPath()); - return CollectionUtils.isNotEmpty(logTailDos); - } - - private void addLogTail(AutoAccessLogParam logParam, MilogLogStoreDO logStoreDO, AppBaseInfo appBaseInfo, String tailName) { - LogTailParam logTailParam = buildLogTailParam(logParam.getSpaceId(), logStoreDO, logParam, appBaseInfo, tailName, logParam.getIps()); - MilogLogTailDo logTailDo = logTailService.buildLogTailDo(logTailParam, logStoreDO, appBaseInfo, DEFAULT_OPERATOR); - logTailDao.add(logTailDo); - - String topicName = buildTopicName(logParam, logTailDo.getTail()); - tailExtensionService.defaultBindingAppTailConfigRel(logTailDo.getId(), logTailDo.getMilogAppId(), logStoreDO.getMqResourceId(), topicName, null); - logTailService.sengMessageToStream(logTailDo, OperateEnum.ADD_OPERATE.getCode()); - - CompletableFuture.runAsync(() -> logTailService.sengMessageToAgent(Long.valueOf(appBaseInfo.getId()), logTailDo)); - } - - private String buildTopicName(AutoAccessLogParam logParam, String tailName) { - return String.format("%s-%s", logParam.getAppId(), tailName); - } - - private String generateTailName(String appName, String envName, Long envId) { - return String.format("%s-%s-%s", appName, envName, envId); - } - - private LogTailParam buildLogTailParam(Long spaceId, MilogLogStoreDO logStoreDO, AutoAccessLogParam logParam, - AppBaseInfo appBaseInfo, String tailName, List ips) { - return LogTailParam.builder() - .spaceId(spaceId) - .storeId(logStoreDO.getId()) - .appId(logParam.getAppId()) - .milogAppId(Long.valueOf(appBaseInfo.getId())) - .envId(logParam.getEnvId()) - .envName(logParam.getEnvName()) - .tail(tailName) - .parseType(LogParserFactory.LogParserEnum.SEPARATOR_PARSE.getCode()) - .parseScript(DEFAULT_TAIL_SEPARATOR) - .logPath(logParam.getLogPath()) - .ips(ips) - .valueList(DEFAULT_VALUE_LIST) - .appType(ProjectTypeEnum.MIONE_TYPE.getCode()) - .deployWay(ProjectTypeEnum.MIONE_TYPE.getCode()) - .build(); - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/Converter.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/Converter.java deleted file mode 100644 index 2f7b89df9..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/Converter.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.nacos; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/15 15:24 - */ -@FunctionalInterface -public interface Converter { - /** - * Data transformation - * @param source - * @return - */ - T convert(S source); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/DynamicConfigProvider.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/DynamicConfigProvider.java deleted file mode 100644 index 34f8b810c..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/DynamicConfigProvider.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.nacos; - -import com.google.gson.Gson; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/16 15:25 - */ -public interface DynamicConfigProvider { - - Gson gson = new Gson(); - - /** - * Get the configuration - * - * @param spaceId - * @return - */ - T getConfig(Long spaceId); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/DynamicConfigPublisher.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/DynamicConfigPublisher.java deleted file mode 100644 index 5682ed4cc..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/DynamicConfigPublisher.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.nacos; - -import com.google.gson.Gson; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/15 15:16 - */ -public interface DynamicConfigPublisher { - Gson gson = new Gson(); - - /** - * Data persistence to NACOS - * - * @param spaceId - * @param configs - */ - void publish(Long spaceId, T configs); - - /** - * Remove the configuration - * @param dataId - */ - void remove(String dataId); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/FetchStreamMachineService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/FetchStreamMachineService.java deleted file mode 100644 index 6c76d145c..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/FetchStreamMachineService.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.nacos; - -import java.util.List; - -public interface FetchStreamMachineService { - /** - * Gets the identity of the stream machine - * - * @return - */ - List streamMachineUnique(); - - List getStreamList(String dataId); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/InitService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/InitService.java deleted file mode 100644 index 9a12b03c2..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/InitService.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.nacos; - -import com.xiaomi.data.push.nacos.NacosNaming; -import com.xiaomi.mone.log.common.NetUtils; -import com.xiaomi.youpin.docean.anno.Service; -import com.xiaomi.youpin.docean.plugin.config.anno.Value; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.PostConstruct; -import javax.annotation.Resource; - -@Service -@Slf4j -public class InitService { - - @Resource - private NacosNaming nacosNaming; - - @Value(value = "$serverNameHttp", defaultValue = "") - private String serverNameHttp; - - @Value(value = "$serverPort", defaultValue = "") - private String httpPort; - - - @Value(value = "$dubbo.group", defaultValue = "") - private String group; - - @PostConstruct - public void init() { - String host = System.getenv("host.ip") == null ? NetUtils.getLocalHost() : System.getenv("host.ip"); - final String port = httpPort; - final String appName = serverNameHttp; - try { - nacosNaming.registerInstance(appName, host, Integer.valueOf(port), group); - - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { - System.out.println("stop"); - nacosNaming.deregisterInstance(appName, host, Integer.valueOf(port), group); - } catch (Exception e) { - log.error("init service err:{}", e.getMessage(), e); - } - })); - } catch (Exception e) { - log.error("init service err:{}", e.getMessage(), e); - } - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/MultipleNacosConfig.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/MultipleNacosConfig.java deleted file mode 100644 index 2d138e706..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/MultipleNacosConfig.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.nacos; - -import com.alibaba.nacos.api.config.ConfigFactory; -import com.alibaba.nacos.api.config.ConfigService; -import com.alibaba.nacos.api.exception.NacosException; -import com.xiaomi.data.push.nacos.NacosNaming; -import com.xiaomi.mone.log.manager.common.MilogConfig; -import com.xiaomi.youpin.docean.anno.Component; -import com.xiaomi.youpin.docean.plugin.config.anno.Value; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.util.*; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/10/18 15:09 - */ -@Component -@Slf4j -public class MultipleNacosConfig { - - @Value("$nacosAddr") - private String nacosAdders; - - private static Map nacosServiceMap = new HashMap<>(); - private static Map nacosNamingMap = new HashMap<>(); - - public void init() { - Arrays.stream(StringUtils.split(nacosAdders, "\\$")).forEach(address -> { - try { - nacosServiceMap.put(address, ConfigFactory.createConfigService(address)); - nacosNamingMap.put(address, MilogConfig.buildNacosNaming(address)); - } catch (NacosException e) { - log.error(String.format("multiple nacos address init error:address:%s", address), e); - } - }); - log.info("multiple nacos service address:{}", nacosServiceMap); - log.info("multiple nacos naming address:{}", nacosNamingMap); - } - - public static List getAllNachosAdders() { - return new ArrayList<>(nacosServiceMap.keySet()); - } - - public static ConfigService getConfigService(String nacosAddress) { - return nacosServiceMap.get(nacosAddress); - } - - public static NacosNaming getNacosNaming(String nacosAddress) { - return nacosNamingMap.get(nacosAddress); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/impl/NacosFetchStreamMachineService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/impl/NacosFetchStreamMachineService.java deleted file mode 100644 index 8e8b2ed33..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/impl/NacosFetchStreamMachineService.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.nacos.impl; - -import com.alibaba.nacos.api.exception.NacosException; -import com.alibaba.nacos.api.naming.pojo.Instance; -import com.google.common.collect.Lists; -import com.xiaomi.data.push.nacos.NacosNaming; -import com.xiaomi.mone.log.manager.common.exception.MilogManageException; -import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionServiceFactory; -import com.xiaomi.mone.log.manager.service.nacos.FetchStreamMachineService; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.util.List; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.common.Constant.STREAM_CONTAINER_POD_NAME_KEY; -import static com.xiaomi.mone.log.common.Constant.STRIKETHROUGH_SYMBOL; - -@Slf4j -public class NacosFetchStreamMachineService implements FetchStreamMachineService { - - private final NacosNaming nacosNaming; - - public NacosFetchStreamMachineService(NacosNaming nacosNaming) { - this.nacosNaming = nacosNaming; - } - - @Override - public List streamMachineUnique() { - preCheckNaming(nacosNaming); - List uniqueKeys = Lists.newArrayList(); - try { - List allInstances = nacosNaming.getAllInstances(CommonExtensionServiceFactory.getCommonExtensionService().getHeraLogStreamServerName()); - return extractUniqueKeys(allInstances); - } catch (NacosException e) { - log.error("nacos queryStreamMachineIps error", e); - } - return uniqueKeys; - } - - @Override - public List getStreamList(String dataId) { - preCheckNaming(nacosNaming); - List uniqueKeys = Lists.newArrayList(); - try { - List allInstances = nacosNaming.getAllInstances(dataId); - return extractUniqueKeys(allInstances); - } catch (NacosException e) { - log.error("nacos queryStreamMachineIps error,dataId:{}", dataId, e); - } - return uniqueKeys; - } - - private void preCheckNaming(NacosNaming nacosNaming) { - if (null == nacosNaming) { - throw new MilogManageException("please set nacos naming first"); - } - } - - private List extractUniqueKeys(List instances) { - return instances.stream() - .map(this::extractKeyFromInstance) - .distinct() - .collect(Collectors.toList()); - } - - private String extractKeyFromInstance(Instance instance) { - if (instance.getMetadata().containsKey(STREAM_CONTAINER_POD_NAME_KEY)) { - return StringUtils.substringAfterLast(instance.getMetadata().get(STREAM_CONTAINER_POD_NAME_KEY), STRIKETHROUGH_SYMBOL); - } else { - return instance.getIp(); - } - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/impl/SpaceConfigNacosProvider.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/impl/SpaceConfigNacosProvider.java deleted file mode 100644 index 50088e867..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/impl/SpaceConfigNacosProvider.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.nacos.impl; - -import com.alibaba.nacos.api.config.ConfigService; -import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionServiceFactory; -import com.xiaomi.mone.log.manager.service.nacos.DynamicConfigProvider; -import com.xiaomi.mone.log.model.MilogSpaceData; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import static com.xiaomi.mone.log.common.Constant.*; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/16 16:32 - */ -@Slf4j -public class SpaceConfigNacosProvider implements DynamicConfigProvider { - @Setter - private ConfigService configService; - - @Override - public MilogSpaceData getConfig(Long spaceId) { - String rules; - String dataId = CommonExtensionServiceFactory.getCommonExtensionService().getLogManagePrefix() + TAIL_CONFIG_DATA_ID + spaceId; - try { - rules = configService.getConfig(dataId, DEFAULT_GROUP_ID, DEFAULT_TIME_OUT_MS); - log.info("Query the log configuration in NACOS,dataId:{},data:{}", dataId, rules); - if (StringUtils.isNotEmpty(rules)) { - return gson.fromJson(rules, MilogSpaceData.class); - } - } catch (Exception e) { - log.error(String.format("Query log configuration data data exceptions, parameters:%s", dataId), e); - } - return null; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/impl/SpaceConfigNacosPublisher.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/impl/SpaceConfigNacosPublisher.java deleted file mode 100644 index 68666bbbb..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/impl/SpaceConfigNacosPublisher.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.nacos.impl; - -import com.alibaba.nacos.api.config.ConfigService; -import com.alibaba.nacos.api.exception.NacosException; -import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionServiceFactory; -import com.xiaomi.mone.log.manager.service.nacos.DynamicConfigPublisher; -import com.xiaomi.mone.log.model.MilogSpaceData; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -import static com.xiaomi.mone.log.common.Constant.DEFAULT_GROUP_ID; -import static com.xiaomi.mone.log.common.Constant.TAIL_CONFIG_DATA_ID; - -/** - * @author wtt - * @version 1.0 - * @description namespace Configure push NACOS - * @date 2021/7/16 10:36 - */ -@Slf4j -public class SpaceConfigNacosPublisher implements DynamicConfigPublisher { - - @Setter - private ConfigService configService; - - @Override - public void publish(Long uniqueSpace, MilogSpaceData config) { - log.info("write the creation namespace configuration:{}", gson.toJson(config)); - String dataId = CommonExtensionServiceFactory.getCommonExtensionService().getLogManagePrefix() + TAIL_CONFIG_DATA_ID + uniqueSpace; - try { - configService.publishConfig(dataId, DEFAULT_GROUP_ID, gson.toJson(config)); - } catch (NacosException e) { - log.error(String.format("Write the creation namespace configuration...,dataId:{},data:%s", dataId, gson.toJson(config)), e); - } - } - - @Override - public void remove(String spaceId) { - String dataId = CommonExtensionServiceFactory.getCommonExtensionService().getLogManagePrefix() + TAIL_CONFIG_DATA_ID + spaceId; - try { - configService.removeConfig(dataId, DEFAULT_GROUP_ID); - } catch (NacosException e) { - log.error(String.format("Delete log configuration data data exceptions,param：%s", dataId), e); - } - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/impl/StreamConfigNacosProvider.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/impl/StreamConfigNacosProvider.java deleted file mode 100644 index dfc123e49..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/impl/StreamConfigNacosProvider.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.nacos.impl; - -import com.alibaba.nacos.api.config.ConfigService; -import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionServiceFactory; -import com.xiaomi.mone.log.manager.service.nacos.DynamicConfigProvider; -import com.xiaomi.mone.log.model.MiLogStreamConfig; -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import static com.xiaomi.mone.log.common.Constant.*; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/16 15:27 - */ -@Slf4j -public class StreamConfigNacosProvider implements DynamicConfigProvider { - - @Setter - @Getter - private ConfigService configService; - - @Override - public MiLogStreamConfig getConfig(Long spaceId) { - String rules; - try { - if (null == spaceId) { - rules = configService.getConfig(CommonExtensionServiceFactory.getCommonExtensionService().getLogManagePrefix() + NAMESPACE_CONFIG_DATA_ID, DEFAULT_GROUP_ID, DEFAULT_TIME_OUT_MS); - } else { - rules = configService.getConfig(CommonExtensionServiceFactory.getCommonExtensionService().getSpaceDataId(spaceId), DEFAULT_GROUP_ID, DEFAULT_TIME_OUT_MS); - - } - log.info("The NACOS query log is initially configured：{}", rules); - if (StringUtils.isNotEmpty(rules)) { - return gson.fromJson(rules, MiLogStreamConfig.class); - } - } catch (Exception e) { - log.error("Query namespace configuration data data exceptions", e); - } - return null; - } - - public MiLogStreamConfig getConfig(String appName, String dataId) { - String rules = null; - try { - rules = configService.getConfig(dataId, DEFAULT_GROUP_ID, DEFAULT_TIME_OUT_MS); - if (StringUtils.isNotEmpty(rules)) { - return gson.fromJson(rules, MiLogStreamConfig.class); - } - } catch (Exception e) { - log.error(String.format("Query namespace configuration data data exceptions, parameters:%s", dataId), e); - } - return null; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/impl/StreamConfigNacosPublisher.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/impl/StreamConfigNacosPublisher.java deleted file mode 100644 index 66042d0d3..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/nacos/impl/StreamConfigNacosPublisher.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.nacos.impl; - -import com.alibaba.nacos.api.config.ConfigService; -import com.alibaba.nacos.api.exception.NacosException; -import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionServiceFactory; -import com.xiaomi.mone.log.manager.service.nacos.DynamicConfigPublisher; -import com.xiaomi.mone.log.model.MiLogStreamConfig; -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -import static com.xiaomi.mone.log.common.Constant.DEFAULT_GROUP_ID; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/7/15 15:14 - */ -@Slf4j -public class StreamConfigNacosPublisher implements DynamicConfigPublisher { - - @Setter - @Getter - private ConfigService configService; - - @Override - public synchronized void publish(Long spaceId, MiLogStreamConfig config) { - if (config == null) { - return; - } - try { - configService.publishConfig(CommonExtensionServiceFactory.getCommonExtensionService().getSpaceDataId(spaceId), DEFAULT_GROUP_ID, gson.toJson(config)); - } catch (NacosException e) { - log.error(String.format("Create namespace push data exceptions, parameters：%s", gson.toJson(config)), e); - } - } - - @Override - public void remove(String dataId) { - - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/path/LogPathMapping.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/path/LogPathMapping.java deleted file mode 100644 index 51d6c64c3..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/path/LogPathMapping.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.path; - -import com.xiaomi.mone.log.manager.model.vo.LogAgentListBo; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description Log path mapping rules - * @date 2022/11/15 18:50 - */ -public interface LogPathMapping { - String LOG_PATH_PREFIX = "/home/work/log"; - /** - * The log path after mapping - * - * @param origin - * @return - */ - String getLogPath(String origin, List logAgentListBos); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/path/LogPathMappingFactory.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/path/LogPathMappingFactory.java deleted file mode 100644 index cecaffdb3..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/path/LogPathMappingFactory.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.path; - -import org.apache.ozhera.app.enums.ProjectTypeEnum; -import com.xiaomi.youpin.docean.anno.Service; - -import javax.annotation.Resource; -import java.util.Objects; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/11/15 19:37 - */ -@Service -public class LogPathMappingFactory { - - @Resource - private MoneLogPathMapping moneLogPathMapping; - - public LogPathMapping queryLogPathMappingByAppType(Integer code) { - if (Objects.equals(ProjectTypeEnum.MIONE_TYPE.getCode(), code)) { - return moneLogPathMapping; - } - return null; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/path/MoneLogPathMapping.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/path/MoneLogPathMapping.java deleted file mode 100644 index 7b7c28f77..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/path/MoneLogPathMapping.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.path; - -import com.xiaomi.mone.log.manager.model.vo.LogAgentListBo; -import com.xiaomi.youpin.docean.anno.Service; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description The mone log path is based on docker, and the path has not changed - * @date 2022/11/15 18:51 - */ -@Service -public class MoneLogPathMapping implements LogPathMapping { - @Override - public String getLogPath(String originLogPath, List logAgentListBos) { - return originLogPath; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/AndAllStatementMatchParse.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/AndAllStatementMatchParse.java deleted file mode 100644 index 8e42e3355..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/AndAllStatementMatchParse.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.statement; - -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; - -import java.util.List; - -public class AndAllStatementMatchParse implements StatementMatchParse { - @Override - public BoolQueryBuilder matchBuild(List queryEntities) { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - for (QueryEntity entity : queryEntities) { - boolQueryBuilder.filter(QueryBuilders.queryStringQuery(entity.getFieldValue())); - } - return boolQueryBuilder; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/MustNotStatementMatchParse.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/MustNotStatementMatchParse.java deleted file mode 100644 index 590d28842..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/MustNotStatementMatchParse.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.statement; - -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; - -import java.util.List; - -public class MustNotStatementMatchParse implements StatementMatchParse { - @Override - public BoolQueryBuilder matchBuild(List queryEntities) { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - for (QueryEntity entity : queryEntities) { - boolQueryBuilder.mustNot(QueryBuilders.matchQuery(entity.getField(), entity.getFieldValue())); - } - return boolQueryBuilder; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/MustStatementMatchParse.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/MustStatementMatchParse.java deleted file mode 100644 index df42183f7..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/MustStatementMatchParse.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.statement; - -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; - -import java.util.List; - -import static com.xiaomi.mone.log.manager.service.statement.StatementMatchParseFactory.DOUBLE_QUOTATION_MARK_SEPARATOR; - -/** - * Must statement match - */ -public class MustStatementMatchParse implements StatementMatchParse { - @Override - public BoolQueryBuilder matchBuild(List queryEntities) { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - for (QueryEntity entity : queryEntities) { - if (entity.getFieldValue().startsWith(DOUBLE_QUOTATION_MARK_SEPARATOR)) { - boolQueryBuilder.must(QueryBuilders.matchPhraseQuery(entity.getField(), entity.getFieldValue())); - } else { - boolQueryBuilder.must(QueryBuilders.matchQuery(entity.getField(), entity.getFieldValue())); - } - } - return boolQueryBuilder; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/NotAllStatementMatchParse.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/NotAllStatementMatchParse.java deleted file mode 100644 index 8cefd22f2..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/NotAllStatementMatchParse.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.statement; - -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; - -import java.util.List; - -public class NotAllStatementMatchParse implements StatementMatchParse { - @Override - public BoolQueryBuilder matchBuild(List queryEntities) { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - for (QueryEntity entity : queryEntities) { - boolQueryBuilder.mustNot(QueryBuilders.queryStringQuery(entity.getFieldValue())); - } - return boolQueryBuilder; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/OperatorData.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/OperatorData.java deleted file mode 100644 index 8477121d5..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/OperatorData.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.statement; - -import com.xiaomi.mone.log.api.enums.EsOperatorEnum; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.List; - -@Data -@AllArgsConstructor -@NoArgsConstructor -@Builder -public class OperatorData { - private EsOperatorEnum operatorEnum; - private List messageList; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/OperatorSlice.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/OperatorSlice.java deleted file mode 100644 index 3d0d7489b..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/OperatorSlice.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.statement; - -import com.xiaomi.mone.log.api.enums.EsOperatorEnum; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@AllArgsConstructor -@NoArgsConstructor -@Builder -public class OperatorSlice { - private EsOperatorEnum operatorEnum; - private String message; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/OrAllStatementMatchParse.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/OrAllStatementMatchParse.java deleted file mode 100644 index 8efd306c8..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/OrAllStatementMatchParse.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.statement; - -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; - -import java.util.List; - -public class OrAllStatementMatchParse implements StatementMatchParse { - @Override - public BoolQueryBuilder matchBuild(List queryEntities) { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - for (QueryEntity entity : queryEntities) { - boolQueryBuilder.should(QueryBuilders.queryStringQuery(entity.getFieldValue())); - } - boolQueryBuilder.minimumShouldMatch(1); - return boolQueryBuilder; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/OrStatementMatchParse.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/OrStatementMatchParse.java deleted file mode 100644 index eac0aa352..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/OrStatementMatchParse.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.statement; - -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; - -import java.util.List; - -public class OrStatementMatchParse implements StatementMatchParse { - @Override - public BoolQueryBuilder matchBuild(List queryEntities) { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - for (QueryEntity entity : queryEntities) { - boolQueryBuilder.should(QueryBuilders.matchQuery(entity.getField(), entity.getFieldValue())); - } - boolQueryBuilder.minimumShouldMatch(1); - return boolQueryBuilder; - } -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/QueryEntity.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/QueryEntity.java deleted file mode 100644 index 908dc2953..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/QueryEntity.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.statement; - -import com.xiaomi.mone.log.api.enums.EsOperatorMatchEnum; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@AllArgsConstructor -@Builder -public class QueryEntity { - /** - * field name - */ - private String field; - /** - * field value - */ - private String fieldValue; - - private EsOperatorMatchEnum matchEnum; -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/StatementMatchParse.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/StatementMatchParse.java deleted file mode 100644 index 5582c7221..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/StatementMatchParse.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.statement; - -import org.elasticsearch.index.query.BoolQueryBuilder; - -import java.util.List; - -/** - * es Statement matching - */ -public interface StatementMatchParse { - - BoolQueryBuilder matchBuild(List queryEntities); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/StatementMatchParseFactory.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/StatementMatchParseFactory.java deleted file mode 100644 index 0aea3b1bc..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/statement/StatementMatchParseFactory.java +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.service.statement; - -import cn.hutool.core.util.StrUtil; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.xiaomi.mone.log.api.enums.EsOperatorEnum; -import com.xiaomi.mone.log.api.enums.EsOperatorMatchEnum; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; - -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static com.xiaomi.mone.log.api.enums.EsOperatorEnum.*; -import static com.xiaomi.mone.log.api.enums.EsOperatorMatchEnum.ALL_MATCH_OPERATOR; -import static com.xiaomi.mone.log.api.enums.EsOperatorMatchEnum.KV_MATCH_OPERATOR; -import static com.xiaomi.mone.log.common.Constant.SYMBOL_COLON; -import static com.xiaomi.mone.log.manager.common.utils.ManagerUtil.MatchKVPrefix; - -public class StatementMatchParseFactory { - - private static final String LOG_LEVEL_KEY = "level"; - private static final String NOT_MATCH_KEY = " not "; - private static final String AND_MATCH_KEY = " and "; - private static final String OR_MATCH_KEY = " or "; - public static final String DOUBLE_QUOTATION_MARK_SEPARATOR = "\""; - - private static Map> operateMatchMap = Maps.newHashMap(); - - /** - * Assemble the corresponding relationship - */ - static { - Map andOperateMap = Maps.newHashMap(); - andOperateMap.put(ALL_MATCH_OPERATOR, new AndAllStatementMatchParse()); - andOperateMap.put(KV_MATCH_OPERATOR, new MustStatementMatchParse()); - - Map notOperateMap = Maps.newHashMap(); - notOperateMap.put(ALL_MATCH_OPERATOR, new NotAllStatementMatchParse()); - notOperateMap.put(KV_MATCH_OPERATOR, new MustNotStatementMatchParse()); - - Map orOperateMap = Maps.newHashMap(); - orOperateMap.put(ALL_MATCH_OPERATOR, new OrAllStatementMatchParse()); - orOperateMap.put(KV_MATCH_OPERATOR, new OrStatementMatchParse()); - - operateMatchMap.put(AND_OPERATOR, andOperateMap); - operateMatchMap.put(NOT_OPERATOR, notOperateMap); - operateMatchMap.put(OR_OPERATOR, orOperateMap); - } - - public static BoolQueryBuilder getStatementMatchParseQueryBuilder(String message, List keyPrefixList) { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - List operatorSliceList = analyseTransformOs(message, keyPrefixList); - List operatorDataList = sliceTransformOd(operatorSliceList); - for (OperatorData operatorData : operatorDataList) { - BoolQueryBuilder queryBuilder = getSplitQueryBuilder(keyPrefixList, operatorData); - boolQueryBuilder.filter(queryBuilder); - } - return boolQueryBuilder; - } - - private static List sliceTransformOd(List operatorSliceList) { - List operatorSlices = Lists.newArrayList(); - Map> operatorEnumListMap = operatorSliceList.stream().collect(Collectors.groupingBy(OperatorSlice::getOperatorEnum)); - for (Map.Entry> enumListEntry : operatorEnumListMap.entrySet()) { - operatorSlices.add(OperatorData.builder().operatorEnum(enumListEntry.getKey()).messageList(enumListEntry.getValue().stream().map(OperatorSlice::getMessage).collect(Collectors.toList())).build()); - } - return operatorSlices; - } - - private static BoolQueryBuilder getSplitQueryBuilder(List keyPerfixList, OperatorData operatorSlice) { - List allMatchEntities = Lists.newArrayList(); - List kvMatchEntities = Lists.newArrayList(); - filterMatchEntities(keyPerfixList, operatorSlice, kvMatchEntities, allMatchEntities); - Map matchParseMap = operateMatchMap.get(operatorSlice.getOperatorEnum()); - BoolQueryBuilder allMatchBuilder = new BoolQueryBuilder(); - BoolQueryBuilder kvMatchBuilder = new BoolQueryBuilder(); - if (CollectionUtils.isNotEmpty(allMatchEntities)) { - allMatchBuilder = matchParseMap.get(allMatchEntities.get(0).getMatchEnum()).matchBuild(allMatchEntities); - } - if (CollectionUtils.isNotEmpty(kvMatchEntities)) { - kvMatchBuilder = matchParseMap.get(kvMatchEntities.get(0).getMatchEnum()).matchBuild(kvMatchEntities); - } - if (CollectionUtils.isNotEmpty(allMatchEntities) && CollectionUtils.isNotEmpty(kvMatchEntities)) { - allMatchBuilder.filter(kvMatchBuilder); - return allMatchBuilder; - } - if (CollectionUtils.isNotEmpty(allMatchEntities) && CollectionUtils.isEmpty(kvMatchEntities)) { - return allMatchBuilder; - } - if (CollectionUtils.isEmpty(allMatchEntities) && CollectionUtils.isNotEmpty(kvMatchEntities)) { - return kvMatchBuilder; - } - return allMatchBuilder; - } - - private static void filterMatchEntities(List keyPerfixList, OperatorData operatorSlice, List kvMatchEntities, List allMatchEntities) { - for (String message : operatorSlice.getMessageList()) { - String kvPrefix = MatchKVPrefix(message, keyPerfixList); - if (StringUtils.isNotBlank(kvPrefix)) { - String field = StringUtils.substringBefore(kvPrefix, SYMBOL_COLON); - String fieldValue = StrUtil.trimStart(StringUtils.substringAfter(message, SYMBOL_COLON)); -// if (Objects.equals(LOG_LEVEL_KEY, field)) { -// fieldValue = String.format("%-5s", fieldValue.trim()); -// } - kvMatchEntities.add(QueryEntity.builder() - .field(field) - .fieldValue(fieldValue) - .matchEnum(KV_MATCH_OPERATOR) - .build()); - } else { - allMatchEntities.add(QueryEntity.builder() - .fieldValue(StrUtil.trimStart(message)) - .matchEnum(ALL_MATCH_OPERATOR) - .build()); - } - } - } - - public static List analyseTransformOs(String message, List keyPrefixList) { - List operatorSlices = Lists.newArrayList(); - String kvPrefix = MatchKVPrefix(message, keyPrefixList); - String[] msgArrays = StringUtils.substringsBetween(message, "\"", "\""); - if (StringUtils.isBlank(kvPrefix) && null != msgArrays && msgArrays.length == 1) { - handleAndLogic(message, operatorSlices); - return operatorSlices; - } - if (message.contains(AND_MATCH_KEY)) { - handleAndLogic(message, operatorSlices); - } else if (message.contains(NOT_MATCH_KEY)) { - handleNotLogic(message, operatorSlices); - } else if (message.contains(OR_MATCH_KEY)) { - operatorSlicesAdd(operatorSlices, message, EsOperatorEnum.OR_OPERATOR); - } else { - buildOperatorToSlices(operatorSlices, EsOperatorEnum.AND_OPERATOR, message); - } - return operatorSlices; - } - - /** - * Handle logic that contains ands - * - * @param message - * @param operatorSlices - */ - private static void handleAndLogic(String message, List operatorSlices) { - List andList = splitBySeparator(message, AND_MATCH_KEY); - for (String andPer : andList) { - //Determine whether the keyword has been matched, and if so, you need to - if (!andPer.contains(NOT_MATCH_KEY) && !andPer.contains(OR_MATCH_KEY)) { - buildOperatorToSlices(operatorSlices, AND_OPERATOR, andPer); - } else if (andPer.contains(NOT_MATCH_KEY) && !andPer.contains(OR_MATCH_KEY)) { - if (andPer.startsWith(NOT_MATCH_KEY)) { - operatorSlicesAdd(operatorSlices, andPer, NOT_OPERATOR); - } else { - List matchList = splitBySeparator(andPer, NOT_OPERATOR.getCode()); - for (int i = 0; i < matchList.size(); i++) { - if (i == 0) { - buildOperatorToSlices(operatorSlices, AND_OPERATOR, matchList.get(i)); - } else { - buildOperatorToSlices(operatorSlices, NOT_OPERATOR, matchList.get(i)); - } - } - } - } else if (!andPer.contains(NOT_MATCH_KEY) && andPer.contains(OR_MATCH_KEY)) { - if (andPer.startsWith(OR_MATCH_KEY)) { - operatorSlicesAdd(operatorSlices, andPer, OR_OPERATOR); - } else { - List matchList = splitBySeparator(andPer, OR_OPERATOR.getCode()); - for (int i = 0; i < matchList.size(); i++) { - if (i == 0) { - buildOperatorToSlices(operatorSlices, AND_OPERATOR, matchList.get(i)); - } else { - buildOperatorToSlices(operatorSlices, OR_OPERATOR, matchList.get(i)); - - } - } - } - } - } - } - - /** - * Handle logic that does not contain AND and contains only NOT - * - * @param message - * @param operatorSlices - */ - private static void handleNotLogic(String message, List operatorSlices) { - List notList = splitBySeparator(message, NOT_MATCH_KEY); - for (int i = 0; i < notList.size(); i++) { - String notPer = notList.get(i); - if (notList.size() > 1 && i == 0) { - if (!notPer.contains(OR_MATCH_KEY)) { - buildOperatorToSlices(operatorSlices, EsOperatorEnum.AND_OPERATOR, notPer); - } else { - operatorSlicesAdd(operatorSlices, notPer, EsOperatorEnum.OR_OPERATOR); - } - } else { - if (!notPer.contains(OR_MATCH_KEY)) { - buildOperatorToSlices(operatorSlices, EsOperatorEnum.NOT_OPERATOR, notPer); - } else { - operatorSlicesAdd(operatorSlices, notPer, EsOperatorEnum.OR_OPERATOR); - } - } - } - } - - public static List splitBySeparator(String message, String separator) { - List sparMsgLists = Lists.newArrayList(); - while (StringUtils.isNotBlank(message)) { - int startIndex = 0; - int endIndex; - if (message.startsWith(DOUBLE_QUOTATION_MARK_SEPARATOR)) { - endIndex = message.indexOf(separator, message.indexOf(DOUBLE_QUOTATION_MARK_SEPARATOR, 1)); - } else { - endIndex = message.indexOf(separator); - } - if (endIndex == -1) { - endIndex = message.length(); - } - sparMsgLists.add(StringUtils.substring(message, startIndex, endIndex)); - message = StringUtils.substring(message, endIndex + separator.length()); - } - return sparMsgLists.stream().filter(StringUtils::isNotBlank).collect(Collectors.toList()); - } - - private static void operatorSlicesAdd(List operatorSlices, String message, EsOperatorEnum operatorEnum) { - List matchList = splitBySeparator(message, operatorEnum.getCode()); - for (String matchPer : matchList) { - buildOperatorToSlices(operatorSlices, operatorEnum, matchPer); - } - } - - private static void buildOperatorToSlices(List operatorSlices, EsOperatorEnum operatorEnum, String matchPer) { - operatorSlices.add(OperatorSlice.builder().operatorEnum(operatorEnum).message(matchPer).build()); - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/user/MoneUser.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/user/MoneUser.java deleted file mode 100644 index 131c75f05..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/user/MoneUser.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.user; - -import com.google.gson.annotations.SerializedName; -import lombok.Builder; -import lombok.Data; -import lombok.ToString; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/2 15:59 - */ -@Data -@ToString -@Builder -public class MoneUser { - - /** - * Username, such as: zhangsan - */ - @SerializedName("cas:user") - private String user; - - private String fullUser; - - /** - * Username, such as: Zhang San - */ - @SerializedName("cas:name") - private String name; - - /** - * User display name, such as: sa zhang Zhangsan - */ - @SerializedName("cas:displayName") - private String displayName; - - /** - * Department name - */ - @SerializedName("cas:departmentName") - private String departmentName; - - /** - * Email - */ - @SerializedName("cas:email") - private String email; - - /** - * miID - */ - @SerializedName("cas:miID") - private String miID; - - /** - * miID - */ - @SerializedName("cas:uid") - private String uID; - - /** - * avatar - */ - @SerializedName("cas:avatar") - private String avatar; - - private Boolean isAdmin; - - private String zone; - /** - * Last level department ID - */ - private String deptId; - - private String company; - - private Integer userType; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/user/MoneUserDetailService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/user/MoneUserDetailService.java deleted file mode 100644 index 038d65fce..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/user/MoneUserDetailService.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.user; - -import com.google.gson.Gson; -import com.google.gson.JsonArray; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/7 10:36 - */ -public interface MoneUserDetailService { - - Gson GSON = new Gson(); - - /** - * Query user details - * - * @param userName - * @return - */ - UseDetailInfo queryUserByUserName(String userName); - - /** - * Query user details - * - * @param uId - * @return - */ - UseDetailInfo queryUser(String uId); - - /** - * Query the user's unique ID based on the mobile phone - * - * @param phone - * @return - */ - String queryUserUIdByPhone(String phone); - - /** - * Query the user's unique ID based on the employee number - * - * @param empId - * @return - */ - String queryUserUIdByEmpId(String empId); - - /** - * Query the user's unique ID based on the user name - * - * @param email - * @return - */ - String queryUserUIdByUserName(String email); - - JsonArray queryChildDept(String deptId); - - List queryDeptPersonIds(String deptId); -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/user/MoneUtil.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/user/MoneUtil.java deleted file mode 100644 index 891f585d4..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/user/MoneUtil.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.user; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2023/4/11 15:49 - */ -public class MoneUtil { - - public final static String MONE_USER_INFO = "x-proxy-mone-user"; - - public final static String MONE_USER_INFO_EXAM = "X-Proxy-Mone-User"; - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/user/UseDetailInfo.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/user/UseDetailInfo.java deleted file mode 100644 index 2fd94d218..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/user/UseDetailInfo.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.user; - -import com.google.gson.annotations.SerializedName; -import lombok.Data; - -import java.util.List; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2021/9/6 11:34 - */ -@Data -public class UseDetailInfo { - - private String uid; - private String personId; - private String originalPersonId; - private String name; - private String displayName; - private String type; - private String userName; - private String email; - private String sex; - private String headUrl; - private String deptId; - @SerializedName("deptDescr") - private String deptDesc; - @SerializedName("fullDeptDescr") - private String fullDeptDesc; - private List fullDeptDescriptorList; - private String company; - @SerializedName("companyDescr") - private String companyDesc; - private String hrStatus; - private String source; - - - @Data - public static class DeptDescriptor { - private String deptEnName; - private String deptId; - private String deptName; - private Integer level; - } - -} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/user/UserIDService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/user/UserIDService.java deleted file mode 100644 index 3212d0cad..000000000 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/user/UserIDService.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.manager.user; - -/** - * @author wtt - * @version 1.0 - * @description Get the user's unique ID - * @date 2021/9/7 10:48 - */ -public interface UserIDService { - - String findUserId(String signature); -} diff --git a/ozhera-log/log-manager/src/main/resources/mapper/MilogAnalyseDashboardGraphRefMapper.xml b/ozhera-log/log-manager/src/main/resources/mapper/MilogAnalyseDashboardGraphRefMapper.xml index 15c15295d..53b491a95 100644 --- a/ozhera-log/log-manager/src/main/resources/mapper/MilogAnalyseDashboardGraphRefMapper.xml +++ b/ozhera-log/log-manager/src/main/resources/mapper/MilogAnalyseDashboardGraphRefMapper.xml @@ -1,6 +1,6 @@ - + DELETE FROM @@ -17,7 +17,7 @@ AND graph_id = #{graphId} - SELECT * FROM @@ -34,7 +34,7 @@ dashboard_id = #{dashboardId} AND graph_id = #{graphId} - SELECT * FROM diff --git a/ozhera-log/log-manager/src/main/resources/mapper/MilogAnalyseDashboardMapper.xml b/ozhera-log/log-manager/src/main/resources/mapper/MilogAnalyseDashboardMapper.xml index ed9964952..60f01cc17 100644 --- a/ozhera-log/log-manager/src/main/resources/mapper/MilogAnalyseDashboardMapper.xml +++ b/ozhera-log/log-manager/src/main/resources/mapper/MilogAnalyseDashboardMapper.xml @@ -1,9 +1,9 @@ - + - SELECT * FROM diff --git a/ozhera-log/log-manager/src/main/resources/mapper/MilogAnalyseGraphMapper.xml b/ozhera-log/log-manager/src/main/resources/mapper/MilogAnalyseGraphMapper.xml index 659487bf4..8398f13b8 100644 --- a/ozhera-log/log-manager/src/main/resources/mapper/MilogAnalyseGraphMapper.xml +++ b/ozhera-log/log-manager/src/main/resources/mapper/MilogAnalyseGraphMapper.xml @@ -1,8 +1,8 @@ - + - SELECT ref.private_name graphPrivateName, graph.`name` graphName, @@ -18,7 +18,7 @@ AND graph.graph_type = type.id and ref.dashboard_id = #{dashboardId} - SELECT graph.`name` name, graph.field_name fieldName, diff --git a/ozhera-log/log-manager/src/main/resources/mapper/MilogAnalyseGraphTypeMapper.xml b/ozhera-log/log-manager/src/main/resources/mapper/MilogAnalyseGraphTypeMapper.xml index c411fc67e..a0c17e45c 100644 --- a/ozhera-log/log-manager/src/main/resources/mapper/MilogAnalyseGraphTypeMapper.xml +++ b/ozhera-log/log-manager/src/main/resources/mapper/MilogAnalyseGraphTypeMapper.xml @@ -1,5 +1,5 @@ - + diff --git a/ozhera-log/log-manager/src/main/resources/mapper/MilogEsClusterMapper.xml b/ozhera-log/log-manager/src/main/resources/mapper/MilogEsClusterMapper.xml index 91a806a49..b71e13732 100644 --- a/ozhera-log/log-manager/src/main/resources/mapper/MilogEsClusterMapper.xml +++ b/ozhera-log/log-manager/src/main/resources/mapper/MilogEsClusterMapper.xml @@ -1,8 +1,8 @@ - + - SELECT * FROM @@ -10,13 +10,13 @@ WHERE tag = #{tag} - SELECT * FROM `milog_es_cluster` - SELECT * FROM @@ -24,7 +24,7 @@ WHERE region = #{region} - SELECT * FROM @@ -33,7 +33,7 @@ area = #{area} and labels like CONCAT('%',#{label},'%'); - SELECT * FROM @@ -41,7 +41,7 @@ WHERE name = #{alias} - SELECT * FROM diff --git a/ozhera-log/log-manager/src/main/resources/mapper/MilogEsIndexMapper.xml b/ozhera-log/log-manager/src/main/resources/mapper/MilogEsIndexMapper.xml index ecda251d8..463e9628d 100644 --- a/ozhera-log/log-manager/src/main/resources/mapper/MilogEsIndexMapper.xml +++ b/ozhera-log/log-manager/src/main/resources/mapper/MilogEsIndexMapper.xml @@ -1,8 +1,8 @@ - + - SELECT idx.* FROM @@ -14,7 +14,7 @@ AND clu.area = #{region} - SELECT idx.* FROM diff --git a/ozhera-log/log-manager/src/main/resources/mapper/MilogLogCountMapper.xml b/ozhera-log/log-manager/src/main/resources/mapper/MilogLogCountMapper.xml index 8a27f8e2f..fbd25d047 100644 --- a/ozhera-log/log-manager/src/main/resources/mapper/MilogLogCountMapper.xml +++ b/ozhera-log/log-manager/src/main/resources/mapper/MilogLogCountMapper.xml @@ -1,6 +1,6 @@ - + INSERT INTO milog_log_count diff --git a/ozhera-log/log-manager/src/main/resources/mapper/MilogLogNumAlertMapper.xml b/ozhera-log/log-manager/src/main/resources/mapper/MilogLogNumAlertMapper.xml index a67a38396..fd8b365c4 100644 --- a/ozhera-log/log-manager/src/main/resources/mapper/MilogLogNumAlertMapper.xml +++ b/ozhera-log/log-manager/src/main/resources/mapper/MilogLogNumAlertMapper.xml @@ -1,6 +1,6 @@ - + INSERT INTO milog_log_num_alert diff --git a/ozhera-log/log-manager/src/main/resources/mapper/MilogLogProcessMapper.xml b/ozhera-log/log-manager/src/main/resources/mapper/MilogLogProcessMapper.xml index 2b33017b0..ff243f680 100644 --- a/ozhera-log/log-manager/src/main/resources/mapper/MilogLogProcessMapper.xml +++ b/ozhera-log/log-manager/src/main/resources/mapper/MilogLogProcessMapper.xml @@ -1,6 +1,6 @@ - + \ No newline at end of file diff --git a/ozhera-log/log-manager/src/main/resources/mapper/MilogLogSearchSaveMapper.xml b/ozhera-log/log-manager/src/main/resources/mapper/MilogLogSearchSaveMapper.xml index 902093435..c1d6e4274 100644 --- a/ozhera-log/log-manager/src/main/resources/mapper/MilogLogSearchSaveMapper.xml +++ b/ozhera-log/log-manager/src/main/resources/mapper/MilogLogSearchSaveMapper.xml @@ -1,6 +1,6 @@ - + DELETE FROM @@ -9,7 +9,7 @@ id = #{id} - SELECT * FROM @@ -39,7 +39,7 @@ AND creator = #{creator} - SELECT t.*, GROUP_CONCAT( DISTINCT CASE WHEN t.tail IS NULL THEN '' ELSE t.tail END ) tailName, @@ -98,7 +98,7 @@ AND sort = 2 - SELECT * FROM diff --git a/ozhera-log/log-manager/src/main/resources/mapper/MilogLogTemplateDetailMapper.xml b/ozhera-log/log-manager/src/main/resources/mapper/MilogLogTemplateDetailMapper.xml index f5acc5270..4cc68d4bc 100644 --- a/ozhera-log/log-manager/src/main/resources/mapper/MilogLogTemplateDetailMapper.xml +++ b/ozhera-log/log-manager/src/main/resources/mapper/MilogLogTemplateDetailMapper.xml @@ -1,8 +1,8 @@ - + - SELECT * FROM diff --git a/ozhera-log/log-manager/src/main/resources/mapper/MilogLogTemplateMapper.xml b/ozhera-log/log-manager/src/main/resources/mapper/MilogLogTemplateMapper.xml index 15a6cda9f..893c9a528 100644 --- a/ozhera-log/log-manager/src/main/resources/mapper/MilogLogTemplateMapper.xml +++ b/ozhera-log/log-manager/src/main/resources/mapper/MilogLogTemplateMapper.xml @@ -1,8 +1,8 @@ - + - SELECT * FROM diff --git a/ozhera-log/log-manager/src/main/resources/mapper/MilogLogstailMapper.xml b/ozhera-log/log-manager/src/main/resources/mapper/MilogLogstailMapper.xml index f033c6b1c..c2448b7df 100644 --- a/ozhera-log/log-manager/src/main/resources/mapper/MilogLogstailMapper.xml +++ b/ozhera-log/log-manager/src/main/resources/mapper/MilogLogstailMapper.xml @@ -1,6 +1,6 @@ - +