FIX : Suppression conf log4j obsolète

Mise en place lecture globale dans kafka avant d'envoyer vers stockage
Changement système de stockage de postgres vers ES

pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>org.springframework.boot</groupId>
         <artifactId>spring-boot-starter-parent</artifactId>
-        <version>3.0.4</version>
+        <version>3.2.8</version>
         <relativePath/> <!-- lookup parent from repository -->
     </parent>
     <groupId>fr.abes</groupId>
@@ -76,28 +76,19 @@
         <dependency>
             <groupId>org.springframework.kafka</groupId>
             <artifactId>spring-kafka</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-data-jpa</artifactId>
             <exclusions>
                 <exclusion>
-                    <groupId>org.springframework.boot</groupId>
-                    <artifactId>spring-boot-starter-logging</artifactId>
+                    <groupId>org.scala-lang</groupId>
+                    <artifactId>scala-library</artifactId>
                 </exclusion>
             </exclusions>
         </dependency>
+
         <dependency>
-            <groupId>org.postgresql</groupId>
-            <artifactId>postgresql</artifactId>
-        </dependency>
-        <!-- BDD -->
-        <dependency>
-            <groupId>com.oracle.database.jdbc</groupId>
-            <artifactId>ojdbc8</artifactId>
-            <version>21.9.0.0</version>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
         </dependency>
+
         <!-- === Logging === -->
         <dependency>
             <groupId>org.springframework.boot</groupId>
@@ -135,11 +126,6 @@
         </dependency>
 
         <!-- Utilitaires -->
-        <dependency>
-            <groupId>org.projectlombok</groupId>
-            <artifactId>lombok</artifactId>
-            <optional>true</optional>
-        </dependency>
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpmime</artifactId>
@@ -150,7 +136,7 @@
         <dependency>
             <groupId>org.springdoc</groupId>
             <artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
-            <version>2.1.0</version>
+            <version>2.3.0</version>
         </dependency>
 
         <!-- gestion des LocalDateTime -->
@@ -161,7 +147,7 @@
         <dependency>
             <groupId>com.fasterxml.jackson.dataformat</groupId>
             <artifactId>jackson-dataformat-xml</artifactId>
-            <version>2.12.6</version>
+            <version>2.17.0</version>
         </dependency>
 
         <!-- TEST -->
@@ -171,9 +157,9 @@
             <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>org.springframework.kafka</groupId>
-            <artifactId>spring-kafka-test</artifactId>
-            <scope>test</scope>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <scope>provided</scope>
         </dependency>
     </dependencies>
 

src/main/java/fr/abes/logskbart/configuration/KafkaConfig.java

@@ -1,5 +1,6 @@
 package fr.abes.logskbart.configuration;
 
+import fr.abes.logskbart.kafka.WorkInProgress;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.common.serialization.StringDeserializer;
 import org.springframework.beans.factory.annotation.Value;
@@ -9,11 +10,13 @@
 import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
 import org.springframework.kafka.core.ConsumerFactory;
 import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
+import org.springframework.kafka.listener.ContainerProperties;
 
 import java.sql.Timestamp;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
 
 @Configuration
 @EnableKafka
@@ -30,8 +33,8 @@ public ConsumerFactory<String, String> consumerLogsFactory() {
         props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
         props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
         props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true);
-        props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 10);
-        props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, 60000);
+        props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 100);
+        props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, 30000);
         return new DefaultKafkaConsumerFactory<>(props);
     }
 
@@ -45,7 +48,7 @@ public ConsumerFactory<String, String> consumerLogsFactory() {
     }
 
     @Bean
-    public Map<String, Timestamp> lastTimeStampByFilename() {
-        return new HashMap<>();
+    public Map<String, WorkInProgress> workInProgressMap() {
+        return new ConcurrentHashMap<>();
     }
 }

src/main/java/fr/abes/logskbart/controller/LogsController.java

@@ -1,7 +1,7 @@
 package fr.abes.logskbart.controller;
 
 import fr.abes.logskbart.dto.LigneLogDto;
-import fr.abes.logskbart.dto.LogDto;
+import fr.abes.logskbart.dto.LogWebDto;
 import fr.abes.logskbart.service.LogsService;
 import fr.abes.logskbart.utils.UtilsMapper;
 import io.swagger.v3.oas.annotations.Operation;
@@ -34,18 +34,18 @@ public LogsController(LogsService service, UtilsMapper mapper) {
             summary = "Transfer kafka to PostgreSQL DB",
             description = "Retrieves kbart load logs from a Kafka bus and stores them in a DB for later availability",
             responses = {
-                    @ApiResponse( responseCode = "200", description = "The request was successful.", content = { @Content(schema = @Schema(implementation = LogDto.class), mediaType = "application/json") } ),
+                    @ApiResponse( responseCode = "200", description = "The request was successful.", content = { @Content(schema = @Schema(implementation = LogWebDto.class), mediaType = "application/json") } ),
                     @ApiResponse( responseCode = "400", description = "An element of the query is badly formulated.", content = { @Content(schema = @Schema()) } ),
                     @ApiResponse( responseCode = "500", description = "An internal server error interrupted processing.", content = { @Content(schema = @Schema()) } ),
             }
     )
     @GetMapping("/logs/{filename}/{date}")
-    public LogDto getLogsFromPackageAndDate(@PathVariable String filename, @PathVariable String date) throws ParseException {
+    public LogWebDto getLogsFromPackageAndDate(@PathVariable String filename, @PathVariable String date) throws ParseException {
         SimpleDateFormat format = new SimpleDateFormat("ddMMyyyy");
         Date dateAnalyse = format.parse(date);
-        LogDto logDto = new LogDto(filename, date);
+        LogWebDto logWebDto = new LogWebDto(filename, date);
         List<LigneLogDto> lignes = mapper.mapList(service.getLogKbartForPackage(filename, dateAnalyse), LigneLogDto.class);
-        logDto.addLignes(lignes);
-        return logDto;
+        logWebDto.addLignes(lignes);
+        return logWebDto;
     }
 }

src/main/java/fr/abes/logskbart/dto/LogDto.java → src/main/java/fr/abes/logskbart/dto/LogWebDto.java

@@ -8,12 +8,12 @@
 
 @Getter
 @Setter
-public class LogDto {
+public class LogWebDto {
     public String filename;
     public String date;
     List<LigneLogDto> ligneLogs;
 
-    public LogDto(String filename, String date) {
+    public LogWebDto(String filename, String date) {
         this.filename = filename;
         this.date = date;
         this.ligneLogs = new ArrayList<>();

src/main/java/fr/abes/logskbart/entity/LogKbart.java

@@ -1,62 +1,57 @@
 package fr.abes.logskbart.entity;
 
 import fr.abes.logskbart.utils.Level;
-import jakarta.persistence.*;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
+import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.data.annotation.Id;
+import org.springframework.data.elasticsearch.annotations.Document;
+import org.springframework.data.elasticsearch.annotations.Field;
 
 import java.io.Serializable;
 import java.util.Date;
 
-@Entity
-@Table(name = "LOGKBART")
-@Getter
-@Setter
-@NoArgsConstructor
+@Document(indexName = "logkbart")
+@Data
 @Slf4j
 public class LogKbart implements Serializable {
     @Id
-    @Column(name = "ID")
-    @GeneratedValue(strategy = GenerationType.IDENTITY)
-    private Long id;
+    @Field(name = "ID")
+    private String id;
 
-    @Column(name = "PACKAGE_NAME")
+    @Field(name = "PACKAGE_NAME")
     private String packageName;
 
-    @Column(name = "TIMESTAMP")
+    @Field(name = "TIMESTAMP")
     private Date timestamp;
 
-    @Column(name = "THREAD")
+    @Field(name = "THREAD")
     private String thread;
 
-    @Column(name = "LEVEL")
-    @Enumerated(EnumType.STRING)
-    private Level level;
+    @Field(name = "LEVEL")
+    private String level;
 
-    @Column(name = "LOGGER_NAME")
+    @Field(name = "LOGGER_NAME")
     private String loggerName;
 
-    @Column(name = "message", length = 2048)
+    @Field(name = "MESSAGE")
     private String message;
 
-    @Column(name = "END_OF_BATCH")
+    @Field(name = "END_OF_BATCH")
     private boolean endOfBatch;
 
-    @Column(name = "LOGGER_FQCN")
+    @Field(name = "LOGGER_FQCN")
     private String loggerFqcn;
 
-    @Column(name = "THREAD_ID")
+    @Field(name = "THREAD_ID")
     private Integer threadId;
 
-    @Column(name = "THREAD_PRIORITY")
+    @Field(name = "THREAD_PRIORITY")
     private Integer threadPriority;
 
-    @Column(name = "NB_LINE", nullable = false)
+    @Field(name = "NB_LINE")
     private Integer nbLine;
 
-    @Column(name = "NB_RUN", nullable = false)
+    @Field(name = "NB_RUN")
     private Integer nbRun = 0;
 
     @Override