FIX : Modification index pour optimisation recherches

Suppression liste dans retour controller Ajout insertion dans un fichier renvoyé par le controller pour récupérer les logs d'un package Mise à jour versions librairies
abes-esr · Oct 23, 2024 · 7cab97e · 7cab97e
1 parent 8696ad5
commit 7cab97e
Show file tree

Hide file tree

Showing 8 changed files with 100 additions and 35 deletions.
diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh
@@ -5,7 +5,7 @@ export SPRING_ELASTICSEARCH_URIS=${SPRING_ELASTICSEARCH_URIS:='http://localhost:
 curl -X PUT "$SPRING_ELASTICSEARCH_URIS/logkbart" -H 'Content-Type: application/json' -d'
      {
      	"settings": {
-     		"number_of_shards": 5,
+     		"number_of_shards": 1,
      		"number_of_replicas": 0
      	},
      	"mappings": {
@@ -14,7 +14,13 @@ curl -X PUT "$SPRING_ELASTICSEARCH_URIS/logkbart" -H 'Content-Type: application/
      				"type": "text"
      			},
      			"PACKAGE_NAME": {
-     				"type": "text"
+     				"type": "text",
+     				"fields": {
+     				  "keyword": {
+     				    "type": "keyword",
+     				    "ignore_above": 256
+     				  }
+     				}
      			},
      			"TIMESTAMP": {
      				"type": "date"

diff --git a/pom.xml b/pom.xml
@@ -87,6 +87,12 @@
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.httpcomponents</groupId>
+                    <artifactId>httpclient</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <!-- === Logging === -->
@@ -105,12 +111,11 @@
                 </exclusion>
             </exclusions>
         </dependency>
-
         <!-- === Mapping === -->
         <dependency>
             <groupId>org.modelmapper</groupId>
             <artifactId>modelmapper</artifactId>
-            <version>2.3.5</version>
+            <version>3.1.1</version>
         </dependency>
 
         <dependency>
@@ -139,17 +144,6 @@
             <version>2.3.0</version>
         </dependency>
 
-        <!-- gestion des LocalDateTime -->
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-json</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.dataformat</groupId>
-            <artifactId>jackson-dataformat-xml</artifactId>
-            <version>2.17.0</version>
-        </dependency>
-
         <!-- TEST -->
         <dependency>
             <groupId>org.springframework.boot</groupId>

diff --git a/src/main/java/fr/abes/logskbart/controller/LogsController.java b/src/main/java/fr/abes/logskbart/controller/LogsController.java
@@ -2,15 +2,25 @@
 
 import fr.abes.logskbart.dto.LigneLogDto;
 import fr.abes.logskbart.dto.LogWebDto;
+import fr.abes.logskbart.entity.LogKbart;
+import fr.abes.logskbart.exception.EmptyFileException;
 import fr.abes.logskbart.service.LogsService;
 import fr.abes.logskbart.utils.UtilsMapper;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.tags.Tag;
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Date;
@@ -23,29 +33,37 @@
 public class LogsController {
     private final LogsService service;
 
-    private final UtilsMapper mapper;
 
-    public LogsController(LogsService service, UtilsMapper mapper) {
+    public LogsController(LogsService service) {
         this.service = service;
-        this.mapper = mapper;
     }
 
     @Operation(
             summary = "Transfer kafka to PostgreSQL DB",
             description = "Retrieves kbart load logs from a Kafka bus and stores them in a DB for later availability",
             responses = {
-                    @ApiResponse( responseCode = "200", description = "The request was successful.", content = { @Content(schema = @Schema(implementation = LogWebDto.class), mediaType = "application/json") } ),
-                    @ApiResponse( responseCode = "400", description = "An element of the query is badly formulated.", content = { @Content(schema = @Schema()) } ),
-                    @ApiResponse( responseCode = "500", description = "An internal server error interrupted processing.", content = { @Content(schema = @Schema()) } ),
+                    @ApiResponse(responseCode = "200", description = "The request was successful.", content = {@Content(schema = @Schema(implementation = LogWebDto.class), mediaType = "application/json")}),
+                    @ApiResponse(responseCode = "400", description = "An element of the query is badly formulated.", content = {@Content(schema = @Schema())}),
+                    @ApiResponse(responseCode = "500", description = "An internal server error interrupted processing.", content = {@Content(schema = @Schema())}),
             }
     )
     @GetMapping("/logs/{filename}/{date}")
-    public LogWebDto getLogsFromPackageAndDate(@PathVariable String filename, @PathVariable String date) throws ParseException {
+    public ResponseEntity<InputStreamResource> getLogsFromPackageAndDate(@PathVariable String filename, @PathVariable String date) throws ParseException, IOException, EmptyFileException {
         SimpleDateFormat format = new SimpleDateFormat("ddMMyyyy");
         Date dateAnalyse = format.parse(date);
-        LogWebDto logWebDto = new LogWebDto(filename, date);
-        List<LigneLogDto> lignes = mapper.mapList(service.getLogKbartForPackage(filename, dateAnalyse), LigneLogDto.class);
-        logWebDto.addLignes(lignes);
-        return logWebDto;
+        File fichier = service.getLogKbartForPackage(filename, dateAnalyse);
+        FileInputStream fs = new FileInputStream(fichier);
+        // Définir les en-têtes pour la réponse HTTP
+        HttpHeaders headers = new HttpHeaders();
+        headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + fichier.getName());
+        headers.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM_VALUE);
+
+        // Retourner la réponse avec le fichier à télécharger
+        return ResponseEntity.ok()
+                .headers(headers)
+                .contentLength(fichier.length())
+                .contentType(MediaType.APPLICATION_OCTET_STREAM)
+                .body(new InputStreamResource(fs));
+
     }
 }
diff --git a/src/main/java/fr/abes/logskbart/entity/LogKbart.java b/src/main/java/fr/abes/logskbart/entity/LogKbart.java
@@ -1,14 +1,12 @@
 package fr.abes.logskbart.entity;
 
-import fr.abes.logskbart.utils.Level;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.data.annotation.Id;
 import org.springframework.data.elasticsearch.annotations.Document;
 import org.springframework.data.elasticsearch.annotations.Field;
 
 import java.io.Serializable;
-import java.util.Comparator;
 import java.util.Date;
 import java.util.Objects;
 

diff --git a/src/main/java/fr/abes/logskbart/exception/EmptyFileException.java b/src/main/java/fr/abes/logskbart/exception/EmptyFileException.java
@@ -0,0 +1,7 @@
+package fr.abes.logskbart.exception;
+
+public class EmptyFileException extends Throwable {
+    public EmptyFileException(String message) {
+        super(message);
+    }
+}
diff --git a/src/main/java/fr/abes/logskbart/exception/ExceptionControllerHandler.java b/src/main/java/fr/abes/logskbart/exception/ExceptionControllerHandler.java
@@ -19,6 +19,10 @@ private ResponseEntity<Object> buildResponseEntity(ApiReturnError apiReturnError
         return new ResponseEntity<>(apiReturnError, apiReturnError.getStatus());
     }
 
+    @ExceptionHandler(EmptyFileException.class)
+    protected ResponseEntity<Object> handleEmptyFileException(EmptyFileException e) {
+        return buildResponseEntity(new ApiReturnError(HttpStatus.NOT_FOUND, e.getMessage()));
+    }
     /**
      * Erreur dans la validité des paramètres de la requête
      *

diff --git a/src/main/java/fr/abes/logskbart/repository/LogKbartRepository.java b/src/main/java/fr/abes/logskbart/repository/LogKbartRepository.java
@@ -1,15 +1,16 @@
 package fr.abes.logskbart.repository;
 
 import fr.abes.logskbart.entity.LogKbart;
+import org.springframework.data.domain.*;
 import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
 import org.springframework.stereotype.Repository;
 
 import java.util.Date;
-import java.util.List;
+import java.util.stream.Stream;
 
 @Repository
 public interface LogKbartRepository extends ElasticsearchRepository<LogKbart, Long> {
-    List<LogKbart> findAllByPackageNameAndTimestampBetweenOrderByNbLineAscTimestampAsc(String filename, Date debut, Date fin);
-
+    long countByPackageNameAndTimestampBetween(String packageName, Date debut, Date fin);
 
+    Stream<LogKbart> findAllByPackageNameAndTimestampBetweenOrderByNbLineAscTimestampAsc(String filename, Date debut, Date fin);
 }
diff --git a/src/main/java/fr/abes/logskbart/service/LogsService.java b/src/main/java/fr/abes/logskbart/service/LogsService.java
@@ -1,30 +1,67 @@
 package fr.abes.logskbart.service;
 
 import fr.abes.logskbart.entity.LogKbart;
+import fr.abes.logskbart.exception.EmptyFileException;
 import fr.abes.logskbart.repository.LogKbartRepository;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
 
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.List;
+import java.util.stream.Stream;
 
 @Service
 @Slf4j
 public class LogsService {
+    private static final Integer MAXSIZE = 5000;
     private final LogKbartRepository repository;
 
-    public LogsService(LogKbartRepository repository) {
-        this.repository = repository;
+    public LogsService(LogKbartRepository logKbartRepository) {
+        this.repository = logKbartRepository;
     }
 
-    public List<LogKbart> getLogKbartForPackage(String packageName, Date date) {
+    @Transactional(readOnly = true)
+    public File getLogKbartForPackage(String packageName, Date date) throws IOException, EmptyFileException {
         Calendar dateChargement = Calendar.getInstance();
         dateChargement.setTime(date);
         Calendar dateFin = (Calendar) dateChargement.clone();
         dateFin.add(Calendar.DAY_OF_MONTH, 1);
         log.debug("packageName {}, Date début {}, Date fin {}", packageName, dateChargement.getTime(), dateFin.getTime());
-        return repository.findAllByPackageNameAndTimestampBetweenOrderByNbLineAscTimestampAsc(packageName, dateChargement.getTime(), dateFin.getTime());
+        if (repository.countByPackageNameAndTimestampBetween(packageName, dateChargement.getTime(), dateFin.getTime()) == 0)
+            throw new EmptyFileException("Aucun log pour le fichier " + packageName);
+        Path tempPath = Path.of("tempLogLocal");
+        if (!Files.exists(tempPath)) {
+            Files.createDirectory(tempPath);
+        }
+        Path pathOfLocal = Path.of("tempLogLocal" + File.separator + packageName.replace(".tsv", ".log"));
+        Files.deleteIfExists(pathOfLocal);
+        try (Stream<LogKbart> logKbarts = repository.findAllByPackageNameAndTimestampBetweenOrderByNbLineAscTimestampAsc(packageName, dateChargement.getTime(), dateFin.getTime())) {
+            logKbarts.forEach(logKbart -> {
+                String message = (logKbart.getNbLine() != -1) ? logKbart.getNbLine() + " : " : "";
+                message += logKbart.getMessage() + System.lineSeparator();
+                try {
+                    if (Files.exists(pathOfLocal)) {
+                        Files.write(pathOfLocal, message.getBytes(), StandardOpenOption.APPEND);
+                    } else {
+                        //  Créer le fichier et inscrit la ligne dedans
+                        Files.createFile(pathOfLocal);
+                        //  Inscrit les informations sur la ligne
+                        Files.write(pathOfLocal, message.getBytes(), StandardOpenOption.APPEND);
+                        log.debug("Fichier temporaire créé.");
+                    }
+                }catch (IOException e) {
+                    log.error("Impossible d'écrire dans le fichier local", e);
+                }
+            });
+        }
+        return pathOfLocal.toFile();
     }
 
     public void saveAll(List<LogKbart> logKbarts) {