From 7cab97ea474feb0be30273d365e72f8dca48b3f9 Mon Sep 17 00:00:00 2001 From: pierre-maraval Date: Wed, 23 Oct 2024 15:28:37 +0200 Subject: [PATCH] =?UTF-8?q?FIX=20:=20Modification=20index=20pour=20optimis?= =?UTF-8?q?ation=20recherches=20Suppression=20liste=20dans=20retour=20cont?= =?UTF-8?q?roller=20Ajout=20insertion=20dans=20un=20fichier=20renvoy=C3=A9?= =?UTF-8?q?=20par=20le=20controller=20pour=20r=C3=A9cup=C3=A9rer=20les=20l?= =?UTF-8?q?ogs=20d'un=20package=20Mise=20=C3=A0=20jour=20versions=20librai?= =?UTF-8?q?ries?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker/docker-entrypoint.sh | 10 ++++- pom.xml | 20 +++------ .../logskbart/controller/LogsController.java | 40 ++++++++++++----- .../fr/abes/logskbart/entity/LogKbart.java | 2 - .../exception/EmptyFileException.java | 7 +++ .../exception/ExceptionControllerHandler.java | 4 ++ .../repository/LogKbartRepository.java | 7 +-- .../abes/logskbart/service/LogsService.java | 45 +++++++++++++++++-- 8 files changed, 100 insertions(+), 35 deletions(-) create mode 100644 src/main/java/fr/abes/logskbart/exception/EmptyFileException.java diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 6a607fd..8d946ea 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -5,7 +5,7 @@ export SPRING_ELASTICSEARCH_URIS=${SPRING_ELASTICSEARCH_URIS:='http://localhost: curl -X PUT "$SPRING_ELASTICSEARCH_URIS/logkbart" -H 'Content-Type: application/json' -d' { "settings": { - "number_of_shards": 5, + "number_of_shards": 1, "number_of_replicas": 0 }, "mappings": { @@ -14,7 +14,13 @@ curl -X PUT "$SPRING_ELASTICSEARCH_URIS/logkbart" -H 'Content-Type: application/ "type": "text" }, "PACKAGE_NAME": { - "type": "text" + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } }, "TIMESTAMP": { "type": "date" diff --git a/pom.xml b/pom.xml index 31b7e5d..edaef9b 100644 --- a/pom.xml +++ b/pom.xml @@ -87,6 +87,12 @@ org.springframework.boot spring-boot-starter-data-elasticsearch + + + org.apache.httpcomponents + httpclient + + @@ -105,12 +111,11 @@ - org.modelmapper modelmapper - 2.3.5 + 3.1.1 @@ -139,17 +144,6 @@ 2.3.0 - - - org.springframework.boot - spring-boot-starter-json - - - com.fasterxml.jackson.dataformat - jackson-dataformat-xml - 2.17.0 - - org.springframework.boot diff --git a/src/main/java/fr/abes/logskbart/controller/LogsController.java b/src/main/java/fr/abes/logskbart/controller/LogsController.java index 8a353d5..4b86756 100644 --- a/src/main/java/fr/abes/logskbart/controller/LogsController.java +++ b/src/main/java/fr/abes/logskbart/controller/LogsController.java @@ -2,6 +2,8 @@ import fr.abes.logskbart.dto.LigneLogDto; import fr.abes.logskbart.dto.LogWebDto; +import fr.abes.logskbart.entity.LogKbart; +import fr.abes.logskbart.exception.EmptyFileException; import fr.abes.logskbart.service.LogsService; import fr.abes.logskbart.utils.UtilsMapper; import io.swagger.v3.oas.annotations.Operation; @@ -9,8 +11,16 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; +import org.springframework.core.io.InputStreamResource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; @@ -23,29 +33,37 @@ public class LogsController { private final LogsService service; - private final UtilsMapper mapper; - public LogsController(LogsService service, UtilsMapper mapper) { + public LogsController(LogsService service) { this.service = service; - this.mapper = mapper; } @Operation( summary = "Transfer kafka to PostgreSQL DB", description = "Retrieves kbart load logs from a Kafka bus and stores them in a DB for later availability", responses = { - @ApiResponse( responseCode = "200", description = "The request was successful.", content = { @Content(schema = @Schema(implementation = LogWebDto.class), mediaType = "application/json") } ), - @ApiResponse( responseCode = "400", description = "An element of the query is badly formulated.", content = { @Content(schema = @Schema()) } ), - @ApiResponse( responseCode = "500", description = "An internal server error interrupted processing.", content = { @Content(schema = @Schema()) } ), + @ApiResponse(responseCode = "200", description = "The request was successful.", content = {@Content(schema = @Schema(implementation = LogWebDto.class), mediaType = "application/json")}), + @ApiResponse(responseCode = "400", description = "An element of the query is badly formulated.", content = {@Content(schema = @Schema())}), + @ApiResponse(responseCode = "500", description = "An internal server error interrupted processing.", content = {@Content(schema = @Schema())}), } ) @GetMapping("/logs/{filename}/{date}") - public LogWebDto getLogsFromPackageAndDate(@PathVariable String filename, @PathVariable String date) throws ParseException { + public ResponseEntity getLogsFromPackageAndDate(@PathVariable String filename, @PathVariable String date) throws ParseException, IOException, EmptyFileException { SimpleDateFormat format = new SimpleDateFormat("ddMMyyyy"); Date dateAnalyse = format.parse(date); - LogWebDto logWebDto = new LogWebDto(filename, date); - List lignes = mapper.mapList(service.getLogKbartForPackage(filename, dateAnalyse), LigneLogDto.class); - logWebDto.addLignes(lignes); - return logWebDto; + File fichier = service.getLogKbartForPackage(filename, dateAnalyse); + FileInputStream fs = new FileInputStream(fichier); + // Définir les en-têtes pour la réponse HTTP + HttpHeaders headers = new HttpHeaders(); + headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + fichier.getName()); + headers.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM_VALUE); + + // Retourner la réponse avec le fichier à télécharger + return ResponseEntity.ok() + .headers(headers) + .contentLength(fichier.length()) + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .body(new InputStreamResource(fs)); + } } diff --git a/src/main/java/fr/abes/logskbart/entity/LogKbart.java b/src/main/java/fr/abes/logskbart/entity/LogKbart.java index a5ed3ff..5266e35 100644 --- a/src/main/java/fr/abes/logskbart/entity/LogKbart.java +++ b/src/main/java/fr/abes/logskbart/entity/LogKbart.java @@ -1,6 +1,5 @@ package fr.abes.logskbart.entity; -import fr.abes.logskbart.utils.Level; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.springframework.data.annotation.Id; @@ -8,7 +7,6 @@ import org.springframework.data.elasticsearch.annotations.Field; import java.io.Serializable; -import java.util.Comparator; import java.util.Date; import java.util.Objects; diff --git a/src/main/java/fr/abes/logskbart/exception/EmptyFileException.java b/src/main/java/fr/abes/logskbart/exception/EmptyFileException.java new file mode 100644 index 0000000..84c47db --- /dev/null +++ b/src/main/java/fr/abes/logskbart/exception/EmptyFileException.java @@ -0,0 +1,7 @@ +package fr.abes.logskbart.exception; + +public class EmptyFileException extends Throwable { + public EmptyFileException(String message) { + super(message); + } +} diff --git a/src/main/java/fr/abes/logskbart/exception/ExceptionControllerHandler.java b/src/main/java/fr/abes/logskbart/exception/ExceptionControllerHandler.java index 157be7d..b42312c 100644 --- a/src/main/java/fr/abes/logskbart/exception/ExceptionControllerHandler.java +++ b/src/main/java/fr/abes/logskbart/exception/ExceptionControllerHandler.java @@ -19,6 +19,10 @@ private ResponseEntity buildResponseEntity(ApiReturnError apiReturnError return new ResponseEntity<>(apiReturnError, apiReturnError.getStatus()); } + @ExceptionHandler(EmptyFileException.class) + protected ResponseEntity handleEmptyFileException(EmptyFileException e) { + return buildResponseEntity(new ApiReturnError(HttpStatus.NOT_FOUND, e.getMessage())); + } /** * Erreur dans la validité des paramètres de la requête * diff --git a/src/main/java/fr/abes/logskbart/repository/LogKbartRepository.java b/src/main/java/fr/abes/logskbart/repository/LogKbartRepository.java index 1979bab..72dde8f 100644 --- a/src/main/java/fr/abes/logskbart/repository/LogKbartRepository.java +++ b/src/main/java/fr/abes/logskbart/repository/LogKbartRepository.java @@ -1,15 +1,16 @@ package fr.abes.logskbart.repository; import fr.abes.logskbart.entity.LogKbart; +import org.springframework.data.domain.*; import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; import org.springframework.stereotype.Repository; import java.util.Date; -import java.util.List; +import java.util.stream.Stream; @Repository public interface LogKbartRepository extends ElasticsearchRepository { - List findAllByPackageNameAndTimestampBetweenOrderByNbLineAscTimestampAsc(String filename, Date debut, Date fin); - + long countByPackageNameAndTimestampBetween(String packageName, Date debut, Date fin); + Stream findAllByPackageNameAndTimestampBetweenOrderByNbLineAscTimestampAsc(String filename, Date debut, Date fin); } diff --git a/src/main/java/fr/abes/logskbart/service/LogsService.java b/src/main/java/fr/abes/logskbart/service/LogsService.java index e9b326c..ed7affa 100644 --- a/src/main/java/fr/abes/logskbart/service/LogsService.java +++ b/src/main/java/fr/abes/logskbart/service/LogsService.java @@ -1,30 +1,67 @@ package fr.abes.logskbart.service; import fr.abes.logskbart.entity.LogKbart; +import fr.abes.logskbart.exception.EmptyFileException; import fr.abes.logskbart.repository.LogKbartRepository; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.Calendar; import java.util.Date; import java.util.List; +import java.util.stream.Stream; @Service @Slf4j public class LogsService { + private static final Integer MAXSIZE = 5000; private final LogKbartRepository repository; - public LogsService(LogKbartRepository repository) { - this.repository = repository; + public LogsService(LogKbartRepository logKbartRepository) { + this.repository = logKbartRepository; } - public List getLogKbartForPackage(String packageName, Date date) { + @Transactional(readOnly = true) + public File getLogKbartForPackage(String packageName, Date date) throws IOException, EmptyFileException { Calendar dateChargement = Calendar.getInstance(); dateChargement.setTime(date); Calendar dateFin = (Calendar) dateChargement.clone(); dateFin.add(Calendar.DAY_OF_MONTH, 1); log.debug("packageName {}, Date début {}, Date fin {}", packageName, dateChargement.getTime(), dateFin.getTime()); - return repository.findAllByPackageNameAndTimestampBetweenOrderByNbLineAscTimestampAsc(packageName, dateChargement.getTime(), dateFin.getTime()); + if (repository.countByPackageNameAndTimestampBetween(packageName, dateChargement.getTime(), dateFin.getTime()) == 0) + throw new EmptyFileException("Aucun log pour le fichier " + packageName); + Path tempPath = Path.of("tempLogLocal"); + if (!Files.exists(tempPath)) { + Files.createDirectory(tempPath); + } + Path pathOfLocal = Path.of("tempLogLocal" + File.separator + packageName.replace(".tsv", ".log")); + Files.deleteIfExists(pathOfLocal); + try (Stream logKbarts = repository.findAllByPackageNameAndTimestampBetweenOrderByNbLineAscTimestampAsc(packageName, dateChargement.getTime(), dateFin.getTime())) { + logKbarts.forEach(logKbart -> { + String message = (logKbart.getNbLine() != -1) ? logKbart.getNbLine() + " : " : ""; + message += logKbart.getMessage() + System.lineSeparator(); + try { + if (Files.exists(pathOfLocal)) { + Files.write(pathOfLocal, message.getBytes(), StandardOpenOption.APPEND); + } else { + // Créer le fichier et inscrit la ligne dedans + Files.createFile(pathOfLocal); + // Inscrit les informations sur la ligne + Files.write(pathOfLocal, message.getBytes(), StandardOpenOption.APPEND); + log.debug("Fichier temporaire créé."); + } + }catch (IOException e) { + log.error("Impossible d'écrire dans le fichier local", e); + } + }); + } + return pathOfLocal.toFile(); } public void saveAll(List logKbarts) {