Feat/met 5960 debias algorithm for detection (#688)

* MET-5960 initial debias client * MET-5960 add exception class * MET-5960 add unit tests to the controller * MET-5960 metis debias client implementation update * MET-5960 update unit tests * MET-5960 sonarcloud improvements * MET-5960 remove unused exceptions * MET-5960 update unused exceptions * MET-5960 code reorganization to be able to use with sandbox * MET-5960 add error handling and update service port * MET-5960 code review comments
europeana · Oct 17, 2024 · c214957 · c214957
1 parent cc62d18
commit c214957
Show file tree

Hide file tree

Showing 44 changed files with 2,158 additions and 0 deletions.
diff --git a/.run/metis-debias.run.xml b/.run/metis-debias.run.xml
@@ -0,0 +1,16 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="metis-debias" type="SpringBootApplicationConfigurationType" factoryName="Spring Boot" folderName="spring-run-configuration">
+    <envs>
+      <env name="logging.config" value="file:///data/metis-configuration/k8s/common-components/log4j2-xml/log4j2.xml" />
+      <env name="server.servlet.context-path" value="/metis-debias" />
+      <env name="spring.config.location" value="file:///data/metis-configuration/metis-debias/metis-debias-rest/k8s/overlays/local/components/properties/application.properties" />
+      <env name="server.port" value="8081" />
+      <env name="truststore.path" value="/data/metis-configuration/k8s/common-components/custom-truststore-jks/custom-truststore.jks" />
+    </envs>
+    <module name="metis-debias-detect-rest" />
+    <option name="SPRING_BOOT_MAIN_CLASS" value="eu.europeana.metis.debias.detect.rest.Application" />
+    <method v="2">
+      <option name="Make" enabled="true" />
+    </method>
+  </configuration>
+</component>
diff --git a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/RestEndpoints.java b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/RestEndpoints.java
@@ -88,6 +88,9 @@ public final class RestEndpoints {
   public static final String REPOSITORY_HTTP_ENDPOINT_ZIP = "/repository/zip/{dataset}.zip";
   public static final String REPOSITORY_OAI_ENDPOINT = "/repository/oai";
 
+  /* METIS-DEBIAS ENDPOINTS */
+  public static final String DEBIAS_DETECTION = "/debias";
+
   private RestEndpoints() {
   }
 

diff --git a/metis-debias/README.MD b/metis-debias/README.MD
@@ -0,0 +1,12 @@
+# Getting Started
+
+### Reference Documentation
+
+This module includes DeBias detection algorithm functionality for Metis services.
+The module includes the following submodels:
+
+| Module                   | Functionality                             |
+|--------------------------|-------------------------------------------|
+| metis-debias-detect-rest | REST client for debias detection REST API |
+
+
diff --git a/metis-debias/metis-debias-detect-rest/.gitignore b/metis-debias/metis-debias-detect-rest/.gitignore
@@ -0,0 +1,15 @@
+
+/src/main/resources/application.properties
+
+target/
+!.mvn/wrapper/maven-wrapper.jar
+!**/src/main/**/target/
+!**/src/test/**/target/
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr
+
+
diff --git a/metis-debias/metis-debias-detect-rest/pom.xml b/metis-debias/metis-debias-detect-rest/pom.xml
@@ -0,0 +1,118 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <artifactId>metis-debias</artifactId>
+    <groupId>eu.europeana.metis</groupId>
+    <version>13-SNAPSHOT</version>
+  </parent>
+  <artifactId>metis-debias-detect-rest</artifactId>
+  <dependencies>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter-web</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>org.springframework.boot</groupId>
+          <artifactId>spring-boot-starter-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>eu.europeana.metis</groupId>
+      <artifactId>metis-common-utils</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>eu.europeana.metis</groupId>
+      <artifactId>metis-common-spring-properties</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter-actuator</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter-log4j2</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-configuration-processor</artifactId>
+      <optional>true</optional>
+    </dependency>
+    <dependency>
+      <groupId>co.elastic.apm</groupId>
+      <artifactId>apm-agent-attach</artifactId>
+      <version>${version.elastic.apm}</version>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.springdoc</groupId>
+      <artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
+      <version>${version.springdoc-openapi-starter-webmvc-ui}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter-test</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework</groupId>
+      <artifactId>spring-test</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>eu.europeana.metis</groupId>
+      <artifactId>metis-debias-detect-service</artifactId>
+      <version>13-SNAPSHOT</version>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.wiremock</groupId>
+      <artifactId>wiremock-standalone</artifactId>
+      <version>${version.org.wiremock}</version>
+      <scope>test</scope>
+    </dependency>
+
+  </dependencies>
+
+  <dependencyManagement>
+    <dependencies>
+      <dependency>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-dependencies</artifactId>
+        <version>${version.spring.boot}</version>
+        <type>pom</type>
+        <scope>import</scope>
+      </dependency>
+    </dependencies>
+  </dependencyManagement>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-maven-plugin</artifactId>
+        <version>${version.spring.boot}</version>
+        <executions>
+          <execution>
+            <goals>
+              <goal>repackage</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>
diff --git a/...s-debias-detect-rest/src/main/java/eu/europeana/metis/debias/detect/rest/Application.java b/...s-debias-detect-rest/src/main/java/eu/europeana/metis/debias/detect/rest/Application.java
@@ -0,0 +1,21 @@
+package eu.europeana.metis.debias.detect.rest;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+/**
+ * The type Application.
+ */
+@SpringBootApplication
+public class Application {
+
+  /**
+   * The entry point of application.
+   *
+   * @param args the input arguments
+   */
+  public static void main(String[] args) {
+    SpringApplication.run(Application.class, args);
+  }
+
+}
diff --git a/.../src/main/java/eu/europeana/metis/debias/detect/rest/config/ApplicationConfiguration.java b/.../src/main/java/eu/europeana/metis/debias/detect/rest/config/ApplicationConfiguration.java
@@ -0,0 +1,78 @@
+package eu.europeana.metis.debias.detect.rest.config;
+
+import eu.europeana.metis.debias.detect.client.DeBiasClient;
+import eu.europeana.metis.debias.detect.service.BiasDetectService;
+import eu.europeana.metis.utils.CustomTruststoreAppender;
+import eu.europeana.metis.utils.CustomTruststoreAppender.TrustStoreConfigurationException;
+import eu.europeana.metis.utils.apm.ElasticAPMConfiguration;
+import java.lang.invoke.MethodHandles;
+import metis.common.config.properties.TruststoreConfigurationProperties;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * The type Application configuration.
+ */
+@Configuration
+@EnableConfigurationProperties({ElasticAPMConfiguration.class, TruststoreConfigurationProperties.class})
+@ComponentScan(basePackages = {
+    "eu.europeana.metis.debias.detect.rest"})
+public class ApplicationConfiguration {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  @Value("${debias.simple.client.detect-url}")
+  private String detectUrl;
+
+  @Value("${debias.simple.client.connect-timeout}")
+  private int connectTimeOut;
+
+  @Value("${debias.simple.client.request-timeout}")
+  private int requestTimeOut;
+
+  /**
+   * Instantiates a new Application configuration.
+   *
+   * @param truststoreConfigurationProperties the truststore configuration properties
+   * @throws TrustStoreConfigurationException the trust store configuration exception
+   */
+  @Autowired
+  public ApplicationConfiguration(TruststoreConfigurationProperties truststoreConfigurationProperties)
+      throws TrustStoreConfigurationException {
+    ApplicationConfiguration.initializeTruststore(truststoreConfigurationProperties);
+  }
+
+  /**
+   * Detect service detect service.
+   *
+   * @return the detect service
+   */
+  @Bean
+  public BiasDetectService detectService() {
+    return new DeBiasClient(this.detectUrl, this.connectTimeOut, this.requestTimeOut);
+  }
+
+  /**
+   * Initialize truststore.
+   *
+   * @param truststoreConfigurationProperties the truststore configuration properties
+   * @throws TrustStoreConfigurationException the trust store configuration exception
+   */
+  static void initializeTruststore(TruststoreConfigurationProperties truststoreConfigurationProperties)
+      throws TrustStoreConfigurationException {
+    if (StringUtils.isNotEmpty(truststoreConfigurationProperties.getPath()) && StringUtils
+        .isNotEmpty(truststoreConfigurationProperties.getPassword())) {
+      CustomTruststoreAppender
+          .appendCustomTruststoreToDefault(truststoreConfigurationProperties.getPath(),
+              truststoreConfigurationProperties.getPassword());
+      LOGGER.info("Custom truststore appended to default truststore");
+    }
+  }
+}
diff --git a/...detect-rest/src/main/java/eu/europeana/metis/debias/detect/rest/config/SwaggerConfig.java b/...detect-rest/src/main/java/eu/europeana/metis/debias/detect/rest/config/SwaggerConfig.java
@@ -0,0 +1,31 @@
+package eu.europeana.metis.debias.detect.rest.config;
+
+import io.swagger.v3.oas.models.OpenAPI;
+import io.swagger.v3.oas.models.info.Info;
+import io.swagger.v3.oas.models.info.License;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * Config for Swagger documentation
+ */
+@Configuration
+public class SwaggerConfig {
+
+  /**
+   * The open api documentation docket.
+   *
+   * @return the docket configuration
+   */
+  @Bean
+  public OpenAPI openAPI() {
+    return new OpenAPI()
+        .info(new Info()
+            .title("DeBias REST API")
+            .description("DeBias REST API for Europeana")
+            .version("v1")
+            .license(new License()
+                .name("EUPL License v1.2")
+                .url("https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12")));
+  }
+}
diff --git a/...t/src/main/java/eu/europeana/metis/debias/detect/rest/controller/DetectionController.java b/...t/src/main/java/eu/europeana/metis/debias/detect/rest/controller/DetectionController.java
@@ -0,0 +1,49 @@
+package eu.europeana.metis.debias.detect.rest.controller;
+
+import eu.europeana.metis.debias.detect.model.DeBiasResult;
+import eu.europeana.metis.debias.detect.model.request.BiasInputLiterals;
+import eu.europeana.metis.debias.detect.model.response.DetectionDeBiasResult;
+import eu.europeana.metis.debias.detect.service.BiasDetectService;
+import eu.europeana.metis.utils.RestEndpoints;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RestController;
+
+/**
+ * The type Detection controller.
+ */
+@RestController
+public class DetectionController {
+
+  /**
+   * The Detect service.
+   */
+  BiasDetectService biasDetectService;
+
+  /**
+   * Instantiates a new Detection controller.
+   *
+   * @param biasDetectService the detect service
+   */
+  @Autowired
+  public DetectionController(BiasDetectService biasDetectService) {
+    this.biasDetectService = biasDetectService;
+  }
+
+  /**
+   * DeBias detection result.
+   *
+   * @param biasInputLiterals {@link BiasInputLiterals} the detection parameter
+   * @return {@link DetectionDeBiasResult} response of result
+   */
+  @PostMapping(value = RestEndpoints.DEBIAS_DETECTION, consumes = MediaType.APPLICATION_JSON_VALUE, produces = {
+      MediaType.APPLICATION_JSON_VALUE})
+  @Operation(description = "DeBias a list of values", responses = {@ApiResponse(responseCode = "200"),@ApiResponse(responseCode = "422")})
+  public DeBiasResult debias(@RequestBody BiasInputLiterals biasInputLiterals) {
+    return biasDetectService.detect(biasInputLiterals);
+  }
+}