MET-6263 performance tests moved from batch project

client/pom.xml

@@ -22,15 +22,55 @@
       <version>1.18.30</version>
       <scope>provided</scope>
     </dependency>
+
+    <!--test dependencies -->
+
     <dependency>
-      <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-slf4j-impl</artifactId>
-      <version>2.23.1</version>
+      <groupId>org.postgresql</groupId>
+      <artifactId>postgresql</artifactId>
+      <version>42.7.2</version>
+      <scope>test</scope>
     </dependency>
+
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+      <version>2.18.2</version>
+    </dependency>
+
     <dependency>
       <groupId>org.springframework</groupId>
       <artifactId>spring-web</artifactId>
       <version>${version.spring-framework}</version>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter-test</artifactId>
+      <version>3.2.4</version>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter-data-jpa</artifactId>
+      <version>3.2.4</version>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+      <version>3.17.0</version>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>eu.europeana.metis-processing-engine</groupId>
+      <artifactId>metis-processing-engine-commons</artifactId>
+      <version>1.0-SNAPSHOT</version>
+<!--      <scope>test</scope>-->
     </dependency>
 
   </dependencies>

client/src/test/java/eu/europeana/processing/AbstractPerformanceTest.java

@@ -0,0 +1,89 @@
+package eu.europeana.processing;
+
+import com.zaxxer.hikari.HikariConfig;
+import eu.europeana.processing.config.OaiSourceConfigurationProperties;
+import eu.europeana.processing.config.TestConfigurationProperties;
+import eu.europeana.processing.config.TestsConfig;
+import eu.europeana.processing.repository.ExecutionRecordExceptionLogRepository;
+import eu.europeana.processing.repository.ExecutionRecordRepository;
+import jakarta.annotation.Resource;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import org.apache.commons.lang3.time.StopWatch;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.TestMethodOrder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.ContextConfiguration;
+
+@SpringBootTest
+@ContextConfiguration(classes = {TestsConfig.class})
+@EnableAutoConfiguration
+@TestMethodOrder(MethodOrderer.MethodName.class)
+public abstract class AbstractPerformanceTest {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  @Autowired
+  protected HikariConfig dbConfig;
+
+  @Resource
+  protected ExecutionRecordRepository executionRecordRepository;
+
+  @Resource
+  protected ExecutionRecordExceptionLogRepository executionRecordExceptionLogRepository;
+
+  private static boolean firstTest = true;
+
+  private static boolean cleared;
+
+  protected StopWatch startWatch;
+
+  @Autowired
+  private DbCleaner dbCleaner;
+
+  @Autowired
+  protected OaiSourceConfigurationProperties sourceProperties;
+
+  @Autowired
+  protected TestConfigurationProperties testProperties;
+
+  //Could not use original jupiter's @BeforeEach because we need to pass test number parameter
+  protected void beforeEach(int stepNumber) throws InterruptedException {
+    if (firstTest) {
+      firstTest = false;
+      dbCleaner.clearDbFor(stepNumber);
+      cleared = true;
+    } else {
+      if (cleared) {
+        waitBetweenTests();
+        LOGGER.info("There is no need to clear DB. It was cleared before first test.");
+      } else {
+        throw new RuntimeException("The DB could not be cleared before first test. Could not execute following tests.");
+      }
+      LOGGER.info("Executing test for workflow step no: {}", stepNumber);
+    }
+  }
+
+  private void waitBetweenTests() throws InterruptedException {
+    if (testProperties.getPauseBetweenTestsMs() > 0) {
+      LOGGER.info("Waiting {} milliseconds between test executions...", testProperties.getPauseBetweenTestsMs());
+      Thread.sleep(testProperties.getPauseBetweenTestsMs());
+    }
+  }
+
+  protected void validateResult(int stepNumber) throws IOException {
+    LOGGER.info("Step: {} - task execution time: {}", stepNumber, startWatch.formatTime());
+    String datasetId = testProperties.getDatasetId();
+    String taskId = String.valueOf(stepNumber);
+    int expectedRecordCount = stepNumber != 1 ? sourceProperties.getValidRecordCount() : sourceProperties.getRecordCount() ;
+    long expectedErrorCount = stepNumber != 2 ? 0: sourceProperties.getRecordCount() - sourceProperties.getValidRecordCount();
+    Assertions.assertThat(executionRecordRepository.countByDatasetIdAndExecutionId(datasetId, taskId)).isEqualTo(expectedRecordCount);
+    Assertions.assertThat(executionRecordExceptionLogRepository.countByDatasetIdAndExecutionId(datasetId, taskId)).isEqualTo(expectedErrorCount);
+  }
+
+}

client/src/test/java/eu/europeana/processing/DbCleaner.java

@@ -0,0 +1,114 @@
+package eu.europeana.processing;
+
+import static java.util.Collections.singleton;
+
+import eu.europeana.processing.config.DbCleaningMode;
+import eu.europeana.processing.config.TestConfigurationProperties;
+import jakarta.persistence.EntityManager;
+import jakarta.persistence.Query;
+import java.lang.invoke.MethodHandles;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.support.TransactionTemplate;
+
+@Component
+public class DbCleaner {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+  public static final String TEST_DATASET_QUERY_PARAMETER = "testDataset";
+
+  @Autowired
+  private EntityManager entityManager;
+
+  @Autowired
+  private TestConfigurationProperties config;
+
+  @Autowired
+  private TransactionTemplate transactionTemplate;
+
+  private static final String[] TABLES = {
+      "Execution_Record",
+      "Execution_Record_Exception_Log",
+      "Execution_Record_External_Identifier",
+      "Task_Info",
+  };
+
+  private static final String[] REQUESTS_CLEANING_ALL_DB_BUT_PREVIOUS_STEPS_OF_TEST_DATASET = {
+      "DELETE from \"batch-framework\".Execution_Record WHERE dataset_Id <> :testDataset OR execution_Id NOT IN :previousSteps",
+      "DELETE from \"batch-framework\".Execution_Record_Exception_Log WHERE dataset_Id <> :testDataset OR execution_Id NOT IN :previousSteps",
+      "DELETE from \"batch-framework\".Execution_Record_External_Identifier WHERE dataset_Id <> :testDataset OR execution_Id NOT IN :previousSteps",
+      "DELETE from \"batch-framework\".Task_Info WHERE CAST(task_id as VARCHAR) NOT IN :previousSteps",
+  };
+
+  private static final String[] REQUESTS_CLEANING_CURRENT_AND_FOLLOWING_STEPS_OF_TEST_DATASET = {
+      "DELETE from \"batch-framework\".Execution_Record WHERE dataset_Id = :testDataset AND execution_Id NOT IN :previousSteps",
+      "DELETE from \"batch-framework\".Execution_Record_Exception_Log WHERE dataset_Id = :testDataset AND execution_Id NOT IN :previousSteps",
+      "DELETE from \"batch-framework\".Execution_Record_External_Identifier WHERE dataset_Id = :testDataset AND execution_Id NOT IN :previousSteps",
+      "DELETE from \"batch-framework\".Task_Info WHERE CAST(task_id as VARCHAR) NOT IN :previousSteps",
+  };
+
+  public void clearDbFor(int stepNumber) {
+    if (config.getDbCleaning() == DbCleaningMode.NO_CLEANING) {
+      LOGGER.warn("Cleaning data from previous tests is turned off: {}", stepNumber);
+      return;
+    }
+
+    LOGGER.info("Clearing data for test for the workflow step number: {}", stepNumber);
+    for (int tableNumber = 0; tableNumber < TABLES.length; tableNumber++) {
+      clearTable(stepNumber, tableNumber);
+    }
+  }
+
+  private void clearTable(int stepNumber, int tableNumber) {
+    Integer result = transactionTemplate.execute(status -> {
+      Query query;
+      DbCleaningMode cleaningMode = config.getDbCleaning();
+      if (cleaningMode == DbCleaningMode.CLEAN_ALL_DB_BUT_PREVIOUS_STEPS_OF_TEST_DATASET) {
+        if (stepNumber == 1) {
+          query = createTruncateWholeTableQuery(tableNumber);
+        } else {
+          query = createDeleteSelectedDataRequest(stepNumber, tableNumber,
+              REQUESTS_CLEANING_ALL_DB_BUT_PREVIOUS_STEPS_OF_TEST_DATASET);
+        }
+      } else if (cleaningMode == DbCleaningMode.CLEAN_CURRENT_AND_FOLLOWING_STEPS_OF_TEST_DATASET) {
+        query = createDeleteSelectedDataRequest(stepNumber, tableNumber,
+            REQUESTS_CLEANING_CURRENT_AND_FOLLOWING_STEPS_OF_TEST_DATASET);
+      } else {
+        throw new UnsupportedOperationException("Method clearTable does not support cleaning mode: " + cleaningMode);
+      }
+      return query.executeUpdate();
+    });
+    LOGGER.info("Cleared data in the table: {}, removed {} rows.", TABLES[tableNumber], result);
+  }
+
+  private Query createDeleteSelectedDataRequest(int stepNumber, int tableNumber, String[] queriesSet) {
+
+    Set<String> previousStepsSet = IntStream.range(1, stepNumber).mapToObj(String::valueOf)
+                                            .collect(Collectors.toUnmodifiableSet());
+    //It is needed cause the db and driver could not work with NOT IN with empty set. So we put an artificial value that not exists in DB.
+    if (previousStepsSet.size() == 0) {
+      previousStepsSet = singleton(UUID.randomUUID().toString());
+    }
+    String sqlString = queriesSet[tableNumber];
+    Query query = entityManager
+        .createNativeQuery(sqlString)
+        .setParameter("previousSteps", previousStepsSet);
+    if (sqlString.contains(":" + TEST_DATASET_QUERY_PARAMETER)) {
+      query.setParameter(TEST_DATASET_QUERY_PARAMETER, config.getDatasetId());
+    }
+
+    return query;
+  }
+
+  private Query createTruncateWholeTableQuery(int tableNumber) {
+    Query query;
+    query = entityManager.createNativeQuery("TRUNCATE TABLE \"batch-framework\"." + TABLES[tableNumber]);
+    return query;
+  }
+}