Skip to content

Commit

Permalink
MET-6263 Corrected unit tests.
Browse files Browse the repository at this point in the history
  • Loading branch information
marcin-rp committed Dec 18, 2024
1 parent 87f6a5a commit 88a19f0
Show file tree
Hide file tree
Showing 15 changed files with 193 additions and 30 deletions.
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -250,4 +250,5 @@ $RECYCLE.BIN/
# Truststores
*.jks

.vscode/settings.json
.vscode/settings.json
/client/src/test/resources/application.properties
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@
import eu.europeana.processing.config.OaiSourceConfigurationProperties;
import eu.europeana.processing.config.TestConfigurationProperties;
import eu.europeana.processing.config.TestsConfig;
import eu.europeana.processing.repository.ExecutionRecordExceptionLogRepository;
import eu.europeana.processing.repository.ExecutionRecordRepository;
import eu.europeana.processing.config.db.entity.ExecutionRecord;
import eu.europeana.processing.config.db.entity.ExecutionRecordExceptionLog;
import eu.europeana.processing.config.db.repositories.ExecutionRecordExceptionLogRepository;
import eu.europeana.processing.config.db.repositories.ExecutionRecordRepository;
import jakarta.annotation.Resource;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
Expand All @@ -31,11 +33,12 @@ public abstract class AbstractPerformanceTest {
@Autowired
protected HikariConfig dbConfig;


@Resource
protected ExecutionRecordRepository executionRecordRepository;
protected ExecutionRecordRepository<ExecutionRecord> executionRecordRepository;

@Resource
protected ExecutionRecordExceptionLogRepository executionRecordExceptionLogRepository;
protected ExecutionRecordExceptionLogRepository<ExecutionRecordExceptionLog> executionRecordExceptionLogRepository;

private static boolean firstTest = true;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,14 +45,15 @@
import eu.europeana.cloud.flink.client.JobExecutor;
import eu.europeana.cloud.flink.client.entities.SubmitJobRequest;
import eu.europeana.processing.config.FlinkConfigurationProperties;
import eu.europeana.processing.config.JarIdsProperties;
import eu.europeana.processing.config.JobsConfigurationProperties;
import eu.europeana.processing.job.JobParamValue;
import java.lang.invoke.MethodHandles;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.lang3.time.StopWatch;
import org.junit.jupiter.api.BeforeAll;

import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -63,56 +64,52 @@ public class FlinkPerformanceTest extends AbstractPerformanceTest {

private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

private static JobExecutor executor;

@Autowired
private FlinkConfigurationProperties flinkConfigurationProperties;

@Autowired
protected JobsConfigurationProperties jobsConfigurationProperties;

@BeforeAll
public static void createExecutor(@Autowired FlinkConfigurationProperties properties) {
executor = new JobExecutor(properties.getJobManagerUrl(), properties.getJobManagerUser(), properties.getJobManagerPassword(),
properties.getJarId());
}
@Autowired
protected JarIdsProperties jarIdsProperties;

@Test
void step1_shouldExecuteOAIHarvestComplietellyWithoutErrors() throws Exception {

executeStep(1, "eu.europeana.processing.oai.OAIJob",
executeStep(1, jarIdsProperties.getOai(), "eu.europeana.processing.oai.OAIJob",
Map.of(OAI_REPOSITORY_URL, sourceProperties.getUrl(), SET_SPEC, sourceProperties.getSetSpec(), METADATA_PREFIX,
sourceProperties.getMetadataPrefix()));
}

@Test
void step2_shouldExecuteExternalValidationWithoutErrors() throws Exception {
executeStep(2, "eu.europeana.processing.validation.ValidationJob",
executeStep(2, jarIdsProperties.getValidation(), "eu.europeana.processing.validation.ValidationJob",
Map.of(VALIDATION_TYPE, JobParamValue.VALIDATION_EXTERNAL));
}


@Test
void step3_shouldExecuteXsltTransformationWithoutErrors() throws Exception {
executeStep(3, "eu.europeana.processing.transformation.TransformationJob",
executeStep(3, jarIdsProperties.getTransformation(), "eu.europeana.processing.transformation.TransformationJob",
Map.of(METIS_DATASET_NAME, "idA_metisDatasetNameA", METIS_DATASET_COUNTRY, "Greece", METIS_DATASET_LANGUAGE, "el",
METIS_XSLT_URL, "https://metis-core-rest.test.eanadev.org/datasets/xslt/6204e5e2514e773e6745f7e9"));
}

@Test
void step4_shouldExecuteIternalValidationWithoutErrors() throws Exception {
executeStep(4, "eu.europeana.processing.validation.ValidationJob",
executeStep(4, jarIdsProperties.getValidation(), "eu.europeana.processing.validation.ValidationJob",
Map.of(VALIDATION_TYPE, JobParamValue.VALIDATION_INTERNAL));
}

@Test
void step5_shouldExecuteNormalizationWithoutErrors() throws Exception {
executeStep(5, "eu.europeana.processing.normalization.NormalizationJob", Collections.emptyMap());
executeStep(5, jarIdsProperties.getNormalization(), "eu.europeana.processing.normalization.NormalizationJob",
Collections.emptyMap());
}

@Test
void step6_shouldExecuteEnrichmentWithoutErrors() throws Exception {
executeStep(6, "eu.europeana.processing.enrichment.EnrichmentJob",
executeStep(6, jarIdsProperties.getEnrichment(), "eu.europeana.processing.enrichment.EnrichmentJob",
Map.of(DEREFERENCE_SERVICE_URL, jobsConfigurationProperties.getEnrichment().getDereferenceUrl(),
ENRICHMENT_ENTITY_MANAGEMENT_URL, jobsConfigurationProperties.getEnrichment().getEntityManagementUrl(),
ENRICHMENT_ENTITY_API_URL, jobsConfigurationProperties.getEnrichment().getEntityApiUrl(), ENRICHMENT_ENTITY_API_KEY,
Expand All @@ -122,7 +119,7 @@ void step6_shouldExecuteEnrichmentWithoutErrors() throws Exception {

@Test
void step7_shouldExecuteMediaWithoutErrors() throws Exception {
executeStep(7, "eu.europeana.processing.media.MediaJob", Collections.emptyMap());
executeStep(7, jarIdsProperties.getMedia(), "eu.europeana.processing.media.MediaJob", Collections.emptyMap());
}

@Test
Expand All @@ -149,11 +146,15 @@ void step8_shouldExecuteIndexingWithoutErrors() throws Exception {
specialParameters.put(INDEXING_MONGOAPPLICATIONNAME,
jobsConfigurationProperties.getIndexing().getZookeeperDefaultCollection());

executeStep(8, "eu.europeana.cloud.job.indexing.IndexingJobWithPostgresMultiThreadedOperation", specialParameters);
executeStep(8, jarIdsProperties.getIndexing(),
"eu.europeana.cloud.job.indexing.IndexingJobWithPostgresMultiThreadedOperation", specialParameters);
}

void executeStep(int stepNumber, String jobClass, Map<String, String> specialParameters)
void executeStep(int stepNumber, String jarId, String jobClass, Map<String, String> specialParameters)
throws Exception {
JobExecutor executor = new JobExecutor(flinkConfigurationProperties.getJobManagerUrl(),
flinkConfigurationProperties.getJobManagerUser(),
flinkConfigurationProperties.getJobManagerPassword(), jarId);
beforeEach(stepNumber);
String datasetId = testProperties.getDatasetId();
String taskId = String.valueOf(stepNumber);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ public class FlinkConfigurationProperties {
String jobManagerUrl;
String jobManagerUser;
String jobManagerPassword;
String jarId;
int readerParallelism;
int operatorParallelism;
int sinkParallelism;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package eu.europeana.processing.config;

import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;

@Setter
@Getter
@ConfigurationProperties(prefix = "flink.jar.id")
public class JarIdsProperties {
private String oai;
private String validation;
private String transformation;
private String normalization;
private String enrichment;
private String media;
private String indexing;
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@
OaiSourceConfigurationProperties.class,
FlinkConfigurationProperties.class,
TestConfigurationProperties.class,
JobsConfigurationProperties.class})
JobsConfigurationProperties.class,
JarIdsProperties.class})
public class TestsConfig {

@Bean
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package eu.europeana.processing.config.db.entity;

import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.Index;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;

@Getter
@Setter
@Entity
@Table(schema = "batch-framework", indexes = {@Index(name = "execution_record_dataset_id_execution_id_idx", columnList = "datasetId, executionId")})
public class ExecutionRecord extends ExecutionRecordIdentifier{

@Column(length = 50)
private String executionName;
@Column(columnDefinition = "TEXT")
private String recordData;
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package eu.europeana.processing.config.db.entity;

import lombok.Getter;
import lombok.Setter;

@Getter
@Setter
public class ExecutionRecordDTO {

private String datasetId;
private String executionId;
private String recordId;
private String executionName;
private String recordData;
private String exception;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package eu.europeana.processing.config.db.entity;

import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.Index;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;

@Getter
@Setter
@Entity
@Table(schema = "batch-framework", indexes = {@Index(name = "execution_record_exception_log_dataset_id_execution_id_idx", columnList = "datasetId, executionId")})
public class ExecutionRecordExceptionLog extends ExecutionRecordIdentifier{

@Column(length = 50)
private String executionName;
@Column(columnDefinition = "TEXT")
private String exception;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package eu.europeana.processing.config.db.entity;

import jakarta.persistence.Entity;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;

@Getter
@Setter
@Entity
@Table(schema = "batch-framework")
public class ExecutionRecordExternalIdentifier extends ExecutionRecordIdentifier{

private boolean isDeleted;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package eu.europeana.processing.config.db.entity;

import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.Id;
import jakarta.persistence.IdClass;
import jakarta.persistence.Inheritance;
import jakarta.persistence.InheritanceType;
import lombok.Getter;
import lombok.Setter;

@Getter
@Setter
@Entity
@IdClass(ExecutionRecordIdentifier.class)
@Inheritance(strategy = InheritanceType.TABLE_PER_CLASS)
public class ExecutionRecordIdentifier {
@Id
@Column(length = 20)
private String datasetId;
@Id
@Column(length = 50)
private String executionId;
@Id
@Column(length = 300)
private String recordId;

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package eu.europeana.processing.config.db.repositories;

import eu.europeana.processing.config.db.entity.ExecutionRecordExceptionLog;
import eu.europeana.processing.config.db.entity.ExecutionRecordIdentifier;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;

@Repository
public interface ExecutionRecordExceptionLogRepository<T extends ExecutionRecordIdentifier> extends JpaRepository<ExecutionRecordExceptionLog, T> {
long countByDatasetIdAndExecutionId(String datasetId, String executionId);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package eu.europeana.processing.config.db.repositories;

import eu.europeana.processing.config.db.entity.ExecutionRecordExternalIdentifier;
import eu.europeana.processing.config.db.entity.ExecutionRecordIdentifier;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;

public interface ExecutionRecordExternalIdentifierRepository<T extends ExecutionRecordIdentifier> extends JpaRepository<ExecutionRecordExternalIdentifier, T> {

Page<ExecutionRecordExternalIdentifier> findAllByExecutionId(String executionId, Pageable pageable);
long countByDatasetIdAndExecutionId(String datasetId, String executionId);

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package eu.europeana.processing.config.db.repositories;

import eu.europeana.processing.config.db.entity.ExecutionRecord;
import eu.europeana.processing.config.db.entity.ExecutionRecordIdentifier;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;

@Repository
public interface ExecutionRecordRepository<T extends ExecutionRecordIdentifier> extends JpaRepository<ExecutionRecord, T> {
Page<ExecutionRecord> findByDatasetIdAndExecutionId(String datasetId, String executionId, Pageable pageable);
long countByDatasetIdAndExecutionId(String datasetId, String executionId);
}
13 changes: 7 additions & 6 deletions client/src/test/resources/application.properties.template
Original file line number Diff line number Diff line change
Expand Up @@ -48,12 +48,13 @@ test.dataset-id=12

#Flink related
flink.jobManagerUrl=
#flink.jarId=448348c0-e256-4a95-8f58-3c0f8d52dcd3_metis-processing-engine-oai-1.0-SNAPSHOT.jar
#flink.jarId=737df145-0b52-4e0d-bdcd-cfee85ed7da3_metis-processing-engine-validation-1.0-SNAPSHOT.jar
#flink.jarId=bb6fdf15-6650-4786-8b91-ff69535d5d8d_metis-processing-engine-transformation-1.0-SNAPSHOT.jar
#flink.jarId=29be3bff-04ff-45a8-8ad1-f44869255cdb_metis-processing-engine-normalization-1.0-SNAPSHOT.jar
#flink.jarId=2d4a2e0b-f7c5-4c30-8ae1-70455d68cbfa_metis-processing-engine-enrichment-1.0-SNAPSHOT.jar
flink.jarId=8717a197-78c7-46b5-84ef-9c2f211dade2_metis-processing-engine-media-1.0-SNAPSHOT.jar
flink.jar.id.oai=3c2af5ad-3005-4f5c-bb38-8eea9482171c_metis-processing-engine-oai-1.0-SNAPSHOT.jar
flink.jar.id.validation=58b42250-5795-4835-8b6a-651d0fed5ca8_metis-processing-engine-validation-1.0-SNAPSHOT.jar
flink.jar.id.transformation=5c88e975-cd98-4996-8e24-c58a6b1394b4_metis-processing-engine-transformation-1.0-SNAPSHOT.jar
flink.jar.id.normalization=063ee029-b1eb-4523-93f6-c2b4a355e3e4_metis-processing-engine-normalization-1.0-SNAPSHOT.jar
flink.jar.id.enrichment=4f6de8e7-dc78-4945-acd0-50b040ed1912_metis-processing-engine-enrichment-1.0-SNAPSHOT.jar
flink.jar.id.media=407bbb55-35c0-4c39-b17b-fb244126f1e2_metis-processing-engine-media-1.0-SNAPSHOT.jar
flink.jar.id.indexing=20cc3068-9caf-4939-871a-fd9722ca0f9c_metis-processing-engine-indexing-1.0-SNAPSHOT.jar

flink.jobManagerUser=
flink.jobManagerPassword=
Expand Down

0 comments on commit 88a19f0

Please sign in to comment.