Skip to content

Commit

Permalink
MET-6199 add coverage unit tests part 3
Browse files Browse the repository at this point in the history
  • Loading branch information
jeortizquan committed Oct 28, 2024
1 parent 062d88f commit 3d62d4e
Show file tree
Hide file tree
Showing 5 changed files with 211 additions and 44 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@ public class DeBiasStatusDto {
@JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss")
private final ZonedDateTime creationDate;

@JsonProperty("total")
@JsonProperty("total-records")
private final Integer total;

@JsonProperty("processed")
@JsonProperty("processed-records")
private final Integer processed;

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,18 +110,22 @@ public void process(List<Record> recordList) {
private void updateProgress(List<Record> recordList) {
recordList.stream()
.collect(groupingBy(Record::getDatasetId))
.forEach( (datasetId, records) -> {
records.forEach(recordToProcess ->
recordLogRepository.updateByRecordIdAndStepAndStatus(recordToProcess.getRecordId(),Step.DEBIAS, Status.SUCCESS));
updateDeBiasProgressCounters(datasetId);
}
);
.forEach((datasetId, records) -> {
LOGGER.info("Updating progress for datasetId: {}", datasetId);
records.forEach(recordToProcess ->
recordLogRepository.updateByRecordIdAndStepAndStatus(recordToProcess.getRecordId(), Step.DEBIAS, Status.SUCCESS));
updateDeBiasProgressCounters(datasetId);
}
);
}

private void updateDeBiasProgressCounters(String datasetId) {
Set<RecordLogEntity> recordLogDeBiasList = recordLogRepository.findRecordLogByDatasetIdAndStepAndStatus(datasetId, Step.DEBIAS, Status.SUCCESS);
Set<RecordLogEntity> recordLogNormalizeList = recordLogRepository.findRecordLogByDatasetIdAndStep(datasetId, Step.NORMALIZE);
LOGGER.info("DeBias PROGRESS datasetId: {}/{}", recordLogDeBiasList.size() ,recordLogNormalizeList.size());
Set<RecordLogEntity> recordLogDeBiasList = recordLogRepository
.findRecordLogByDatasetIdAndStepAndStatus(datasetId, Step.DEBIAS, Status.SUCCESS);
Set<RecordLogEntity> recordLogNormalizeList = recordLogRepository
.findRecordLogByDatasetIdAndStep(datasetId, Step.NORMALIZE);
LOGGER.info("DeBias PROGRESS datasetId: {}/{}",
recordLogDeBiasList.size(), recordLogNormalizeList.size());
if (recordLogDeBiasList.size() == recordLogNormalizeList.size()) {
datasetDeBiasRepository.updateState(Integer.parseInt(datasetId), "COMPLETED");
LOGGER.info("DeBias COMPLETED datasetId: {}", datasetId);
Expand All @@ -137,8 +141,8 @@ private void updateDeBiasProgressCounters(String datasetId) {
* @param deBiasReport the DeBias report
*/
private void doDeBiasAndGenerateReport(List<Record> recordList, List<DeBiasReportRow> deBiasReport) {
List<DeBiasInputRecord> info = getDeBiasSourceFieldsFromRecords(recordList);
info
List<DeBiasInputRecord> deBiasSourceFieldsFromRecords = getDeBiasSourceFieldsFromRecords(recordList);
deBiasSourceFieldsFromRecords
.stream()
.collect(groupingBy(DeBiasInputRecord::language))
.forEach(((deBiasSupportedLanguage, recordDescriptions) ->
Expand Down Expand Up @@ -235,7 +239,7 @@ private List<DeBiasInputRecord> getDeBiasSourceFieldsFromRecords(List<Record> re

} catch (SerializationException e) {
deBiasInputRecords = Collections.emptyList();
LOGGER.error("Serialization {}",e.getMessage(), e);
LOGGER.error("Serialization {}", e.getMessage(), e);
}
return deBiasInputRecords;
}).flatMap(Collection::stream).toList();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,33 @@ private <T, U> List<DeBiasInputRecord> getLiteralsAndLanguagesFromRdf(
.toList();
}

/**
* Gets contextual class labels by rdf about.
*
* @return the contextual class labels by rdf about
*/
private Map<String, List<PrefLabel>> getContextualClassLabelsByRdfAbout() {
final Map<String, List<PrefLabel>> result = new HashMap<>();
Optional.ofNullable(rdf.getAgentList())
.stream().flatMap(Collection::stream)
.forEach(agent -> result.put(agent.getAbout(), agent.getPrefLabelList()));
Optional.ofNullable(rdf.getConceptList()).stream().flatMap(Collection::stream).forEach(
concept -> result.put(concept.getAbout(),
Optional.ofNullable(concept.getChoiceList())
.stream().flatMap(Collection::stream).filter(Concept.Choice::ifPrefLabel)
.map(Concept.Choice::getPrefLabel).filter(Objects::nonNull).toList()));
Optional.ofNullable(rdf.getOrganizationList())
.stream().flatMap(Collection::stream)
.forEach(organization -> result.put(organization.getAbout(), organization.getPrefLabelList()));
Optional.ofNullable(rdf.getPlaceList())
.stream().flatMap(Collection::stream)
.forEach(place -> result.put(place.getAbout(), place.getPrefLabelList()));
Optional.ofNullable(rdf.getTimeSpanList())
.stream().flatMap(Collection::stream)
.forEach(timespan -> result.put(timespan.getAbout(), timespan.getPrefLabelList()));
return result;
}

/**
* Gets descriptions and language from rdf.
*
Expand Down Expand Up @@ -241,33 +268,6 @@ List<DeBiasInputRecord> getReferencesAndLanguageFromRdf(
sourceField);
}

/**
* Gets contextual class labels by rdf about.
*
* @return the contextual class labels by rdf about
*/
private Map<String, List<PrefLabel>> getContextualClassLabelsByRdfAbout() {
final Map<String, List<PrefLabel>> result = new HashMap<>();
Optional.ofNullable(rdf.getAgentList())
.stream().flatMap(Collection::stream)
.forEach(agent -> result.put(agent.getAbout(), agent.getPrefLabelList()));
Optional.ofNullable(rdf.getConceptList()).stream().flatMap(Collection::stream).forEach(
concept -> result.put(concept.getAbout(),
Optional.ofNullable(concept.getChoiceList())
.stream().flatMap(Collection::stream).filter(Concept.Choice::ifPrefLabel)
.map(Concept.Choice::getPrefLabel).filter(Objects::nonNull).toList()));
Optional.ofNullable(rdf.getOrganizationList())
.stream().flatMap(Collection::stream)
.forEach(organization -> result.put(organization.getAbout(), organization.getPrefLabelList()));
Optional.ofNullable(rdf.getPlaceList())
.stream().flatMap(Collection::stream)
.forEach(place -> result.put(place.getAbout(), place.getPrefLabelList()));
Optional.ofNullable(rdf.getTimeSpanList())
.stream().flatMap(Collection::stream)
.forEach(timespan -> result.put(timespan.getAbout(), timespan.getPrefLabelList()));
return result;
}

/**
* Partition list stream.
*
Expand All @@ -279,7 +279,7 @@ private Map<String, List<PrefLabel>> getContextualClassLabelsByRdfAbout() {
static <T> Stream<List<T>> partitionList(List<T> sourceList, int partitionSize) {
if (partitionSize <= 0) {
throw new IllegalArgumentException(
String.format("The partition size cannot be smaller than 0. Actual value: %s", partitionSize));
String.format("The partition size cannot be smaller than or equal 0. Actual value: %s", partitionSize));
}
if (sourceList.isEmpty()) {
return Stream.empty();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -856,7 +856,9 @@ void getDebiasStatus_expectSuccess(String status) throws Exception {
.andExpect(jsonPath("$.dataset-id", is(datasetId)))
.andExpect(jsonPath("$.state", is(status)))
.andExpect(jsonPath("$.creation-date", is(dateTime.toOffsetDateTime()
.format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss")))));
.format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss")))))
.andExpect(jsonPath("$.total-records", is(1)))
.andExpect(jsonPath("$.processed-records", is(1)));
}

@ParameterizedTest
Expand All @@ -873,7 +875,9 @@ void getDebiasReport_expectSuccess(String status) throws Exception {
.andExpect(jsonPath("$.dataset-id", is(datasetId)))
.andExpect(jsonPath("$.state", is(status)))
.andExpect(jsonPath("$.creation-date", is(dateTime.toOffsetDateTime()
.format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss")))));
.format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss")))))
.andExpect(jsonPath("$.total-records", is(1)))
.andExpect(jsonPath("$.processed-records", is(1)));
}

@ParameterizedTest
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
package eu.europeana.metis.sandbox.service.workflow;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;

import eu.europeana.metis.sandbox.common.TestUtils;
import eu.europeana.metis.sandbox.common.locale.Country;
import eu.europeana.metis.sandbox.common.locale.Language;
import eu.europeana.metis.sandbox.domain.Record;
import eu.europeana.metis.sandbox.service.workflow.DeBiasProcessServiceImpl.DeBiasInputRecord;
import eu.europeana.metis.schema.convert.RdfConversionUtils;
import eu.europeana.metis.schema.jibx.RDF;
import java.io.File;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Set;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

class DeBiasRdfInfoExtractorTest {

DeBiasRdfInfoExtractor extractor;

@BeforeEach
void setUp() throws Exception {
String testRecordContent = new TestUtils().readFileToString(
"record" + File.separator + "debias" + File.separator + "debias-video-record.xml");
var testRecord = Record.builder()
.recordId(1L)
.europeanaId("europeanaId1")
.content(testRecordContent.getBytes(StandardCharsets.UTF_8))
.language(Language.NL).country(Country.NETHERLANDS)
.datasetName("datasetName")
.datasetId("1")
.build();
RDF rdf = new RdfConversionUtils().convertStringToRdf(new String(testRecord.getContent(), StandardCharsets.UTF_8));

extractor = new DeBiasRdfInfoExtractor(rdf, testRecord);
}

@Test
void getDescriptionsAndLanguageFromRdf() {
var expected = Set.of(
"Interactive quicktime panorama of link path 1-5.Skara Brae is an archaeological site ",
"Interaktives Quicktime-Panorama des Verbindungspfads 1-5. Skara Brae ist eine archäol",
"Panorama quicktime interattivo del percorso di collegamento 1-5. Skara Brae è un sito",
"Interactief quicktime panorama van linkpad 1-5. Skara Brae is een archeologische vind",
"Panorama interactif en temps réel du chemin de liaison 1-5. Skara Brae est un site ar");
List<DeBiasInputRecord> list = extractor.getDescriptionsAndLanguageFromRdf();
assertNotNull(list);
list.forEach(item -> assertEquals(DeBiasSourceField.DC_DESCRIPTION, item.sourceField()));
assertEquals(Set.of("nl", "de", "it", "fr", "en"), list.stream()
.map(DeBiasInputRecord::language)
.map(DeBiasSupportedLanguage::getCodeISO6391)
.collect(Collectors.toSet()));
assertEquals(expected, list.stream()
.map(DeBiasInputRecord::literal)
.map(item -> item.subSequence(0, 85))
.collect(Collectors.toSet())
);
}

@Test
void getTitlesAndLanguageFromRdf() {
var expected = Set.of("Panorama Movie of link path 1-5, Skara Brae addict and aboriginal",
"Panoramafilm des Verbindungspfads 1-5, Skara Brae, Süchtiger und Ureinwohner",
"Film panoramico del percorso di collegamento 1-5, Skara Brae, tossicodipendente e aborigeno",
"Panoramafilm van linkpad 1-5, Skara Brae, verslaafde en aboriginal",
"Film panoramique du chemin de liaison 1-5, Skara Brae, toxicomane et aborigène");
List<DeBiasInputRecord> list = extractor.getTitlesAndLanguageFromRdf();
assertNotNull(list);
list.forEach(item -> assertEquals(DeBiasSourceField.DC_TITLE, item.sourceField()));
assertEquals(Set.of("nl", "de", "it", "fr", "en"), list.stream()
.map(DeBiasInputRecord::language)
.map(DeBiasSupportedLanguage::getCodeISO6391)
.collect(Collectors.toSet()));
assertEquals(expected, list.stream()
.map(DeBiasInputRecord::literal)
.collect(Collectors.toSet()));
}

@Test
void getAlternativeAndLanguageFromRdf() {
List<DeBiasInputRecord> list = extractor.getAlternativeAndLanguageFromRdf();
assertNotNull(list);
assertTrue(list.isEmpty());
}

@Test
void getSubjectAndLanguageFromRdf() {
var expected = Set.of("Settlement clans", "Clans de la colonie", "Clan di insediamento", "Siedlungsclans");
List<DeBiasInputRecord> list = extractor.getSubjectAndLanguageFromRdf();
assertNotNull(list);
list.forEach(item -> assertEquals(DeBiasSourceField.DC_SUBJECT_LITERAL, item.sourceField()));
assertEquals(Set.of("nl", "de", "it", "fr", "en"), list.stream()
.map(DeBiasInputRecord::language)
.map(DeBiasSupportedLanguage::getCodeISO6391)
.collect(Collectors.toSet()));
assertEquals(expected, list.stream()
.map(DeBiasInputRecord::literal)
.collect(Collectors.toSet()));
}

@Test
void getTypeAndLanguageFromRdf() {
var expected = Set.of("Movie housewife, guy, chairman, addict, victims, homemaker",
"Femme au foyer, homme, président, toxicomane, victimes, femme au foyer",
"Filmhuisvrouw, man, voorzitter, verslaafde, slachtoffers, huisvrouw",
"Casalinga del cinema, ragazzo, presidente, tossicodipendente, vittime, casalinga",
"Filmhausfrau, Typ, Vorsitzender, Süchtiger, Opfer, Hausfrau");
List<DeBiasInputRecord> list = extractor.getTypeAndLanguageFromRdf();
assertNotNull(list);
list.forEach(item -> assertEquals(DeBiasSourceField.DC_TYPE_LITERAL, item.sourceField()));
assertEquals(Set.of("nl", "de", "it", "fr", "en"), list.stream()
.map(DeBiasInputRecord::language)
.map(DeBiasSupportedLanguage::getCodeISO6391)
.collect(Collectors.toSet()));
assertEquals(expected, list.stream()
.map(DeBiasInputRecord::literal)
.collect(Collectors.toSet()));
}

@Test
void getSubjectReferencesAndTypeReferencesFromRdf() {
List<DeBiasInputRecord> list = extractor.getSubjectReferencesAndTypeReferencesFromRdf();
assertNotNull(list);
assertTrue(list.isEmpty());
}

@Test
void partitionList() {
List<String> testPartition = List.of("A", "B", "C", "D", "E", "F", "G", "H");
Set<List<String>> expectedPartitions = Set.of(List.of("A", "B"), List.of("C", "D"),
List.of("E", "F"), List.of("G", "H"));
Stream<List<String>> listStream = DeBiasRdfInfoExtractor.partitionList(testPartition, 2);
assertNotNull(listStream);
listStream.forEach(item -> assertTrue(expectedPartitions.contains(item)));
}

@Test
void partitionListEmpty() {
Stream<List<String>> listStream = DeBiasRdfInfoExtractor.partitionList(List.of(), 2);
assertNotNull(listStream);
assertTrue(listStream.toList().isEmpty());
}

@Test
void partitionListError() {
Supplier<Object> makePartition = () -> DeBiasRdfInfoExtractor.partitionList(List.of(), -2);
IllegalArgumentException illegalArgumentException = assertThrows(IllegalArgumentException.class, makePartition::get);
assertNotNull(illegalArgumentException);
assertEquals("The partition size cannot be smaller than or equal 0. Actual value: -2", illegalArgumentException.getMessage());
}
}

0 comments on commit 3d62d4e

Please sign in to comment.