diff --git a/src/main/java/eu/europeana/metis/sandbox/service/workflow/DeBiasProcessServiceImpl.java b/src/main/java/eu/europeana/metis/sandbox/service/workflow/DeBiasProcessServiceImpl.java index ad512f31..2fd06ff6 100644 --- a/src/main/java/eu/europeana/metis/sandbox/service/workflow/DeBiasProcessServiceImpl.java +++ b/src/main/java/eu/europeana/metis/sandbox/service/workflow/DeBiasProcessServiceImpl.java @@ -137,7 +137,8 @@ private void updateDeBiasProgressCounters(String datasetId) { * @param deBiasReport the DeBias report */ private void doDeBiasAndGenerateReport(List recordList, List deBiasReport) { - getDeBiasSourceFieldsFromRecords(recordList) + List info = getDeBiasSourceFieldsFromRecords(recordList); + info .stream() .collect(groupingBy(DeBiasInputRecord::language)) .forEach(((deBiasSupportedLanguage, recordDescriptions) -> @@ -234,6 +235,7 @@ private List getDeBiasSourceFieldsFromRecords(List re } catch (SerializationException e) { deBiasInputRecords = Collections.emptyList(); + LOGGER.error("Serialization {}",e.getMessage(), e); } return deBiasInputRecords; }).flatMap(Collection::stream).toList(); diff --git a/src/test/java/eu/europeana/metis/sandbox/service/debias/RecordPublishDeBiasQueueServiceTest.java b/src/test/java/eu/europeana/metis/sandbox/service/debias/RecordPublishDeBiasQueueServiceTest.java new file mode 100644 index 00000000..920fe47a --- /dev/null +++ b/src/test/java/eu/europeana/metis/sandbox/service/debias/RecordPublishDeBiasQueueServiceTest.java @@ -0,0 +1,71 @@ +package eu.europeana.metis.sandbox.service.debias; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import eu.europeana.metis.sandbox.common.locale.Country; +import eu.europeana.metis.sandbox.common.locale.Language; +import eu.europeana.metis.sandbox.domain.Record; +import eu.europeana.metis.sandbox.domain.RecordInfo; +import eu.europeana.metis.sandbox.domain.RecordProcessEvent; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.amqp.AmqpException; +import org.springframework.amqp.core.AmqpTemplate; + +@ExtendWith(MockitoExtension.class) +class RecordPublishDeBiasQueueServiceTest { + + @Mock + private AmqpTemplate amqpTemplate; + + private RecordPublishDeBiasQueueService service; + + @BeforeEach + void setUp() { + service = new RecordPublishDeBiasQueueService(amqpTemplate, "deBiasReadyQueue"); + } + + @Test + void publishToDeBiasQueue() { + var testRecord = Record.builder() + .datasetId("1") + .datasetName("") + .recordId(1L) + .europeanaId("1") + .country(Country.NETHERLANDS) + .language(Language.NL) + .content("content".getBytes()) + .build(); + service.publishToDeBiasQueue(new RecordInfo(testRecord)); + verify(amqpTemplate, times(1)) + .convertAndSend(eq("deBiasReadyQueue"), any(RecordProcessEvent.class)); + } + + @Test + void publishToDeBiasQueueException() { + doThrow(new AmqpException("error in queue")).when(amqpTemplate).convertAndSend(eq("deBiasReadyQueue"), any(RecordProcessEvent.class)); + var testRecord = Record.builder() + .datasetId("1") + .datasetName("") + .recordId(1L) + .europeanaId("1") + .country(Country.NETHERLANDS) + .language(Language.NL) + .content("content".getBytes()) + .build(); + service.publishToDeBiasQueue(new RecordInfo(testRecord)); + verify(amqpTemplate, times(1)) + .convertAndSend(eq("deBiasReadyQueue"), any(RecordProcessEvent.class)); + } +} diff --git a/src/test/java/eu/europeana/metis/sandbox/service/workflow/DeBiasProcessServiceImplTest.java b/src/test/java/eu/europeana/metis/sandbox/service/workflow/DeBiasProcessServiceImplTest.java new file mode 100644 index 00000000..7bdfdb84 --- /dev/null +++ b/src/test/java/eu/europeana/metis/sandbox/service/workflow/DeBiasProcessServiceImplTest.java @@ -0,0 +1,73 @@ +package eu.europeana.metis.sandbox.service.workflow; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import eu.europeana.metis.debias.detect.client.DeBiasClient; +import eu.europeana.metis.sandbox.common.TestUtils; +import eu.europeana.metis.sandbox.common.locale.Country; +import eu.europeana.metis.sandbox.common.locale.Language; +import eu.europeana.metis.sandbox.domain.Record; +import eu.europeana.metis.sandbox.repository.RecordLogRepository; +import eu.europeana.metis.sandbox.repository.RecordRepository; +import eu.europeana.metis.sandbox.repository.debias.DatasetDeBiasRepository; +import eu.europeana.metis.sandbox.repository.debias.RecordDeBiasDetailRepository; +import eu.europeana.metis.sandbox.repository.debias.RecordDeBiasMainRepository; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class DeBiasProcessServiceImplTest { + + @Mock + DeBiasClient deBiasClient; + @Mock + RecordDeBiasMainRepository recordDeBiasMainRepository; + @Mock + RecordDeBiasDetailRepository recordDeBiasDetailRepository; + @Mock + DatasetDeBiasRepository datasetDeBiasRepository; + @Mock + RecordLogRepository recordLogRepository; + @Mock + RecordRepository recordRepository; + + @InjectMocks + DeBiasProcessServiceImpl deBiasProcessService; + + @Test + void processSuccess() throws IOException { + String testRecord1 = new TestUtils().readFileToString("record"+ File.separator+"debias"+File.separator+"debias-text-record.xml"); + String testRecord2 = new TestUtils().readFileToString("record"+ File.separator+"debias"+File.separator+"debias-video-record.xml"); + var testRecords = List.of( + Record.builder() + .recordId(1L) + .europeanaId("europeanaId1") + .content(testRecord1.getBytes(StandardCharsets.UTF_8)) + .language(Language.NL).country(Country.NETHERLANDS) + .datasetName("datasetName") + .datasetId("1") + .build(), + Record.builder() + .recordId(2L) + .europeanaId("europeanaId2") + .content(testRecord2.getBytes()) + .language(Language.NL).country(Country.NETHERLANDS) + .datasetName("datasetName") + .datasetId("1") + .build()); + + deBiasProcessService.process(testRecords); + + verify(deBiasClient,times(5)).detect(any()); + } +} diff --git a/src/test/resources/record/debias/debias-text-record.xml b/src/test/resources/record/debias/debias-text-record.xml new file mode 100644 index 00000000..e09482f8 --- /dev/null +++ b/src/test/resources/record/debias/debias-text-record.xml @@ -0,0 +1,71 @@ + + + + + + + + 1923 + + 1923-01-01 + 1923-12-31 + 1923 + + + + Małopolska Biblioteka Cyfrowa + + + Federacja Bibliotek Cyfrowych + + + + Beaupre, Antoni.(Red.) + 1923 + Pan i niewolnik Wojewódzka Biblioteka Publiczna w Krakowie dla ludzkości + Master and slave Provincial Public Library in Krakow + Herr und Sklave Öffentliche Provinzbibliothek in Krakau für die Menschheit + Biblioteca pubblica provinciale del padrone e dello schiavo di Cracovia per l'umanità + + Bibliothèque publique provinciale Maître et esclave de Cracovie pour l'humanité + Meester en slaaf Provinciale Openbare Bibliotheek in Krakau voor de mensheid + image/vnd.djvu + pol + Spółka Wydawnicza "Czas" + Biblioteka Książąt Czartoryskich + 20 w. + gazety polskie + Czas. 1923, nr 251 (10 XI) + czasopismo + + Dziennik polityczny i literacko-społeczny; organ konserwatystów. - Opis fiz.: 4 s. ; 61 cm. + false + + + TEXT + + + + #nnhSX08 + pl + true + + + + + + + Europeana Foundation + Europeana Foundation + 352_local_23102024_1633 + Netherlands + nl + + diff --git a/src/test/resources/record/debias/debias-video-record.xml b/src/test/resources/record/debias/debias-video-record.xml new file mode 100644 index 00000000..a7913d6f --- /dev/null +++ b/src/test/resources/record/debias/debias-video-record.xml @@ -0,0 +1,145 @@ + + + + + Quicktime interactive panorama of the link path between houses 1 and 5 at Skara Brae + + mov + © CMC Associates + CMC + + + + 59.04861 + -3.343056 + 7.0 + Skara Brae + + + Neolithic + -3180 + -2500 + + + + + + CMC - CMC Associates Ltd. + + + + CARARE + + + + Interactive quicktime panorama of link path 1-5.Skara Brae is an archaeological site with + exceptionally well preserved remains that show the stone built furniture and internal structure of ten Neolithic houses and + their narrow connecting passageways. Radio Carbon dates show that the village was occupied for around 600 years between 3200 + and 2200 BC. There appear to be two main structural phases to the occupation.Between the later houses at Skara Brae is a + network of low winding passages. These made it possible to travel between Houses 1-7 without going outside. The passages are + around a meter high and roofed with stone slabs. The main passage has bar-holes in the walls at either end suggesting that + it could have been sealed from the inside.There were further bar-holes at the entrances to each of the houses allowing them + to be closed and barred with stone slabs.The monument is now managed by Historic Scotland and is a key element of the Heart + of Neolithic Orkney World Heritage Site. + + Interaktives Quicktime-Panorama des Verbindungspfads 1-5. Skara Brae ist eine archäologische + Stätte mit außergewöhnlich gut erhaltenen Überresten, die die aus Stein gebauten Möbel und die innere Struktur von zehn + neolithischen Häusern und ihren schmalen Verbindungsgängen zeigen. Radiokarbon-Datierungen zeigen, dass das Dorf zwischen + 3200 und 2200 v. Chr. etwa 600 Jahre lang bewohnt war. Es scheint zwei Hauptphasen der Besiedlung zu geben. Zwischen den + späteren Häusern von Skara Brae befindet sich ein Netzwerk niedriger, gewundener Gänge. Diese ermöglichten es, zwischen den + Häusern 1-7 zu reisen, ohne das Haus zu verlassen. Die Gänge sind etwa einen Meter hoch und mit Steinplatten überdacht. Der + Hauptgang weist an beiden Enden Gitterlöcher in den Wänden auf, was darauf schließen lässt, dass er von innen abgeriegelt + gewesen sein könnte. An den Eingängen zu den einzelnen Häusern befanden sich weitere Gitterlöcher, sodass diese geschlossen + und mit Steinplatten verriegelt werden konnten. Das Denkmal wird jetzt von Historic Scotland verwaltet und ist ein zentrales + Element der Weltkulturerbestätte „Heart of Neolithic Orkney“. + + Panorama quicktime interattivo del percorso di collegamento 1-5. Skara Brae è un sito + archeologico con resti eccezionalmente ben conservati che mostrano i mobili in pietra e la struttura interna di dieci case + neolitiche e i loro stretti passaggi di collegamento. Le datazioni al radiocarbonio mostrano che il villaggio fu occupato + per circa 600 anni tra il 3200 e il 2200 a.C. Sembra che ci siano due fasi strutturali principali dell'occupazione. Tra le + case successive a Skara Brae c'è una rete di passaggi bassi e tortuosi. Questi hanno reso possibile viaggiare tra le case + 1-7 senza uscire. I passaggi sono alti circa un metro e coperti con lastre di pietra. Il passaggio principale presenta delle + aperture per le sbarre nei muri a entrambe le estremità, il che suggerisce che potrebbe essere stato sigillato dall'interno. + C'erano altre aperture per le sbarre agli ingressi di ciascuna casa, che consentivano di chiuderli e sbarrarli con lastre di + pietra. Il monumento è ora gestito da Historic Scotland ed è un elemento chiave del sito patrimonio mondiale dell'umanità + Heart of Neolithic Orkney. + + Interactief quicktime panorama van linkpad 1-5. Skara Brae is een archeologische vindplaats met + uitzonderlijk goed bewaarde overblijfselen die het stenen meubilair en de interne structuur van tien neolithische huizen en + hun smalle verbindingsgangen laten zien. Radiokoolstofdateringen laten zien dat het dorp ongeveer 600 jaar bewoond was + tussen 3200 en 2200 v.Chr. Er lijken twee belangrijke structurele fasen te zijn geweest in de bewoning. Tussen de latere + huizen in Skara Brae bevindt zich een netwerk van lage kronkelende gangen. Deze maakten het mogelijk om tussen huizen 1-7 te + reizen zonder naar buiten te gaan. De gangen zijn ongeveer een meter hoog en overdekt met stenen platen. De hoofdgang heeft + aan beide uiteinden gaten in de muren, wat erop wijst dat deze van binnenuit is afgesloten. Bij de ingangen van elk huis + zaten nog meer gaten, waardoor ze konden worden afgesloten en afgesloten met stenen platen. Het monument wordt nu beheerd + door Historic Scotland en is een belangrijk onderdeel van het Werelderfgoed Heart of Neolithic Orkney. + + Panorama interactif en temps réel du chemin de liaison 1-5. Skara Brae est un site archéologique + dont les vestiges sont exceptionnellement bien préservés et qui montrent le mobilier en pierre et la structure interne de + dix maisons néolithiques et leurs étroits passages de communication. Les datations au radiocarbone montrent que le village a + été occupé pendant environ 600 ans entre 3200 et 2200 av. J.-C. Il semble y avoir deux phases structurelles principales de + l'occupation. Entre les maisons ultérieures de Skara Brae se trouve un réseau de passages sinueux et bas. Ceux-ci + permettaient de voyager entre les maisons 1 à 7 sans sortir. Les passages mesurent environ un mètre de haut et sont couverts + de dalles de pierre. Le passage principal présente des trous dans les murs à chaque extrémité, suggérant qu'il aurait pu + être scellé de l'intérieur. Il y avait d'autres trous dans les entrées de chacune des maisons, ce qui permettait de les + fermer et de les barrer avec des dalles de pierre. Le monument est désormais géré par Historic Scotland et constitue un + élément clé du site du patrimoine mondial du cœur du Néolithique des Orcades. + + mov + http://3dicons.dcu.gr/object/HA/1255 + English + + CMC Associates + CMC + + + Settlement clans + Clans de la colonie + Settlement clans + Clan di insediamento + Siedlungsclans + Panorama Movie of link path 1-5, Skara Brae addict and aboriginal + Panoramafilm des Verbindungspfads 1-5, Skara Brae, Süchtiger und Ureinwohner + Film panoramico del percorso di collegamento 1-5, Skara Brae, tossicodipendente e aborigeno + Panoramafilm van linkpad 1-5, Skara Brae, verslaafde en aboriginal + Film panoramique du chemin de liaison 1-5, Skara Brae, toxicomane et aborigène + Movie housewife, guy, chairman, addict, victims, homemaker + Femme au foyer, homme, président, toxicomane, victimes, femme au foyer + Filmhuisvrouw, man, voorzitter, verslaafde, slachtoffers, huisvrouw + Casalinga del cinema, ragazzo, presidente, tossicodipendente, vittime, casalinga + Filmhausfrau, Typ, Vorsitzender, Süchtiger, Opfer, Hausfrau + 3D ICONS + + Neolithic + + false + + + VIDEO + + + CMC_HA/1255 + en + true + + + + + + + Europeana Foundation + Europeana Foundation + 352_local_23102024_1633 + Netherlands + nl + +