From fc62e6286937bab214ed7aaa06992eb83f41bbc2 Mon Sep 17 00:00:00 2001 From: GordeaS <4517853+gsergiu@users.noreply.github.com> Date: Tue, 29 Oct 2024 17:45:50 +0100 Subject: [PATCH] improvements for generation and logging of depictions #EA-3950 --- .../migration/DepictionGenerationReport.java | 35 ++++++++++++++ .../migration/GalleryDepictionMigration.java | 47 +++++++++++++------ .../service/impl/SearchApiClientImpl.java | 2 +- .../web/service/impl/UserSetServiceImpl.java | 6 ++- 4 files changed, 74 insertions(+), 16 deletions(-) create mode 100644 set-integration-testing/src/test/java/eu/europeana/api/set/integration/migration/DepictionGenerationReport.java diff --git a/set-integration-testing/src/test/java/eu/europeana/api/set/integration/migration/DepictionGenerationReport.java b/set-integration-testing/src/test/java/eu/europeana/api/set/integration/migration/DepictionGenerationReport.java new file mode 100644 index 00000000..32ca40ef --- /dev/null +++ b/set-integration-testing/src/test/java/eu/europeana/api/set/integration/migration/DepictionGenerationReport.java @@ -0,0 +1,35 @@ +package eu.europeana.api.set.integration.migration; + +public class DepictionGenerationReport { + int skipped = 0; + int notGenerated = 0; + int generated = 0; + int getSkipped() { + return skipped; + } + void setSkipped(int skipped) { + this.skipped = skipped; + } + void increaseSkipped() { + this.skipped++; + } + int getNotGenerated() { + return notGenerated; + } + void setNotGenerated(int notGenerated) { + this.notGenerated = notGenerated; + } + void increaseNotGenerated() { + this.notGenerated++; + } + + int getGenerated() { + return generated; + } + void setGenerated(int generated) { + this.generated = generated; + } + void increaseGenerated() { + this.generated++; + } +} diff --git a/set-integration-testing/src/test/java/eu/europeana/api/set/integration/migration/GalleryDepictionMigration.java b/set-integration-testing/src/test/java/eu/europeana/api/set/integration/migration/GalleryDepictionMigration.java index 83f4cbaf..da4e2901 100644 --- a/set-integration-testing/src/test/java/eu/europeana/api/set/integration/migration/GalleryDepictionMigration.java +++ b/set-integration-testing/src/test/java/eu/europeana/api/set/integration/migration/GalleryDepictionMigration.java @@ -4,6 +4,7 @@ import java.util.List; import javax.annotation.Resource; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.security.core.Authentication; import org.springframework.test.context.DynamicPropertyRegistry; @@ -49,16 +50,18 @@ public static void initTokens() { @DynamicPropertySource static void setProperties(DynamicPropertyRegistry registry) { - // registry.add("mongodb.set.connectionUrl", MONGO_CONTAINER::getConnectionUrl); - registry.add("mongodb.set.connectionUrl", () -> "mongodb://127.0.0.1:27017/set_test"); + registry.add("mongodb.set.connectionUrl", () -> "mongodb://127.0.0.1:27017/set_test"); + //registry.add("mongodb.set.connectionUrl", () -> ""); + //registry.add("mongodb.set.truststore", () -> ""); + //registry.add("mongodb.set.truststorepass", () -> ""); } /* * Generate isShownBy field for all sets in the (local) db */ - //@Test + @Test public void generateGalleriesWithDepiction() throws Exception { - createTestUserSet(USER_SET_REGULAR, regularUserToken); +// createTestUserSet(USER_SET_REGULAR, regularUserToken); // create object in database UserSetQueryBuilder queryBuilder = new UserSetQueryBuilder(); @@ -72,12 +75,13 @@ public void generateGalleriesWithDepiction() throws Exception { queryBuilder.buildUserSetQuery("type:Collection", null, sort, page, pageSize, getConfiguration()); final ArrayList profiles = new ArrayList(); profiles.add(LdProfiles.STANDARD); + DepictionGenerationReport report = new DepictionGenerationReport(); ResultSet results = null; do { results = getUserSetService().search(searchQuery, null, profiles, adminAuth); - generateDepictions(results.getResults()); + generateDepictions(results.getResults(), report); //move to next page page++; @@ -89,34 +93,49 @@ public void generateGalleriesWithDepiction() throws Exception { } - private void generateDepictions(List results) { + private void generateDepictions(List results, DepictionGenerationReport report) { + for (UserSet userSet : results) { + if(userSet.isOpenSet() || userSet.isBookmarksFolder() || userSet.isEntityBestItemsSet()) { + //bookmarks and entity best items sets is redundant, but we keep it for future + //open/dynamic sets must not be migrated + report.increaseSkipped(); + continue; + } + if(userSet.getIsShownBy() != null) { + report.increaseSkipped(); + continue; + } + final WebResource isShownBy = generateGalleryDepiction(userSet); //do not update set if the depiction cannot be generated if(isShownBy != null) { userSet.setIsShownBy(isShownBy); userSet.setCollectionType(WebUserSetFields.TYPE_GALLERY); mongoPersistance.store(userSet); + report.increaseGenerated(); + } else { + report.increaseNotGenerated(); } } + System.out.println("Generated depictions: " + report.getGenerated()); + System.out.println("Skipped sets: " + report.getSkipped()); + System.out.println("Not generated: " + report.getNotGenerated()); } private WebResource generateGalleryDepiction(UserSet userSet){ - if(userSet.isOpenSet() || userSet.isBookmarksFolder() || userSet.isEntityBestItemsSet()) { - //bookmarks and entity best items sets is redundant, but we keep it for future - //open/dynamic sets must not be migrated - return null; - } - + try { return getUserSetService().generateDepiction(userSet); } catch (SearchApiClientException e) { //work with best user effort - System.out.println(e.getMessage()); - e.printStackTrace(); + System.out.println("Cannot generate depiciton for set: "+ userSet.getIdentifier() + ", " + e.getMessage()); + //e.printStackTrace(); return null; } } + + private boolean hasNext(final int pageSize, ResultSet results) { return results.getResultSize() < pageSize; diff --git a/set-search-api-client/src/main/java/eu/europeana/set/search/service/impl/SearchApiClientImpl.java b/set-search-api-client/src/main/java/eu/europeana/set/search/service/impl/SearchApiClientImpl.java index d5670b62..7e50be50 100644 --- a/set-search-api-client/src/main/java/eu/europeana/set/search/service/impl/SearchApiClientImpl.java +++ b/set-search-api-client/src/main/java/eu/europeana/set/search/service/impl/SearchApiClientImpl.java @@ -223,7 +223,7 @@ public void fillDepiction(String searchApiFullUrl, String itemId, BaseWebResourc private String getResourceId(String thumbnailUrl) throws SearchApiClientException { - final String queryString = StringUtils.substringAfter(thumbnailUrl, '?'); + final String queryString = StringUtils.substringAfter(thumbnailUrl, "?"); List params = URLEncodedUtils.parse(queryString, StandardCharsets.UTF_8); for (NameValuePair param : params) { if ("uri".equals(param.getName())) { diff --git a/set-web/src/main/java/eu/europeana/set/web/service/impl/UserSetServiceImpl.java b/set-web/src/main/java/eu/europeana/set/web/service/impl/UserSetServiceImpl.java index bd7d098d..da8b5047 100644 --- a/set-web/src/main/java/eu/europeana/set/web/service/impl/UserSetServiceImpl.java +++ b/set-web/src/main/java/eu/europeana/set/web/service/impl/UserSetServiceImpl.java @@ -956,9 +956,13 @@ private boolean isPublishingPrevented(PersistentUserSet userSet) { @Override public WebResource generateDepiction(UserSet userSet) throws SearchApiClientException { + if(userSet.getItems() == null || userSet.getItems().isEmpty()) { + return null; + } + String itemId = userSet.getItems().get(0); String url = SearchApiUtils.getInstance().buildSearchApiUrlForItem(getConfiguration().getSearchApiUrl(), - itemId, getConfiguration().getSearchApiKey(), "minimal"); + itemId, getConfiguration().getSearchApiKey(), getConfiguration().getSearchApiProfileForItemDescriptions()); WebResource depiction = new WebResource(); getSearchApiClient().fillDepiction(url, itemId, depiction);