diff --git a/CONTRIBUTING.MD b/CONTRIBUTING.MD index 1e59b1733..1b650ed48 100644 --- a/CONTRIBUTING.MD +++ b/CONTRIBUTING.MD @@ -128,14 +128,40 @@ If there is any doubt on fixing the merge conflicts while merging, the implement worked on the relevant changes that were introduced on the destination branch. The merge can be performed on the github pull request page or manually(especially for conflicts). -To do this manually, checkout the destination branch(usually `develop`) and execute the merge command with the `--no-ff` -parameter: +To do this manually, checkout the destination branch(usually `develop`). +We prefer squash merging or alternatively non fast forward merging. + +- A squash merge can be performed with the `--squash` parameter: + + `git merge --squash ` + + To complete the squash merge, a commit has to also be performed if done locally. + The commit should be formatted as the following template(replicating github squashed commits): + ``` + / (#) + (optionally as description) + * List of all commit messages from the pull requst + ``` + Example message: + ``` + Debt/met 4250 refactor code to remove mock maker inline (#508) + * MET-4250 Update NetworkUtil + + * MET-4250 Update RdfConversionUtils + + * MET-4250 Javadocs and cleanup + + * MET-4250 Remove mockito inline from root pom + ``` + + +- A non fast forward merge can be performed with the `--no-ff` parameter: `git merge --no-ff ` -The merger should check that the local branch is building before and after merging. If there were merge conflicts that were -resolved during the merge, then a local deployment should be triggered and verified. If the build succeeds the destination branch -can be pushed to the remote repository and the pull request will be resolved. +The merger should check that the local branch is building before and after merging. +If there were merge conflicts that were resolved during the merge, then a local deployment should be triggered and verified. +If the build succeeds the destination branch can be pushed to the remote repository and the pull request will be resolved. The reviewer can now move the ticket ahead in the board and re-assign it to the implementor. diff --git a/metis-authentication/metis-authentication-common/pom.xml b/metis-authentication/metis-authentication-common/pom.xml index a1ebce213..30795e225 100644 --- a/metis-authentication/metis-authentication-common/pom.xml +++ b/metis-authentication/metis-authentication-common/pom.xml @@ -4,7 +4,7 @@ metis-authentication eu.europeana.metis - 6 + 7 metis-authentication-common diff --git a/metis-authentication/metis-authentication-rest-client/pom.xml b/metis-authentication/metis-authentication-rest-client/pom.xml index fa5dcbd8f..472ffd320 100644 --- a/metis-authentication/metis-authentication-rest-client/pom.xml +++ b/metis-authentication/metis-authentication-rest-client/pom.xml @@ -4,7 +4,7 @@ metis-authentication eu.europeana.metis - 6 + 7 metis-authentication-rest-client diff --git a/metis-authentication/metis-authentication-rest-client/src/test/java/eu/europeana/metis/authentication/rest/client/TestAuthenticationClient.java b/metis-authentication/metis-authentication-rest-client/src/test/java/eu/europeana/metis/authentication/rest/client/TestAuthenticationClient.java index 07e4633b5..16fd1c40b 100644 --- a/metis-authentication/metis-authentication-rest-client/src/test/java/eu/europeana/metis/authentication/rest/client/TestAuthenticationClient.java +++ b/metis-authentication/metis-authentication-rest-client/src/test/java/eu/europeana/metis/authentication/rest/client/TestAuthenticationClient.java @@ -28,7 +28,7 @@ class TestAuthenticationClient { static { try { - portForWireMock = NetworkUtil.getAvailableLocalPort(); + portForWireMock = new NetworkUtil().getAvailableLocalPort(); } catch (IOException e) { e.printStackTrace(); } diff --git a/metis-authentication/metis-authentication-rest/pom.xml b/metis-authentication/metis-authentication-rest/pom.xml index c566c4380..9146f6ce4 100644 --- a/metis-authentication/metis-authentication-rest/pom.xml +++ b/metis-authentication/metis-authentication-rest/pom.xml @@ -4,7 +4,7 @@ metis-authentication eu.europeana.metis - 6 + 7 metis-authentication-rest war diff --git a/metis-authentication/metis-authentication-service/pom.xml b/metis-authentication/metis-authentication-service/pom.xml index 0dea6f614..57030a4f6 100644 --- a/metis-authentication/metis-authentication-service/pom.xml +++ b/metis-authentication/metis-authentication-service/pom.xml @@ -4,7 +4,7 @@ metis-authentication eu.europeana.metis - 6 + 7 metis-authentication-service diff --git a/metis-authentication/metis-authentication-service/src/main/java/eu/europeana/metis/authentication/service/AuthenticationService.java b/metis-authentication/metis-authentication-service/src/main/java/eu/europeana/metis/authentication/service/AuthenticationService.java index bcd0cec80..d02017fbd 100644 --- a/metis-authentication/metis-authentication-service/src/main/java/eu/europeana/metis/authentication/service/AuthenticationService.java +++ b/metis-authentication/metis-authentication-service/src/main/java/eu/europeana/metis/authentication/service/AuthenticationService.java @@ -4,9 +4,9 @@ import eu.europeana.metis.authentication.dao.PsqlMetisUserDao; import eu.europeana.metis.authentication.user.AccountRole; import eu.europeana.metis.authentication.user.Credentials; -import eu.europeana.metis.authentication.user.MetisUserView; -import eu.europeana.metis.authentication.user.MetisUserAccessToken; import eu.europeana.metis.authentication.user.MetisUser; +import eu.europeana.metis.authentication.user.MetisUserAccessToken; +import eu.europeana.metis.authentication.user.MetisUserView; import eu.europeana.metis.authentication.utils.ZohoMetisUserUtils; import eu.europeana.metis.exception.BadContentException; import eu.europeana.metis.exception.GenericMetisException; @@ -36,8 +36,7 @@ import org.springframework.stereotype.Service; /** - * Service that handles all related operations to authentication including communication between a - * psql database and Zoho. + * Service that handles all related operations to authentication including communication between a psql database and Zoho. * * @author Simon Tzanakis (Simon.Tzanakis@europeana.eu) * @since 2018-12-05 @@ -47,10 +46,10 @@ public class AuthenticationService { private static final int LOG_ROUNDS = 13; private static final int CREDENTIAL_FIELDS_NUMBER = 2; + @SuppressWarnings("java:S6418") // It is not an actual token private static final String ACCESS_TOKEN_CHARACTER_BASKET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; private static final int ACCESS_TOKEN_LENGTH = 32; - private static final Pattern TOKEN_MATCHING_PATTERN = Pattern - .compile("^[" + ACCESS_TOKEN_CHARACTER_BASKET + "]*$"); + private static final Pattern TOKEN_MATCHING_PATTERN = Pattern.compile("^[" + ACCESS_TOKEN_CHARACTER_BASKET + "]*$"); public static final Supplier COULD_NOT_CONVERT_EXCEPTION_SUPPLIER = () -> new BadContentException( "Could not convert internal user"); private final PsqlMetisUserDao psqlMetisUserDao; @@ -243,7 +242,7 @@ public String validateAuthorizationHeaderWithAccessToken(String authorization) } //Check that the token is of valid structure if (accessToken.length() != ACCESS_TOKEN_LENGTH || !TOKEN_MATCHING_PATTERN.matcher(accessToken) - .matches()) { + .matches()) { throw new UserUnauthorizedException("Access token invalid"); } return accessToken; @@ -368,7 +367,8 @@ public boolean hasPermissionToRequestUserUpdate(String accessToken, String userE } MetisUser storedMetisUser = authenticateUserInternal(accessToken); return storedMetisUser.getAccountRole() == AccountRole.METIS_ADMIN || storedMetisUser.getEmail() - .equals(storedMetisUserToUpdate.getEmail()); + .equals( + storedMetisUserToUpdate.getEmail()); } String generateAccessToken() { @@ -480,14 +480,14 @@ public List getAllUsers() { return convert(psqlMetisUserDao.getAllMetisUsers()); } - private static MetisUserView convert(MetisUser record) throws BadContentException { - return Optional.ofNullable(record).map(MetisUserView::new) - .orElseThrow(COULD_NOT_CONVERT_EXCEPTION_SUPPLIER); + private static MetisUserView convert(MetisUser metisUser) throws BadContentException { + return Optional.ofNullable(metisUser).map(MetisUserView::new) + .orElseThrow(COULD_NOT_CONVERT_EXCEPTION_SUPPLIER); } private static List convert(List records) { return Optional.ofNullable(records).stream().flatMap(Collection::stream).map(MetisUserView::new) - .collect(Collectors.toList()); + .collect(Collectors.toList()); } } diff --git a/metis-authentication/pom.xml b/metis-authentication/pom.xml index a91b9daa4..096a31b2e 100644 --- a/metis-authentication/pom.xml +++ b/metis-authentication/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 6 + 7 metis-authentication pom diff --git a/metis-common/metis-common-mongo/pom.xml b/metis-common/metis-common-mongo/pom.xml index 9f45d1369..b395f5b97 100644 --- a/metis-common/metis-common-mongo/pom.xml +++ b/metis-common/metis-common-mongo/pom.xml @@ -4,7 +4,7 @@ metis-common eu.europeana.metis - 6 + 7 metis-common-mongo @@ -28,7 +28,6 @@ eu.europeana.metis metis-schema - ${project.version} dev.morphia.morphia diff --git a/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoClientProvider.java b/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoClientProvider.java index 7b6ae4333..fb79ff0ed 100644 --- a/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoClientProvider.java +++ b/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoClientProvider.java @@ -9,6 +9,7 @@ import com.mongodb.ServerAddress; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; +import com.mongodb.connection.ConnectionPoolSettings; import eu.europeana.metis.mongo.connection.MongoProperties.ReadPreferenceValue; import java.util.List; import java.util.Optional; @@ -17,8 +18,7 @@ import java.util.function.Supplier; /** - * This class can set up and provide a Mongo client given the Mongo properties. It applies the - * following default values: + * This class can set up and provide a Mongo client given the Mongo properties. It applies the following default values: *
    *
  • * The read preference for the connection is defaulted to {@link ReadPreference#secondaryPreferred()}. @@ -46,6 +46,7 @@ public class MongoClientProvider { private static final ReadPreference DEFAULT_READ_PREFERENCE = ReadPreference.secondaryPreferred(); private static final int DEFAULT_MAX_CONNECTION_IDLE_MILLIS = 30_000; + private static final int DEFAULT_MAX_CONNECTIONS = 20; private static final boolean DEFAULT_RETRY_WRITES = false; private static final String DEFAULT_APPLICATION_NAME = "Europeana Application Suite"; @@ -53,8 +54,8 @@ public class MongoClientProvider { private final String authenticationDatabase; /** - * Constructor from a connection URI string (see the documentation of {@link MongoClientURI} for - * the details). The connection URL can provide settings that will override the default settings. + * Constructor from a connection URI string (see the documentation of {@link MongoClientURI} for the details). The connection + * URL can provide settings that will override the default settings. * * @param connectionUri The connection URI as a string * @param exceptionCreator How to report exceptions. @@ -79,28 +80,27 @@ public MongoClientProvider(String connectionUri, Function exceptionCr } /** - * Constructor from a {@link MongoProperties} object. The caller needs to provide settings that - * will be used instead of the default settings. + * Constructor from a {@link MongoProperties} object. The caller needs to provide settings that will be used instead of the + * default settings. * - * @param properties The properties of the Mongo connection. Note that if the passed properties - * object is changed after calling this method, those changes will not be reflected when creating - * mongo clients. - * @param clientSettingsBuilder The settings to be applied. The default settings will not be used. - * The caller can however choose to incorporate the default settings as needed by using a client - * settings builder obtained from {@link #getDefaultClientSettingsBuilder()} as input. + * @param properties The properties of the Mongo connection. Note that if the passed properties object is changed after calling + * this method, those changes will not be reflected when creating mongo clients. + * @param clientSettingsBuilder The settings to be applied. The default settings will not be used. The caller can however choose + * to incorporate the default settings as needed by using a client settings builder obtained from {@link + * #getDefaultClientSettingsBuilder()} as input. * @throws E In case the properties are wrong */ public MongoClientProvider(MongoProperties properties, Builder clientSettingsBuilder) throws E { final ReadPreference readPreference = Optional.ofNullable(properties.getReadPreferenceValue()) - .map(ReadPreferenceValue::getReadPreferenceSupplier).map(Supplier::get) - .orElse(DEFAULT_READ_PREFERENCE); + .map(ReadPreferenceValue::getReadPreferenceSupplier).map(Supplier::get) + .orElse(DEFAULT_READ_PREFERENCE); clientSettingsBuilder.readPreference(readPreference); final List mongoHosts = properties.getMongoHosts(); final MongoCredential mongoCredential = properties.getMongoCredentials(); this.authenticationDatabase = Optional.ofNullable(mongoCredential) - .map(MongoCredential::getSource).orElse(null); + .map(MongoCredential::getSource).orElse(null); clientSettingsBuilder .applyToSslSettings(builder -> builder.enabled(properties.mongoEnableSsl())); clientSettingsBuilder.applyToClusterSettings(builder -> builder.hosts(mongoHosts)); @@ -109,6 +109,9 @@ public MongoClientProvider(MongoProperties properties, Builder clientSettings } Optional.ofNullable(properties.getApplicationName()).filter(name -> !name.isBlank()) .ifPresent(clientSettingsBuilder::applicationName); + + clientSettingsBuilder.applyToConnectionPoolSettings( + builder -> builder.applySettings(createConnectionPoolSettings(properties.getMaxConnectionPoolSize()))); final MongoClientSettings mongoClientSettings = clientSettingsBuilder.build(); this.creator = () -> MongoClients.create(mongoClientSettings); @@ -117,9 +120,8 @@ public MongoClientProvider(MongoProperties properties, Builder clientSettings /** * Constructor from a {@link MongoProperties} object, using the default settings. * - * @param properties The properties of the Mongo connection. Note that if the passed properties - * object is changed after calling this method, those changes will not be reflected when calling - * {@link #createMongoClient()}. + * @param properties The properties of the Mongo connection. Note that if the passed properties object is changed after calling + * this method, those changes will not be reflected when calling {@link #createMongoClient()}. * @throws E In case the properties are wrong */ public MongoClientProvider(MongoProperties properties) throws E { @@ -131,19 +133,17 @@ public MongoClientProvider(MongoProperties properties) throws E { * * @return A new instance of {@link Builder} with the default settings. */ - public static Builder getDefaultClientSettingsBuilder() { + public static MongoClientSettings.Builder getDefaultClientSettingsBuilder() { return MongoClientSettings.builder() - // TODO: 7/16/20 Remove default retry writes after upgrade to mongo server version 4.2 - .retryWrites(DEFAULT_RETRY_WRITES) - .applyToConnectionPoolSettings(builder -> builder - .maxConnectionIdleTime(DEFAULT_MAX_CONNECTION_IDLE_MILLIS, TimeUnit.MILLISECONDS)) - .readPreference(DEFAULT_READ_PREFERENCE) - .applicationName(DEFAULT_APPLICATION_NAME); + // TODO: 7/16/20 Remove default retry writes after upgrade to mongo server version 4.2 + .retryWrites(DEFAULT_RETRY_WRITES) + .applyToConnectionPoolSettings(builder -> builder.applySettings(getDefaultConnectionPoolSettings())) + .readPreference(DEFAULT_READ_PREFERENCE) + .applicationName(DEFAULT_APPLICATION_NAME); } /** - * Convenience method for {@link #MongoClientProvider(String, Function)}. See that - * constructor for the details. + * Convenience method for {@link #MongoClientProvider(String, Function)}. See that constructor for the details. * * @param connectionUri The connection URI. * @return An instance. @@ -153,8 +153,7 @@ public static MongoClientProvider create(String connec } /** - * Convenience method for {@link #MongoClientProvider(String, Function)}. See that - * constructor for the details. + * Convenience method for {@link #MongoClientProvider(String, Function)}. See that constructor for the details. * * @param connectionUri The connection URI. * @return A supplier for {@link MongoClient} instances based on this class. @@ -164,8 +163,17 @@ public static Supplier createAsSupplier(String connectionUri) { } /** - * Returns the authentication database for mongo connections that are provided. Can be null - * (signifying that the default is to be used or that no authentication is specified). + * Get the default connection pool settings + * + * @return the default connection pool settings + */ + private static ConnectionPoolSettings getDefaultConnectionPoolSettings() { + return createConnectionPoolSettings(null); + } + + /** + * Returns the authentication database for mongo connections that are provided. Can be null (signifying that the default is to + * be used or that no authentication is specified). * * @return The authentication database. */ @@ -174,9 +182,8 @@ public final String getAuthenticationDatabase() { } /** - * Creates a Mongo client. This method can be called multiple times and will create and return a - * different client each time. The calling code is responsible for properly closing the created - * client. + * Creates a Mongo client. This method can be called multiple times and will create and return a different client each time. The + * calling code is responsible for properly closing the created client. * * @return A mongo client. * @throws E In case there is a problem with creating the client. @@ -185,6 +192,23 @@ public final MongoClient createMongoClient() throws E { return creator.createMongoClient(); } + /** + * Create a connection pool settings object. Settings that are null will be set to default settings. + * + * @param maxPoolSize the maximum connection pool size + * @return the connection pool settings + */ + static ConnectionPoolSettings createConnectionPoolSettings(Integer maxPoolSize) { + final ConnectionPoolSettings.Builder builder = ConnectionPoolSettings.builder(); + builder.maxConnectionIdleTime(DEFAULT_MAX_CONNECTION_IDLE_MILLIS, TimeUnit.MILLISECONDS); + if (maxPoolSize != null && maxPoolSize > 0) { + builder.maxSize(maxPoolSize); + } else { + builder.maxSize(DEFAULT_MAX_CONNECTIONS); + } + return builder.build(); + } + private interface MongoClientCreator { MongoClient createMongoClient() throws E; diff --git a/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoProperties.java b/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoProperties.java index 497215e66..e71ee360d 100644 --- a/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoProperties.java +++ b/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoProperties.java @@ -28,6 +28,7 @@ public class MongoProperties { private MongoCredential mongoCredentials; private boolean mongoEnableSsl; private ReadPreferenceValue readPreferenceValue; + private Integer maxConnectionPoolSize; private String applicationName; /** @@ -150,8 +151,16 @@ public void setReadPreferenceValue(ReadPreferenceValue readPreferenceValue) { } /** - * Set the application name. Can be null, in which case a default generic application name is - * to be used. + * Get the maximum connection pol size + * + * @return the maximum connection pool size + */ + public Integer getMaxConnectionPoolSize() { + return maxConnectionPoolSize; + } + + /** + * Set the application name. Can be null, in which case a default generic application name is to be used. * * @param applicationName The application name, or null for the default. */ @@ -221,6 +230,15 @@ public ReadPreferenceValue getReadPreferenceValue() { return readPreferenceValue; } + /** + * Set the maximum connection poll size. Can be null, in which case the default applies. + * + * @param maxConnectionPoolSize the maximum connection pool size + */ + public void setMaxConnectionPoolSize(Integer maxConnectionPoolSize) { + this.maxConnectionPoolSize = maxConnectionPoolSize; + } + /** * This method returns the value of the application name (or null for the default). * diff --git a/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/embedded/EmbeddedLocalhostMongo.java b/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/embedded/EmbeddedLocalhostMongo.java index 3667fb795..5249b59a3 100644 --- a/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/embedded/EmbeddedLocalhostMongo.java +++ b/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/embedded/EmbeddedLocalhostMongo.java @@ -39,7 +39,7 @@ public EmbeddedLocalhostMongo() { public void start() { if (mongodExecutable == null) { try { - mongoPort = NetworkUtil.getAvailableLocalPort(); + mongoPort = new NetworkUtil().getAvailableLocalPort(); RuntimeConfig runtimeConfig = Defaults.runtimeConfigFor(Command.MongoD, LOGGER) .processOutput(ProcessOutput.getDefaultInstanceSilent()) .build(); diff --git a/metis-common/metis-common-mongo/src/test/java/eu/europeana/metis/mongo/connection/MongoClientProviderTest.java b/metis-common/metis-common-mongo/src/test/java/eu/europeana/metis/mongo/connection/MongoClientProviderTest.java index 4b68389e5..145bba9fc 100644 --- a/metis-common/metis-common-mongo/src/test/java/eu/europeana/metis/mongo/connection/MongoClientProviderTest.java +++ b/metis-common/metis-common-mongo/src/test/java/eu/europeana/metis/mongo/connection/MongoClientProviderTest.java @@ -8,8 +8,7 @@ import com.mongodb.MongoClientSettings; import com.mongodb.ReadPreference; import com.mongodb.client.MongoClient; -import eu.europeana.metis.mongo.connection.MongoClientProvider; -import eu.europeana.metis.mongo.connection.MongoProperties; +import com.mongodb.connection.ConnectionPoolSettings; import eu.europeana.metis.mongo.embedded.EmbeddedLocalhostMongo; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -40,14 +39,26 @@ static void tearDown() { embeddedLocalhostMongo.stop(); } - @Test - void getDefaultClientSettingsBuilder() { - MongoClientSettings.Builder actual = MongoClientProvider.getDefaultClientSettingsBuilder(); + private static MongoProperties getMongoProperties() { + final String mongoHost = embeddedLocalhostMongo.getMongoHost(); + final int mongoPort = embeddedLocalhostMongo.getMongoPort(); + final MongoProperties mongoProperties = new MongoProperties<>( + IllegalArgumentException::new); + mongoProperties.setMongoHosts(new String[]{mongoHost}, new int[]{mongoPort}); + mongoProperties.setMongoCredentials("user", "wachtwoord", "authenticationDB"); + mongoProperties.setApplicationName(DATABASE_NAME); + mongoProperties.setMaxConnectionPoolSize(10); + return mongoProperties; + } - assertFalse(actual.build().getRetryWrites()); - assertEquals(ReadPreference.secondaryPreferred(), actual.build().getReadPreference()); - assertEquals("Europeana Application Suite", actual.build().getApplicationName()); - assertEquals(30_000, actual.build().getConnectionPoolSettings().getMaxConnectionIdleTime(TimeUnit.MILLISECONDS)); + @Test + void getClientSettingsBuilder() { + final MongoClientSettings mongoClientSettings = MongoClientProvider.getDefaultClientSettingsBuilder().build(); + assertFalse(mongoClientSettings.getRetryWrites()); + assertEquals(ReadPreference.secondaryPreferred(), mongoClientSettings.getReadPreference()); + assertEquals("Europeana Application Suite", mongoClientSettings.getApplicationName()); + assertEquals(30_000, mongoClientSettings.getConnectionPoolSettings().getMaxConnectionIdleTime(TimeUnit.MILLISECONDS)); + assertEquals(20, mongoClientSettings.getConnectionPoolSettings().getMaxSize()); } @Test @@ -87,14 +98,10 @@ void createMongoClient() { assertTrue(mongoClient instanceof MongoClient); } - private static MongoProperties getMongoProperties() { - final String mongoHost = embeddedLocalhostMongo.getMongoHost(); - final int mongoPort = embeddedLocalhostMongo.getMongoPort(); - final MongoProperties mongoProperties = new MongoProperties<>( - IllegalArgumentException::new); - mongoProperties.setMongoHosts(new String[]{mongoHost}, new int[]{mongoPort}); - mongoProperties.setMongoCredentials("user","wachtwoord","authenticationDB"); - mongoProperties.setApplicationName(DATABASE_NAME); - return mongoProperties; + @Test + void createConnectionPoolSettings() { + final ConnectionPoolSettings connectionPoolSettings = MongoClientProvider.createConnectionPoolSettings(10); + assertEquals(30_000, connectionPoolSettings.getMaxConnectionIdleTime(TimeUnit.MILLISECONDS)); + assertEquals(10, connectionPoolSettings.getMaxSize()); } } \ No newline at end of file diff --git a/metis-common/metis-common-mongo/src/test/java/eu/europeana/metis/mongo/connection/MongoPropertiesTest.java b/metis-common/metis-common-mongo/src/test/java/eu/europeana/metis/mongo/connection/MongoPropertiesTest.java index 4df4b749a..321694e9a 100644 --- a/metis-common/metis-common-mongo/src/test/java/eu/europeana/metis/mongo/connection/MongoPropertiesTest.java +++ b/metis-common/metis-common-mongo/src/test/java/eu/europeana/metis/mongo/connection/MongoPropertiesTest.java @@ -5,7 +5,6 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import eu.europeana.metis.mongo.connection.MongoProperties; import eu.europeana.metis.mongo.connection.MongoProperties.ReadPreferenceValue; import java.net.InetSocketAddress; import org.junit.jupiter.api.Test; @@ -33,6 +32,8 @@ void setAllProperties() throws Exception { "testAplication"); assertMongoProperties(mongoProperties); + mongoProperties.setMaxConnectionPoolSize(10); + assertEquals(10, mongoProperties.getMaxConnectionPoolSize()); } @Test diff --git a/metis-common/metis-common-network/pom.xml b/metis-common/metis-common-network/pom.xml index 4184a069e..276bc1c2e 100644 --- a/metis-common/metis-common-network/pom.xml +++ b/metis-common/metis-common-network/pom.xml @@ -4,7 +4,7 @@ metis-common eu.europeana.metis - 6 + 7 metis-common-network @@ -52,11 +52,6 @@ org.mockito mockito-core - - org.mockito - mockito-inline - test - org.glassfish.jersey.core jersey-common diff --git a/metis-common/metis-common-network/src/main/java/eu/europeana/metis/network/AbstractHttpClient.java b/metis-common/metis-common-network/src/main/java/eu/europeana/metis/network/AbstractHttpClient.java index 876548abf..e30d4144f 100644 --- a/metis-common/metis-common-network/src/main/java/eu/europeana/metis/network/AbstractHttpClient.java +++ b/metis-common/metis-common-network/src/main/java/eu/europeana/metis/network/AbstractHttpClient.java @@ -135,8 +135,8 @@ public R download(I link) throws IOException { public R download(I link, Map requestHeaders) throws IOException { // Set up the connection. - final String resourceUlr = getResourceUrl(link); - final HttpGet httpGet = new HttpGet(resourceUlr); + final String resourceUrl = getResourceUrl(link); + final HttpGet httpGet = new HttpGet(resourceUrl); requestHeaders.forEach(httpGet::setHeader); final HttpClientContext context = HttpClientContext.create(); @@ -146,7 +146,7 @@ public R download(I link, Map requestHeaders) throws IOException public void run() { synchronized (httpGet) { if (httpGet.cancel()) { - LOGGER.info("Aborting request due to time limit: {}.", resourceUlr); + LOGGER.info("Aborting request due to time limit: {}.", resourceUrl); } } } @@ -166,8 +166,8 @@ public void run() { final HttpEntity responseEntity = performThrowingFunction(responseObject, response -> { final int status = response.getCode(); if (!httpCallIsSuccessful(status)) { - throw new IOException("Download failed of resource " + resourceUlr + ". Status code " + - status + " (message: " + response.getReasonPhrase() + ")."); + throw new IOException("Download failed of resource " + resourceUrl + ". Status code " + + status + " (message: " + response.getReasonPhrase() + ")."); } return response.getEntity(); }); @@ -210,7 +210,7 @@ public void run() { // Cancel the request to stop downloading. synchronized (httpGet) { if (httpGet.cancel()) { - LOGGER.debug("Aborting request after all processing is completed: {}.", resourceUlr); + LOGGER.debug("Aborting request after all processing is completed: {}.", resourceUrl); } } diff --git a/metis-common/metis-common-network/src/main/java/eu/europeana/metis/network/NetworkUtil.java b/metis-common/metis-common-network/src/main/java/eu/europeana/metis/network/NetworkUtil.java index efce93017..ee3ac060d 100644 --- a/metis-common/metis-common-network/src/main/java/eu/europeana/metis/network/NetworkUtil.java +++ b/metis-common/metis-common-network/src/main/java/eu/europeana/metis/network/NetworkUtil.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.net.InetAddress; import java.net.ServerSocket; +import javax.net.ServerSocketFactory; import javax.net.ssl.SSLServerSocketFactory; /** @@ -11,26 +12,30 @@ * @author Simon Tzanakis (Simon.Tzanakis@europeana.eu) * @since 2017-02-24 */ -public final class NetworkUtil { +public class NetworkUtil { private static final int BACKLOG = 100; - private NetworkUtil() { - } - /** - * This method can be used in JUnit tests to get a random available port on localhost to run a - * service. It should not be used for normal operation, otherwise ssl checks should be followed to - * avoid man-in-the-middle attacks. + * This method can be used in JUnit tests to get a random available port on localhost to run a service. It should not be used + * for normal operation, otherwise ssl checks should be followed to avoid man-in-the-middle attacks. * * @return the available port number * @throws IOException if the specified localhost is not available */ - public static int getAvailableLocalPort() throws IOException { - ServerSocket s = SSLServerSocketFactory.getDefault() - .createServerSocket(0, BACKLOG, InetAddress.getByName("localhost")); + public int getAvailableLocalPort() throws IOException { + ServerSocket s = getServerSocketFactory().createServerSocket(0, BACKLOG, InetAddress.getByName("localhost")); int localPort = s.getLocalPort(); s.close(); return localPort; } + + /** + * Get a server socket factory. + * + * @return the server socket factory + */ + ServerSocketFactory getServerSocketFactory() { + return SSLServerSocketFactory.getDefault(); + } } diff --git a/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/ExternalRequestUtilTest.java b/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/ExternalRequestUtilTest.java index 8a06425b5..eb254c8f0 100644 --- a/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/ExternalRequestUtilTest.java +++ b/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/ExternalRequestUtilTest.java @@ -64,25 +64,19 @@ void testRetryableExternalRequestWithMap() { @Test void testRetryableExternalRequestThrowsExceptionOutOfSpecifiedMap() { - assertThrows(RuntimeException.class, () -> { - ExternalRequestUtil.retryableExternalRequest( - () -> { - throw new RuntimeException(new ClassNotFoundException("Class pointer test exception")); - }, - UNMODIFIABLE_MAP_WITH_TEST_EXCEPTIONS); - }); + assertThrows(RuntimeException.class, () -> ExternalRequestUtil.retryableExternalRequest( + () -> { + throw new RuntimeException(new ClassNotFoundException("Class pointer test exception")); + }, UNMODIFIABLE_MAP_WITH_TEST_EXCEPTIONS)); } @Disabled("TODO: MET-4255 Improve execution time") @Test void testRetryableExternalRequestThrowsException() { - assertThrows(RuntimeException.class, () -> { - ExternalRequestUtil.retryableExternalRequest( - () -> { - throw new RuntimeException(new ClassNotFoundException("Class pointer test exception")); - }, - null); - }); + assertThrows(RuntimeException.class, () -> ExternalRequestUtil.retryableExternalRequest( + () -> { + throw new RuntimeException(new ClassNotFoundException("Class pointer test exception")); + }, null)); } @Test diff --git a/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/NetworkUtilTest.java b/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/NetworkUtilTest.java index 3d47f33a7..ae091ef99 100644 --- a/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/NetworkUtilTest.java +++ b/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/NetworkUtilTest.java @@ -3,16 +3,13 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import java.io.IOException; import java.net.InetAddress; import javax.net.ServerSocketFactory; -import javax.net.ssl.SSLServerSocketFactory; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import org.mockito.MockedStatic; /** * Unit test for {@link NetworkUtil} @@ -24,21 +21,17 @@ class NetworkUtilTest { @Test void getAvailableLocalPort() throws IOException { - int availableLocalPort = NetworkUtil.getAvailableLocalPort(); - + int availableLocalPort = new NetworkUtil().getAvailableLocalPort(); assertTrue(availableLocalPort > 0); } - @Disabled("TODO: MET-4250 Handle MockMaker in Jenkins") @Test void getAvailableLocalPortWithException() throws IOException { final int BACKLOG = 100; - try (MockedStatic sslServerSocketFactory = mockStatic(SSLServerSocketFactory.class)) { - ServerSocketFactory serverSocketFactory = mock(ServerSocketFactory.class); - sslServerSocketFactory.when(SSLServerSocketFactory::getDefault).thenReturn(serverSocketFactory); - when(serverSocketFactory.createServerSocket(0, BACKLOG, InetAddress.getByName("localhost"))).thenThrow(IOException.class); - - assertThrows(IOException.class, () -> NetworkUtil.getAvailableLocalPort()); - } + final ServerSocketFactory sslServerSocketFactory = mock(ServerSocketFactory.class); + when(sslServerSocketFactory.createServerSocket(0, BACKLOG, InetAddress.getByName("localhost"))).thenThrow(IOException.class); + final NetworkUtil networkUtil = spy(NetworkUtil.class); + when(networkUtil.getServerSocketFactory()).thenReturn(sslServerSocketFactory); + assertThrows(IOException.class, networkUtil::getAvailableLocalPort); } } \ No newline at end of file diff --git a/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/StringHttpClientTest.java b/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/StringHttpClientTest.java index c58837c58..4181574f7 100644 --- a/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/StringHttpClientTest.java +++ b/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/StringHttpClientTest.java @@ -10,7 +10,6 @@ import java.io.ByteArrayInputStream; import java.io.Closeable; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; @@ -19,7 +18,6 @@ import org.apache.hc.core5.http.HttpEntity; import org.apache.hc.core5.http.io.entity.BasicHttpEntity; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; /** @@ -56,8 +54,7 @@ void getResourceUrlWithException() { void createResult() throws URISyntaxException, IOException { List closeables = new ArrayList<>(); HttpEntity responseEntity = new BasicHttpEntity(new ByteArrayInputStream("content".getBytes()), ContentType.TEXT_PLAIN); - final ContentRetriever contentRetriever = ContentRetriever.forNonCloseableContent( - responseEntity == null ? InputStream::nullInputStream : responseEntity::getContent, + final ContentRetriever contentRetriever = ContentRetriever.forNonCloseableContent(responseEntity::getContent, closeables::add); StringContent actualContent = stringHttpClient.createResult(new URI("/resource/provided"), new URI("/resource/actual"), @@ -65,18 +62,17 @@ void createResult() throws URISyntaxException, IOException { assertEquals("content", actualContent.getContent()); assertEquals("text/plain", actualContent.getContentType()); + assertEquals(1, closeables.size()); } - @Disabled("TODO: MET-4250 Handle MockMaker in Jenkins") @Test void createResultWithException() throws IOException { final ContentRetriever contentRetriever = mock(ContentRetriever.class); when(contentRetriever.getContent()).thenThrow(IOException.class); - assertThrows(IOException.class, () -> { - stringHttpClient.createResult(new URI("/resource/provided"), new URI("/resource/actual"), - "text/plain", 7L, contentRetriever); - }); + assertThrows(IOException.class, + () -> stringHttpClient.createResult(new URI("/resource/provided"), new URI("/resource/actual"), + "text/plain", 7L, contentRetriever)); } @Test diff --git a/metis-common/metis-common-solr/pom.xml b/metis-common/metis-common-solr/pom.xml index 041ed3dcc..54283c869 100644 --- a/metis-common/metis-common-solr/pom.xml +++ b/metis-common/metis-common-solr/pom.xml @@ -4,7 +4,7 @@ metis-common eu.europeana.metis - 6 + 7 metis-common-solr diff --git a/metis-common/metis-common-utils/pom.xml b/metis-common/metis-common-utils/pom.xml index ae8ccaa39..cab5d2ff8 100644 --- a/metis-common/metis-common-utils/pom.xml +++ b/metis-common/metis-common-utils/pom.xml @@ -4,7 +4,7 @@ metis-common eu.europeana.metis - 6 + 7 metis-common-utils diff --git a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/GeoUriWGS84Parser.java b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/GeoUriWGS84Parser.java new file mode 100644 index 000000000..9bed28c0c --- /dev/null +++ b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/GeoUriWGS84Parser.java @@ -0,0 +1,219 @@ +package eu.europeana.metis.utils; + +import eu.europeana.metis.exception.BadContentException; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * Contains functionality to parse and validate geo uri + */ +public final class GeoUriWGS84Parser { + + private static final String DECIMAL_POINT_REGEX = "(?:\\.\\d+)?"; + private static final String ZEROES_DECIMAL_POINT_REGEX = "(?:\\.0+)?"; + private static final String LATITUDE_REGEX = + "^[+-]?(?:90" + ZEROES_DECIMAL_POINT_REGEX + "|(?:\\d|[1-8]\\d)" + DECIMAL_POINT_REGEX + ")$"; + private static final Pattern LATITUDE_PATTERN = Pattern.compile(LATITUDE_REGEX); + private static final String LONGITUDE_REGEX = + "^[+-]?(?:180" + ZEROES_DECIMAL_POINT_REGEX + "|(?:\\d|[1-9]\\d|1[0-7]\\d)" + DECIMAL_POINT_REGEX + ")$"; + private static final Pattern LONGITUDE_PATTERN = Pattern.compile(LONGITUDE_REGEX); + private static final String ALTITUDE_REGEX = "^[+-]?\\d+" + DECIMAL_POINT_REGEX + "$"; + private static final Pattern ALTITUDE_PATTERN = Pattern.compile(ALTITUDE_REGEX); + private static final String CRS_WGS_84 = "wgs84"; + private static final int MAX_NUMBER_COORDINATES = 3; + private static final int MAX_DECIMAL_POINTS_TO_KEEP = 7; + + private GeoUriWGS84Parser() { + } + + /** + * Parse a provided geo uri in wgs84 coordinate reference system (CRS) and validate its contents. + *

    The parsing of the string follows closely but not exhaustively the specification located at + * https://datatracker.ietf.org/doc/html/rfc5870

    + *

    The checks that are performed to the provided string are as follows: + *

      + *
    • There should not be any spaces
      • + *
    • It should start with "geo:"
      • + *
    • There should be at least one part after the scheme and that should be the coordinates
      • + *
    • If crs parameter is present it should be "wgs84"
      • + *
    • The "u" parameter should be just after crs if crs is present or just after the coordinates
      • + *
    • The coordinates should have 2 or 3 dimensions
      • + *
    • The coordinates should be of valid structure and valid range
      • + *
    • The coordinates if they have decimal points they will be truncated after 7th point
      • + *
    + *

    + * + * @param geoUriString the geo uri string + * @return the geo coordinates, null will never be returned + * @throws BadContentException if the geo uri parsing encountered an error + */ + public static GeoCoordinates parse(String geoUriString) throws BadContentException { + final String[] geoUriParts = validateGeoUriAndGetParts(geoUriString); + + //Finally, check the coordinates part and validate + return validateGeoCoordinatesAndGet(geoUriParts[0]); + } + + private static String[] validateGeoUriAndGetParts(String geoUriString) throws BadContentException { + //Validate that there aren't any space characters in the URI + if (!geoUriString.matches("^\\S+$")) { + throw new BadContentException("URI cannot have spaces"); + } + //Validate geo URI + if (!geoUriString.matches("^geo:.*$")) { + throw new BadContentException("Invalid scheme value"); + } + + final String[] schemeAndParts = geoUriString.split(":"); + if (schemeAndParts.length <= 1) { + throw new BadContentException("There are no parts in the geo URI"); + } + + //Find all parts + final String[] geoUriParts = schemeAndParts[1].split(";"); + //Must be at least one part available + if (geoUriParts.length < 1) { + throw new BadContentException("Invalid geo uri parts length"); + } + + //Find all other parameters + final LinkedList geoUriParameters = Arrays.stream(geoUriParts, 1, geoUriParts.length).map(s -> { + final String[] split = s.split("="); + return new GeoUriParameter(split[0], split[1]); + }).collect(Collectors.toCollection(LinkedList::new)); + + //If crs present, it must be the exact first after the dimensions. If not present then there is a default + String crs = CRS_WGS_84; + for (int i = 0; i < geoUriParameters.size(); i++) { + if ("crs".equalsIgnoreCase(geoUriParameters.get(i).getName())) { + crs = geoUriParameters.get(i).getValue(); + if (i != 0) { + throw new BadContentException("Invalid geo uri 'crs' parameter position"); + } + } + if ("u".equalsIgnoreCase(geoUriParameters.get(i).getName()) && i > 1) { + throw new BadContentException("Invalid geo uri 'u' parameter position"); + } + } + //Validate value of crs + if (!CRS_WGS_84.equalsIgnoreCase(crs)) { + throw new BadContentException(String.format("Crs parameter value is not %s", CRS_WGS_84)); + } + return geoUriParts; + } + + /** + * Generate a geo coordinates from a geoUriPart string. + *

    The provided string is validated against: + *

      + *
    • the total coordinates available
      • + *
    • the validity of each number and its range
      • + *
    • the convertibility to a {@link Double}
      • + *
    + * The decimal points are also truncated up to a maximum allowed. + *

    + * + * @param geoUriPart the string that should contain the coordinates + * @return the geo coordinates + * @throws BadContentException if the geo coordinates were not valid + */ + private static GeoCoordinates validateGeoCoordinatesAndGet(String geoUriPart) throws BadContentException { + final String[] coordinates = geoUriPart.split(","); + if (coordinates.length < 2 || coordinates.length > MAX_NUMBER_COORDINATES) { + throw new BadContentException("Coordinates are not of valid length"); + } + final Matcher latitudeMatcher = LATITUDE_PATTERN.matcher(coordinates[0]); + final Matcher longitudeMatcher = LONGITUDE_PATTERN.matcher(coordinates[1]); + final GeoCoordinates geoCoordinates; + if (latitudeMatcher.matches() && longitudeMatcher.matches()) { + Double altitude = null; + if (coordinates.length == MAX_NUMBER_COORDINATES) { + final Matcher altitudeMatcher = ALTITUDE_PATTERN.matcher(coordinates[2]); + if (altitudeMatcher.matches()) { + altitude = Double.parseDouble(truncateDecimalPoints(altitudeMatcher.group(0))); + } + } + geoCoordinates = new GeoCoordinates( + Double.parseDouble(truncateDecimalPoints(latitudeMatcher.group(0))), + Double.parseDouble(truncateDecimalPoints(longitudeMatcher.group(0))), altitude); + } else { + throw new BadContentException("Coordinates are invalid"); + } + return geoCoordinates; + } + + private static String truncateDecimalPoints(String decimalNumber) { + final String[] decimalNumberParts = decimalNumber.split("\\."); + final StringBuilder decimalNumberTruncated = new StringBuilder(); + if (decimalNumberParts.length >= 1) { + decimalNumberTruncated.append(decimalNumberParts[0]); + } + if (decimalNumberParts.length > 1) { + decimalNumberTruncated.append("."); + decimalNumberTruncated.append(decimalNumberParts[1], 0, + Math.min(decimalNumberParts[1].length(), MAX_DECIMAL_POINTS_TO_KEEP)); + } + return decimalNumberTruncated.toString(); + } + + /** + * Class containing geo coordinates (latitude, longitude) + */ + public static class GeoCoordinates { + + private final Double latitude; + private final Double longitude; + private final Double altitude; + + /** + * Constructor with required parameters + * + * @param latitude the latitude + * @param longitude the longitude + * @param altitude the altitude + */ + public GeoCoordinates(Double latitude, Double longitude, Double altitude) { + this.latitude = latitude; + this.longitude = longitude; + this.altitude = altitude; + } + + public Double getLatitude() { + return latitude; + } + + public Double getLongitude() { + return longitude; + } + + public Double getAltitude() { + return altitude; + } + } + + /** + * Class wrapping the name and value of geo uri parameters. + */ + private static class GeoUriParameter { + + private final String name; + private final String value; + + public GeoUriParameter(String name, String value) { + this.name = name; + this.value = value; + } + + public String getName() { + return name; + } + + public String getValue() { + return value; + } + } + +} diff --git a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/RestEndpoints.java b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/RestEndpoints.java index 1b7d2731b..103c05c3b 100644 --- a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/RestEndpoints.java +++ b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/RestEndpoints.java @@ -48,12 +48,16 @@ public final class RestEndpoints { public static final String ORCHESTRATOR_WORKFLOWS_SCHEDULE = "/orchestrator/workflows/schedule"; public static final String ORCHESTRATOR_WORKFLOWS_SCHEDULE_DATASETID = "/orchestrator/workflows/schedule/{datasetId}"; public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_EXECUTIONID = "/orchestrator/workflows/executions/{executionId}"; - public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_EXECUTIONID_PLUGINS_DATA_AVAILABILITY = "/orchestrator/workflows/executions/{executionId}/plugins/data-availability"; + public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_EXECUTIONID_PLUGINS_DATA_AVAILABILITY + = "/orchestrator/workflows/executions/{executionId}/plugins/data-availability"; public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_DATASET_DATASETID = "/orchestrator/workflows/executions/dataset/{datasetId}"; - public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_DATASET_DATASETID_ALLOWED_INCREMENTAL = "/orchestrator/workflows/executions/dataset/{datasetId}/allowed_incremental"; - public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_DATASET_DATASETID_ALLOWED_PLUGIN = "/orchestrator/workflows/executions/dataset/{datasetId}/allowed_plugin"; + public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_DATASET_DATASETID_ALLOWED_INCREMENTAL + = "/orchestrator/workflows/executions/dataset/{datasetId}/allowed_incremental"; + public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_DATASET_DATASETID_ALLOWED_PLUGIN + = "/orchestrator/workflows/executions/dataset/{datasetId}/allowed_plugin"; public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_DATASET_DATASETID_HISTORY = "/orchestrator/workflows/executions/dataset/{datasetId}/history"; - public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_DATASET_DATASETID_INFORMATION = "/orchestrator/workflows/executions/dataset/{datasetId}/information"; + public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_DATASET_DATASETID_INFORMATION + = "/orchestrator/workflows/executions/dataset/{datasetId}/information"; public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS = "/orchestrator/workflows/executions"; public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_OVERVIEW = "/orchestrator/workflows/executions/overview"; public static final String ORCHESTRATOR_WORKFLOWS_EVOLUTION = "/orchestrator/workflows/evolution/{workflowExecutionId}/{pluginType}"; @@ -69,6 +73,9 @@ public final class RestEndpoints { public static final String DEREFERENCE = "/dereference"; public static final String VOCABULARIES = "/vocabularies"; public static final String CACHE_EMPTY = "/cache"; + public static final String CACHE_EMPTY_VOCABULARY = "/cache/vocabulary"; + public static final String CACHE_EMPTY_RESOURCE = "/cache/resource"; + public static final String CACHE_EMPTY_XML = "/cache/emptyxml"; public static final String LOAD_VOCABULARIES = "/load_vocabularies"; /* METIS ENRICHMENT Endpoint */ @@ -91,8 +98,7 @@ private RestEndpoints() { } /** - * Resolves an endpoint with parameters wrapped around "{" and "}" by providing the endpoint and - * all the required parameters. + * Resolves an endpoint with parameters wrapped around "{" and "}" by providing the endpoint and all the required parameters. * * @param endpoint the endpoint to resolve * @param params all the parameters specified diff --git a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/TempFileUtils.java b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/TempFileUtils.java new file mode 100644 index 000000000..feb3373de --- /dev/null +++ b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/TempFileUtils.java @@ -0,0 +1,119 @@ +package eu.europeana.metis.utils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.EnumSet; +import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * File utilities class + */ +public final class TempFileUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(TempFileUtils.class); + private static final EnumSet OWNER_PERMISSIONS_ONLY_SET = EnumSet.of(PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE); + private static final FileAttribute> OWNER_PERMISSIONS_ONLY_FILE_ATTRIBUTE = PosixFilePermissions.asFileAttribute( + OWNER_PERMISSIONS_ONLY_SET); + public static final String PNG_FILE_EXTENSION = ".png"; + public static final String JPEG_FILE_EXTENSION = ".jpeg"; + + private TempFileUtils() { + //Private constructor + } + + /** + * Creates a secure temporary file(owner permissions only) for posix and other file systems. + *

    This method is not responsible of removing the temporary file. + * An implementation that uses this method should delete the temp files by itself.

    + * + * @param prefix the prefix, (e.g. the class simple name that generates the temp file) + * @param suffix the suffix + * @return the secure temporary file + * @throws IOException if the file failed to be created + */ + public static Path createSecureTempFile(String prefix, String suffix) throws IOException { + //Set permissions only to owner, posix style + final Path secureTempFile = Files.createTempFile(prefix, suffix, OWNER_PERMISSIONS_ONLY_FILE_ATTRIBUTE); + //Set again for non posix systems + setPosixIndependentOwnerOnlyFilePermissions(secureTempFile); + + return secureTempFile; + } + + /** + * Creates a secure temporary file(owner permissions only) for posix and other file systems. + *

    + * This is equivalent to calling {@link #createSecureTempFile(String, String)} and in addition it declares that it will remove + * the temporary file with {@link File#deleteOnExit()}. + *

    + * + *

    CAUTION: This method can have a memory impact if too many files are created, and that is because + * {@link File#deleteOnExit()} keeps an in memory cache of the file paths. If possible prefer the use of + * {@link #createSecureTempFile(String, String)} and make your implementation remove the temporary files created + * explicitly.

    + * + * @param prefix the prefix + * @param suffix the suffix + * @return the secure temporary file + * @throws IOException if the file failed to be created + */ + @SuppressWarnings("java:S2308") //Delete on exit is intended here and javadoc warns the user + public static Path createSecureTempFileDeleteOnExit(String prefix, String suffix) throws IOException { + final Path secureTempFile = createSecureTempFile(prefix, suffix); + secureTempFile.toFile().deleteOnExit(); + return secureTempFile; + } + + /** + * Creates a secure temporary directory(owner permissions only) with the {@code directoryPrefix} specified and then creates a + * secure temporary file(owner permissions only) inside that directory with the {@code prefix} and {@code suffix} specified. + * + * @param directoryPrefix the directory prefix + * @param prefix the file prefix + * @param suffix the file suffix + * @return the secure temporary file in the newly created secure temporary directory + * @throws IOException if the directory or file failed to be created + */ + public static Path createSecureTempDirectoryAndFile(String directoryPrefix, String prefix, String suffix) throws IOException { + Path tempSecureParentDir = createSecureTempDirectory(directoryPrefix); + //Set permissions only to owner, posix style + final Path secureTempFile = Files.createTempFile(tempSecureParentDir, prefix, suffix, OWNER_PERMISSIONS_ONLY_FILE_ATTRIBUTE); + //Set again for non posix systems + setPosixIndependentOwnerOnlyFilePermissions(secureTempFile); + + return secureTempFile; + } + + /** + * Creates a secure temporary directory(owner permissions only) with the {@code prefix} specified. + * + * @param prefix the prefix + * @return the secure temporary directory + * @throws IOException if the directory failed to be created + */ + public static Path createSecureTempDirectory(String prefix) throws IOException { + //Set permissions only to owner, posix style + final Path secureTempFile = Files.createTempDirectory(prefix, OWNER_PERMISSIONS_ONLY_FILE_ATTRIBUTE); + //Set again for non posix systems + setPosixIndependentOwnerOnlyFilePermissions(secureTempFile); + + return secureTempFile; + } + + private static void setPosixIndependentOwnerOnlyFilePermissions(Path path) { + File file = path.toFile(); + //Set again for non posix systems + if (!(file.setReadable(true, true) && file.setWritable(true, true) && file.setExecutable(true, true))) { + LOGGER.debug("Setting permissions failed on file {}", file.getAbsolutePath()); + } + } + +} diff --git a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/ZipFileReader.java b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/ZipFileReader.java index 0b610a0ad..f7c711fed 100644 --- a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/ZipFileReader.java +++ b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/ZipFileReader.java @@ -1,5 +1,7 @@ package eu.europeana.metis.utils; +import static eu.europeana.metis.utils.TempFileUtils.createSecureTempFile; + import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; @@ -8,7 +10,6 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.UUID; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import org.apache.commons.io.FileUtils; @@ -18,9 +19,8 @@ /** * This class provides the functionality of reading zip files. - * - * @author jochen * + * @author jochen */ public class ZipFileReader { @@ -37,33 +37,30 @@ public ZipFileReader() { } /** - * This method extracts all files from a ZIP file and returns them as strings. This method only - * considers files in the main directory. This method creates (and then removes) a temporary file. + * This method extracts all files from a ZIP file and returns them as strings. This method only considers files in the main + * directory. This method creates (and then removes) a temporary file. * - * @param providedZipFile Input stream containing the zip file. This method is not responsible for - * closing the stream. + * @param providedZipFile Input stream containing the zip file. This method is not responsible for closing the stream. * @return A list of records. * @throws IOException In case of problems with the temporary file or with reading the zip file. */ public List getRecordsFromZipFile(InputStream providedZipFile) throws IOException { - try (final ZipFile zipFile = createTempZipFile(providedZipFile)) { + try (final ZipFile zipFile = createInMemoryZipFileObject(providedZipFile)) { return getRecordsFromZipFile(zipFile); } } /** - * This method extracts all files from a ZIP file and returns them as byte arrays. This method - * only considers files in the main directory. This method creates (and then removes) a temporary - * file. + * This method extracts all files from a ZIP file and returns them as byte arrays. This method only considers files in the main + * directory. This method creates (and then removes) a temporary file. * - * @param providedZipFile Input stream containing the zip file. This method is not responsible for - * closing the stream. + * @param providedZipFile Input stream containing the zip file. This method is not responsible for closing the stream. * @return A list of records. * @throws IOException In case of problems with the temporary file or with reading the zip file. */ public List getContentFromZipFile(InputStream providedZipFile) - throws IOException { - try (final ZipFile zipFile = createTempZipFile(providedZipFile)) { + throws IOException { + try (final ZipFile zipFile = createInMemoryZipFileObject(providedZipFile)) { final List streams = getContentFromZipFile(zipFile); final List result = new ArrayList<>(streams.size()); for (InputStream stream : streams) { @@ -73,9 +70,8 @@ public List getContentFromZipFile(InputStream providedZipF } } - private ZipFile createTempZipFile(InputStream content) throws IOException { - final String prefix = UUID.randomUUID().toString(); - final File tempFile = File.createTempFile(prefix, ".zip"); + private ZipFile createInMemoryZipFileObject(InputStream content) throws IOException { + final File tempFile = createSecureTempFile(ZipFileReader.class.getSimpleName(), ".zip").toFile(); FileUtils.copyInputStreamToFile(content, tempFile); LOGGER.info("Temp file: {} created.", tempFile); return new ZipFile(tempFile, ZipFile.OPEN_READ | ZipFile.OPEN_DELETE); diff --git a/metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/GeoUriWGS84ParserTest.java b/metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/GeoUriWGS84ParserTest.java new file mode 100644 index 000000000..048735b52 --- /dev/null +++ b/metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/GeoUriWGS84ParserTest.java @@ -0,0 +1,81 @@ +package eu.europeana.metis.utils; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import eu.europeana.metis.exception.BadContentException; +import eu.europeana.metis.utils.GeoUriWGS84Parser.GeoCoordinates; +import org.junit.jupiter.api.Test; + +class GeoUriWGS84ParserTest { + + @Test + void parse_invalid() { + + //URI cannot have spaces + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo: 37.786971,-122.399677")); + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:37.786971,-122.399677; u=35")); + + //Non geo + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("test:")); + + //URI cannot be without dimensions + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:")); + //URI must have at least one part + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:;")); + + //Validate order of crs and u parameters + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:37.786971,-122.399677;u=35;crs=wgs84")); + assertThrows(BadContentException.class, + () -> GeoUriWGS84Parser.parse("geo:37.786971,-122.399677;crs=wgs84;parameter1=value1;u=35")); + assertThrows(BadContentException.class, + () -> GeoUriWGS84Parser.parse("geo:37.786971,-122.399677;parameter1=value1;crs=wgs84;u=35")); + + //Validate crs value + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:37.786971,-122.399677;crs=Moon-2011;u=35")); + + //Coordinates must be present and of correct length + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:;crs=wgs84")); + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:37.786971,;crs=wgs84")); + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:37.786971;crs=wgs84")); + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:37.786971,100,100,10;crs=wgs84")); + //Invalid coordinate + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:test,-122.399677;crs=wgs84")); + //Invalid range coordinates + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:-100,200;crs=wgs84")); + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:578991.875,578991.875")); + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:-90.123456,100")); + + + } + + @Test + void parse_valid() throws Exception { + assertDoesNotThrow(() -> GeoUriWGS84Parser.parse("geo:37.786971,-122.399677;crs=wgs84;u=35")); + assertDoesNotThrow(() -> GeoUriWGS84Parser.parse("geo:37.786971,-122.399677;u=35")); + assertDoesNotThrow(() -> GeoUriWGS84Parser.parse("geo:37.786971,-122.399677;crs=wgs84;u=35;parameter1=value1")); + assertDoesNotThrow(() -> GeoUriWGS84Parser.parse("geo:37.786971,-122.399677;u=35;parameter1=value1")); + + assertDoesNotThrow(() -> GeoUriWGS84Parser.parse("geo:37.786971,-122.399677")); + assertDoesNotThrow(() -> GeoUriWGS84Parser.parse("geo:37.786971,-122.399677,10")); + assertDoesNotThrow(() -> GeoUriWGS84Parser.parse("geo:37.1234567,-122.1234567,10")); + assertDoesNotThrow(() -> GeoUriWGS84Parser.parse("geo:37,-122")); + + final GeoCoordinates geoCoordinates = GeoUriWGS84Parser.parse("geo:37.786971,-122.399677"); + assertEquals(Double.parseDouble("37.786971"), geoCoordinates.getLatitude()); + assertEquals(Double.parseDouble("-122.399677"), geoCoordinates.getLongitude()); + + final GeoCoordinates geoCoordinatesWithAltitude = GeoUriWGS84Parser.parse("geo:37.786971,-122.399677,1000.500600"); + assertEquals(Double.parseDouble("37.786971"), geoCoordinatesWithAltitude.getLatitude()); + assertEquals(Double.parseDouble("-122.399677"), geoCoordinatesWithAltitude.getLongitude()); + assertEquals(Double.parseDouble("1000.500600"), geoCoordinatesWithAltitude.getAltitude()); + + //Should truncate the extra decimal points + final GeoCoordinates geoCoordinatesWithLongDecimalPoints = GeoUriWGS84Parser.parse( + "geo:40.123456789,45.123456789,1000.123456789"); + assertEquals(Double.parseDouble("40.1234567"), geoCoordinatesWithLongDecimalPoints.getLatitude()); + assertEquals(Double.parseDouble("45.1234567"), geoCoordinatesWithLongDecimalPoints.getLongitude()); + assertEquals(Double.parseDouble("1000.1234567"), geoCoordinatesWithLongDecimalPoints.getAltitude()); + } +} \ No newline at end of file diff --git a/metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/TempFileUtilsTest.java b/metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/TempFileUtilsTest.java new file mode 100644 index 000000000..8cb2612dc --- /dev/null +++ b/metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/TempFileUtilsTest.java @@ -0,0 +1,86 @@ +package eu.europeana.metis.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermission; +import java.util.EnumSet; +import java.util.LinkedHashSet; +import java.util.Set; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; + +class TempFileUtilsTest { + + public static final boolean IS_POSIX = FileSystems.getDefault().supportedFileAttributeViews().contains("posix"); + + @Test + void createSecureTempFile() throws IOException { + final Path secureTempFile = TempFileUtils.createSecureTempFile("prefix", "suffix"); + assertFilePermissions(secureTempFile); + assertTrue(Files.deleteIfExists(secureTempFile)); + } + + @Test + void createSecureTempFileDeleteOnExit() + throws ClassNotFoundException, IOException, NoSuchFieldException, IllegalAccessException { + final Path secureTempFile = TempFileUtils.createSecureTempFileDeleteOnExit("prefix", "suffix"); + assertFilePermissions(secureTempFile); + + final LinkedHashSet filesForDeletion = getFileListMarkedForDeletion(); + assertTrue(filesForDeletion.contains(secureTempFile.toString())); + } + + private LinkedHashSet getFileListMarkedForDeletion() + throws ClassNotFoundException, NoSuchFieldException, IllegalAccessException { + //Check that deletion on exit is in place + final Class deleteOnExitHook = Class.forName("java.io.DeleteOnExitHook"); + final Field filesField = deleteOnExitHook.getDeclaredField("files"); + filesField.setAccessible(true); + return castList(filesField.get(null)); + } + + private LinkedHashSet castList(Object objectList) { + @SuppressWarnings("rawtypes") final LinkedHashSet linkedHashSet = (LinkedHashSet) objectList; + LinkedHashSet result = new LinkedHashSet<>(); + for (Object object : linkedHashSet) { + result.add((String) object); + } + return result; + } + + @Test + void createSecureTempDirectoryAndFile() throws IOException { + final Path secureTempDirectoryAndFile = TempFileUtils.createSecureTempDirectoryAndFile("directoryPrefix", "prefix", "suffix"); + final Path parent = secureTempDirectoryAndFile.getParent(); + assertFilePermissions(parent); + assertFilePermissions(secureTempDirectoryAndFile); + FileUtils.deleteDirectory(parent.toFile()); + } + + @Test + void createSecureTempDirectory() throws IOException { + final Path secureTempDirectory = TempFileUtils.createSecureTempDirectory("directoryPrefix"); + assertFilePermissions(secureTempDirectory); + FileUtils.deleteDirectory(secureTempDirectory.toFile()); + } + + private void assertFilePermissions(Path secureTempFile) throws IOException { + if (IS_POSIX) { + final Set posixFilePermissions = Files.getPosixFilePermissions(secureTempFile); + assertTrue(posixFilePermissions.containsAll( + EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE))); + //Check that permissions to others are denied + assertEquals(3, posixFilePermissions.size()); + } + + assertTrue(Files.isReadable(secureTempFile)); + assertTrue(Files.isWritable(secureTempFile)); + assertTrue(Files.isExecutable(secureTempFile)); + } +} \ No newline at end of file diff --git a/metis-common/metis-common-zoho/pom.xml b/metis-common/metis-common-zoho/pom.xml index b3ba39426..ddbf39195 100644 --- a/metis-common/metis-common-zoho/pom.xml +++ b/metis-common/metis-common-zoho/pom.xml @@ -4,7 +4,7 @@ metis-common eu.europeana.metis - 6 + 7 metis-common-zoho diff --git a/metis-common/metis-common-zoho/src/test/java/eu/europeana/metis/zoho/ZohoUtilsTest.java b/metis-common/metis-common-zoho/src/test/java/eu/europeana/metis/zoho/ZohoUtilsTest.java index 54ae0b2df..dc638df13 100644 --- a/metis-common/metis-common-zoho/src/test/java/eu/europeana/metis/zoho/ZohoUtilsTest.java +++ b/metis-common/metis-common-zoho/src/test/java/eu/europeana/metis/zoho/ZohoUtilsTest.java @@ -33,9 +33,7 @@ void stringListSupplier() { final Record recordOrganization = new Record(); final List expectedChoiceList = List.of("Organization1Role", "Organization2Role"); recordOrganization.addKeyValue(ZohoConstants.ORGANIZATION_ROLE_FIELD, - expectedChoiceList.stream() - .map(choice -> new Choice<>(choice)) - .collect(Collectors.toList())); + expectedChoiceList.stream().map(Choice::new).collect(Collectors.toList())); final List organizationRoleStringList = ZohoUtils.stringListSupplier( recordOrganization.getKeyValue(ZohoConstants.ORGANIZATION_ROLE_FIELD)); diff --git a/metis-common/pom.xml b/metis-common/pom.xml index 6e7f111a8..818e476d0 100644 --- a/metis-common/pom.xml +++ b/metis-common/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 6 + 7 metis-common pom diff --git a/metis-core/metis-core-common/pom.xml b/metis-core/metis-core-common/pom.xml index 612171884..7cbf13c58 100644 --- a/metis-core/metis-core-common/pom.xml +++ b/metis-core/metis-core-common/pom.xml @@ -4,7 +4,7 @@ metis-core eu.europeana.metis - 6 + 7 metis-core-common diff --git a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/common/Country.java b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/common/Country.java index 3301b436d..ac842da5a 100644 --- a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/common/Country.java +++ b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/common/Country.java @@ -1,5 +1,9 @@ package eu.europeana.metis.core.common; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + /** * Countries supported by METIS */ @@ -44,7 +48,7 @@ public enum Country { LIECHTENSTEIN("Liechtenstein", "LI"), LITHUANIA("Lithuania", "LT"), LUXEMBOURG("Luxembourg", "LU"), - MACEDONIA("Macedonia", "MK"), + NORTH_MACEDONIA("North Macedonia", "MK"), MALTA("Malta", "MT"), MOLDOVA("Moldova", "MD"), MONACO("Monaco", "MC"), @@ -115,4 +119,15 @@ public static Country getCountryFromIsoCode(String isoCode) { } return null; } + + /** + * Provides the countries sorted by the {@link #getName()} field + * + * @return the list of countries sorted + */ + public static List getCountryListSortedByName() { + List countries = Arrays.asList(Country.values()); + countries.sort(Comparator.comparing(Country::getName)); + return countries; + } } diff --git a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/AbstractExecutablePlugin.java b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/AbstractExecutablePlugin.java index c6543115f..918ba2cf1 100644 --- a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/AbstractExecutablePlugin.java +++ b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/AbstractExecutablePlugin.java @@ -156,7 +156,7 @@ DpsTask createDpsTaskForProcessPlugin(EcloudBasePluginParameters ecloudBasePlugi } DpsTask createDpsTaskForIndexPlugin(EcloudBasePluginParameters ecloudBasePluginParameters, String datasetId, - boolean incrementalIndexing, Date harvestDate, boolean useAlternativeIndexingEnvironment, boolean preserveTimestamps, + boolean incrementalIndexing, Date harvestDate, boolean preserveTimestamps, List datasetIdsToRedirectFrom, boolean performRedirects, String targetDatabase) { final DateFormat dateFormat = new SimpleDateFormat(CommonStringValues.DATE_FORMAT_Z, Locale.US); dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); @@ -165,7 +165,6 @@ DpsTask createDpsTaskForIndexPlugin(EcloudBasePluginParameters ecloudBasePluginP extraParameters.put(PluginParameterKeys.INCREMENTAL_INDEXING, String.valueOf(incrementalIndexing)); extraParameters.put(PluginParameterKeys.HARVEST_DATE, dateFormat.format(harvestDate)); extraParameters.put(PluginParameterKeys.METIS_TARGET_INDEXING_DATABASE, targetDatabase); - extraParameters.put(PluginParameterKeys.METIS_USE_ALT_INDEXING_ENV, String.valueOf(useAlternativeIndexingEnvironment)); extraParameters.put(PluginParameterKeys.METIS_RECORD_DATE, dateFormat.format(getStartedDate())); extraParameters.put(PluginParameterKeys.METIS_PRESERVE_TIMESTAMPS, String.valueOf(preserveTimestamps)); extraParameters.put(PluginParameterKeys.DATASET_IDS_TO_REDIRECT_FROM, String.join(",", datasetIdsToRedirectFrom)); diff --git a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/AbstractIndexPluginMetadata.java b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/AbstractIndexPluginMetadata.java index 2fd8d4c64..ccf8a7c77 100644 --- a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/AbstractIndexPluginMetadata.java +++ b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/AbstractIndexPluginMetadata.java @@ -10,7 +10,6 @@ */ public abstract class AbstractIndexPluginMetadata extends AbstractExecutablePluginMetadata { - private boolean useAlternativeIndexingEnvironment; private boolean preserveTimestamps; private boolean performRedirects; private List datasetIdsToRedirectFrom = new ArrayList<>(); @@ -21,14 +20,6 @@ public AbstractIndexPluginMetadata() { //Required for json serialization } - public boolean isUseAlternativeIndexingEnvironment() { - return useAlternativeIndexingEnvironment; - } - - public void setUseAlternativeIndexingEnvironment(boolean useAlternativeIndexingEnvironment) { - this.useAlternativeIndexingEnvironment = useAlternativeIndexingEnvironment; - } - public boolean isPreserveTimestamps() { return preserveTimestamps; } diff --git a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/DepublishPlugin.java b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/DepublishPlugin.java index 155e69582..4e4bb705e 100644 --- a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/DepublishPlugin.java +++ b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/DepublishPlugin.java @@ -49,13 +49,9 @@ public String getTopologyName() { @Override public DpsTask prepareDpsTask(String datasetId, EcloudBasePluginParameters ecloudBasePluginParameters) { - boolean useAlternativeIndexingEnvironment = getPluginMetadata() - .isUseAlternativeIndexingEnvironment(); Map extraParameters = new HashMap<>(); extraParameters.put(PluginParameterKeys.METIS_DATASET_ID, datasetId); - extraParameters.put(PluginParameterKeys.METIS_USE_ALT_INDEXING_ENV, - String.valueOf(useAlternativeIndexingEnvironment)); //Do set the records ids parameter only if record ids depublication enabled and there are record ids if (!getPluginMetadata().isDatasetDepublish()) { if (CollectionUtils.isEmpty(getPluginMetadata().getRecordIdsToDepublish())) { diff --git a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/DepublishPluginMetadata.java b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/DepublishPluginMetadata.java index 890a47209..0d615b393 100644 --- a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/DepublishPluginMetadata.java +++ b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/DepublishPluginMetadata.java @@ -15,7 +15,6 @@ public class DepublishPluginMetadata extends AbstractExecutablePluginMetadata { private static final ExecutablePluginType pluginType = ExecutablePluginType.DEPUBLISH; - private boolean useAlternativeIndexingEnvironment; private boolean datasetDepublish; private Set recordIdsToDepublish; @@ -28,14 +27,6 @@ public ExecutablePluginType getExecutablePluginType() { return pluginType; } - public boolean isUseAlternativeIndexingEnvironment() { - return useAlternativeIndexingEnvironment; - } - - public void setUseAlternativeIndexingEnvironment(boolean useAlternativeIndexingEnvironment) { - this.useAlternativeIndexingEnvironment = useAlternativeIndexingEnvironment; - } - public boolean isDatasetDepublish() { return datasetDepublish; } diff --git a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/IndexToPreviewPlugin.java b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/IndexToPreviewPlugin.java index 01bc8aa7f..a828690ae 100644 --- a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/IndexToPreviewPlugin.java +++ b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/IndexToPreviewPlugin.java @@ -2,7 +2,6 @@ import eu.europeana.cloud.service.dps.DpsTask; import eu.europeana.cloud.service.dps.metis.indexing.TargetIndexingDatabase; -import eu.europeana.cloud.service.dps.metis.indexing.TargetIndexingEnvironment; /** * Index to Preview Plugin. @@ -38,7 +37,6 @@ public DpsTask prepareDpsTask(String datasetId, EcloudBasePluginParameters eclou return createDpsTaskForIndexPlugin(ecloudBasePluginParameters, datasetId, getPluginMetadata().isIncrementalIndexing(), getPluginMetadata().getHarvestDate(), - getPluginMetadata().isUseAlternativeIndexingEnvironment(), getPluginMetadata().isPreserveTimestamps(), getPluginMetadata().getDatasetIdsToRedirectFrom(), getPluginMetadata().isPerformRedirects(), getTargetIndexingDatabase().name()); @@ -57,14 +55,4 @@ public String getTopologyName() { public TargetIndexingDatabase getTargetIndexingDatabase() { return TargetIndexingDatabase.PREVIEW; } - - /** - * Get the target indexing environment. - * - * @return the target indexing environment - */ - public TargetIndexingEnvironment getTargetIndexingEnvironment() { - return getPluginMetadata().isUseAlternativeIndexingEnvironment() ? TargetIndexingEnvironment.ALTERNATIVE - : TargetIndexingEnvironment.DEFAULT; - } } diff --git a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/IndexToPublishPlugin.java b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/IndexToPublishPlugin.java index d57c81ba5..368ba7128 100644 --- a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/IndexToPublishPlugin.java +++ b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/IndexToPublishPlugin.java @@ -2,7 +2,6 @@ import eu.europeana.cloud.service.dps.DpsTask; import eu.europeana.cloud.service.dps.metis.indexing.TargetIndexingDatabase; -import eu.europeana.cloud.service.dps.metis.indexing.TargetIndexingEnvironment; /** * Index to Publish Plugin. @@ -37,7 +36,6 @@ public DpsTask prepareDpsTask(String datasetId, EcloudBasePluginParameters eclou return createDpsTaskForIndexPlugin(ecloudBasePluginParameters, datasetId, getPluginMetadata().isIncrementalIndexing(), getPluginMetadata().getHarvestDate(), - getPluginMetadata().isUseAlternativeIndexingEnvironment(), getPluginMetadata().isPreserveTimestamps(), getPluginMetadata().getDatasetIdsToRedirectFrom(), getPluginMetadata().isPerformRedirects(), getTargetIndexingDatabase().name()); @@ -56,14 +54,4 @@ public String getTopologyName() { public TargetIndexingDatabase getTargetIndexingDatabase() { return TargetIndexingDatabase.PUBLISH; } - - /** - * Get the target indexing environment. - * - * @return the target indexing environment - */ - public TargetIndexingEnvironment getTargetIndexingEnvironment() { - return getPluginMetadata().isUseAlternativeIndexingEnvironment() ? TargetIndexingEnvironment.ALTERNATIVE - : TargetIndexingEnvironment.DEFAULT; - } } diff --git a/metis-core/metis-core-rest/pom.xml b/metis-core/metis-core-rest/pom.xml index a01e06941..f27213bf1 100644 --- a/metis-core/metis-core-rest/pom.xml +++ b/metis-core/metis-core-rest/pom.xml @@ -4,7 +4,7 @@ metis-core eu.europeana.metis - 6 + 7 metis-core-rest war diff --git a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/DatasetController.java b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/DatasetController.java index 28c1d5074..9f7b82e0d 100644 --- a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/DatasetController.java +++ b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/DatasetController.java @@ -613,7 +613,8 @@ public ResponseListWrapper getAllDatasetsByOrganizationName( public List getDatasetsCountries( @RequestHeader("Authorization") String authorization) throws GenericMetisException { authenticationClient.getUserByAccessTokenInHeader(authorization); - return Arrays.stream(Country.values()).map(CountryView::new).collect(Collectors.toList()); + return Country.getCountryListSortedByName().stream().map(CountryView::new) + .collect(Collectors.toList()); } /** diff --git a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/ConfigurationPropertiesHolder.java b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/ConfigurationPropertiesHolder.java index dafee7c30..151297a1f 100644 --- a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/ConfigurationPropertiesHolder.java +++ b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/ConfigurationPropertiesHolder.java @@ -95,8 +95,6 @@ public class ConfigurationPropertiesHolder { private int maxDepublishRecordIdsPerDataset; // Ecloud configuration - @Value("${metis.use.alternative.indexing.environment}") - private boolean metisUseAlternativeIndexingEnvironment; @Value("${metis.link.checking.default.sampling.size}") private int metisLinkCheckingDefaultSamplingSize; @Value("${solr.commit.period.in.mins}") @@ -304,10 +302,6 @@ public int getMaxDepublishRecordIdsPerDataset() { return maxDepublishRecordIdsPerDataset; } - public boolean isMetisUseAlternativeIndexingEnvironment() { - return metisUseAlternativeIndexingEnvironment; - } - public int getMetisLinkCheckingDefaultSamplingSize() { return metisLinkCheckingDefaultSamplingSize; } diff --git a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/OrchestratorConfig.java b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/OrchestratorConfig.java index c69fbd5b5..605ef384b 100644 --- a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/OrchestratorConfig.java +++ b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/OrchestratorConfig.java @@ -25,8 +25,8 @@ import eu.europeana.metis.core.service.ProxiesService; import eu.europeana.metis.core.service.ScheduleWorkflowService; import eu.europeana.metis.core.service.WorkflowExecutionFactory; -import java.io.File; import java.net.MalformedURLException; +import java.nio.file.Paths; import java.time.Duration; import java.util.concurrent.TimeUnit; import javax.annotation.PreDestroy; @@ -86,7 +86,7 @@ RedissonClient getRedissonClient() throws MalformedURLException { LOGGER.info("Redis enabled SSL"); if (propertiesHolder.isRedisEnableCustomTruststore()) { singleServerConfig - .setSslTruststore(new File(propertiesHolder.getTruststorePath()).toURI().toURL()); + .setSslTruststore(Paths.get(propertiesHolder.getTruststorePath()).toUri().toURL()); singleServerConfig.setSslTruststorePassword(propertiesHolder.getTruststorePassword()); LOGGER.info("Redis enabled SSL using custom Truststore"); } @@ -139,8 +139,6 @@ public WorkflowExecutionFactory getWorkflowExecutionFactory( .setValidationExternalProperties(propertiesHolder.getValidationExternalProperties()); workflowExecutionFactory .setValidationInternalProperties(propertiesHolder.getValidationInternalProperties()); - workflowExecutionFactory.setMetisUseAlternativeIndexingEnvironment( - propertiesHolder.isMetisUseAlternativeIndexingEnvironment()); workflowExecutionFactory.setDefaultSamplingSizeForLinkChecking( propertiesHolder.getMetisLinkCheckingDefaultSamplingSize()); return workflowExecutionFactory; diff --git a/metis-core/metis-core-rest/src/main/resources/metis.properties.example b/metis-core/metis-core-rest/src/main/resources/metis.properties.example index f7f7050da..07bec7656 100644 --- a/metis-core/metis-core-rest/src/main/resources/metis.properties.example +++ b/metis-core/metis-core-rest/src/main/resources/metis.properties.example @@ -88,7 +88,4 @@ metis.core.baseUrl= #Metis Core (regardless on whether the list is paginated). metis.core.max.served.execution.list.length= metis.core.max.depublish.record.ids.per.dataset= -#In the combination of TEST and ACCEPTANCE, TEST=false, ACCEPTANCE=true -#For the production environment it should be false -metis.use.alternative.indexing.environment= metis.link.checking.default.sampling.size= \ No newline at end of file diff --git a/metis-core/metis-core-rest/src/test/java/eu/europeana/metis/core/rest/utils/TestObjectFactory.java b/metis-core/metis-core-rest/src/test/java/eu/europeana/metis/core/rest/utils/TestObjectFactory.java index fdc1ed04d..4dc8d7598 100644 --- a/metis-core/metis-core-rest/src/test/java/eu/europeana/metis/core/rest/utils/TestObjectFactory.java +++ b/metis-core/metis-core-rest/src/test/java/eu/europeana/metis/core/rest/utils/TestObjectFactory.java @@ -253,11 +253,11 @@ public static MetisUserView createMetisUser(String email) { * @return the created sub task info */ public static List createListOfSubTaskInfo() { - SubTaskInfo subTaskInfo1 = new SubTaskInfo(1, "some_resource_id1", RecordState.SUCCESS, "", - "Sensitive Information"); - final int resourceNum = 2; - SubTaskInfo subTaskInfo2 = new SubTaskInfo(resourceNum, "some_resource_id1", RecordState.SUCCESS, "", - "Sensitive Information"); + + SubTaskInfo subTaskInfo1 = new SubTaskInfo(1, "some_resource_id1", RecordState.SUCCESS, "info", + "additional info", "europeanaId", 0L); + SubTaskInfo subTaskInfo2 = new SubTaskInfo(2, "some_resource_id2", RecordState.SUCCESS, "info", + "additional info", "europeanaId", 0L); ArrayList subTaskInfos = new ArrayList<>(); subTaskInfos.add(subTaskInfo1); subTaskInfos.add(subTaskInfo2); diff --git a/metis-core/metis-core-service/pom.xml b/metis-core/metis-core-service/pom.xml index 1d8d2a29c..85e4dccf2 100644 --- a/metis-core/metis-core-service/pom.xml +++ b/metis-core/metis-core-service/pom.xml @@ -4,7 +4,7 @@ metis-core eu.europeana.metis - 6 + 7 metis-core-service diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowExecutor.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowExecutor.java index 2c9393d0c..6281b87d0 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowExecutor.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowExecutor.java @@ -24,6 +24,7 @@ import eu.europeana.metis.core.workflow.plugins.ExecutablePluginType; import eu.europeana.metis.core.workflow.plugins.PluginStatus; import eu.europeana.metis.core.workflow.plugins.PluginType; +import eu.europeana.metis.exception.BadContentException; import eu.europeana.metis.exception.ExternalTaskException; import eu.europeana.metis.network.ExternalRequestUtil; import java.util.Date; @@ -491,7 +492,7 @@ private boolean applyPostProcessing(MonitorResult monitorResult, AbstractExecuta if (monitorResult.getTaskState() == TaskState.PROCESSED) { try { this.workflowPostProcessor.performPluginPostProcessing(plugin, datasetId); - } catch (DpsException | InvalidIndexPluginException | RuntimeException e) { + } catch (DpsException | InvalidIndexPluginException | BadContentException | RuntimeException e) { processingAppliedOrNotRequired = false; LOGGER.warn("Problem occurred during Metis post-processing.", e); plugin.setFinishedDate(null); diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java index 2cbc2fc9d..5864a613c 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java @@ -7,7 +7,6 @@ import eu.europeana.cloud.common.model.dps.SubTaskInfo; import eu.europeana.cloud.service.dps.exception.DpsException; import eu.europeana.cloud.service.dps.metis.indexing.TargetIndexingDatabase; -import eu.europeana.cloud.service.dps.metis.indexing.TargetIndexingEnvironment; import eu.europeana.metis.core.common.DepublishRecordIdUtils; import eu.europeana.metis.core.dao.DatasetDao; import eu.europeana.metis.core.dao.DepublishRecordIdDao; @@ -18,6 +17,8 @@ import eu.europeana.metis.core.dataset.DepublishRecordId.DepublicationStatus; import eu.europeana.metis.core.exceptions.InvalidIndexPluginException; import eu.europeana.metis.core.service.OrchestratorService; +import eu.europeana.metis.core.util.DepublishRecordIdSortField; +import eu.europeana.metis.core.util.SortDirection; import eu.europeana.metis.core.workflow.WorkflowExecution; import eu.europeana.metis.core.workflow.plugins.AbstractExecutablePlugin; import eu.europeana.metis.core.workflow.plugins.AbstractMetisPlugin; @@ -27,6 +28,7 @@ import eu.europeana.metis.core.workflow.plugins.IndexToPublishPlugin; import eu.europeana.metis.core.workflow.plugins.MetisPlugin; import eu.europeana.metis.core.workflow.plugins.PluginType; +import eu.europeana.metis.exception.BadContentException; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -34,19 +36,21 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.util.CollectionUtils; /** - * This object can perform post processing for workflows. + * This object can perform post-processing for workflows. */ public class WorkflowPostProcessor { private static final Logger LOGGER = LoggerFactory.getLogger(WorkflowPostProcessor.class); - private static final int ECLOUD_REQUEST_BATCH_SIZE = 100; + private static final int ECLOUD_REQUEST_BATCH_SIZE = 1000; private final DepublishRecordIdDao depublishRecordIdDao; private final DatasetDao datasetDao; @@ -56,9 +60,9 @@ public class WorkflowPostProcessor { /** * Constructor. * - * @param depublishRecordIdDao The DAO for depublished records. + * @param depublishRecordIdDao The DAO for de-published records * @param datasetDao The DAO for datasets - * @param workflowExecutionDao The DAO for workflow executions. + * @param workflowExecutionDao The DAO for workflow executions * @param dpsClient the dps client */ public WorkflowPostProcessor(DepublishRecordIdDao depublishRecordIdDao, @@ -69,62 +73,70 @@ public WorkflowPostProcessor(DepublishRecordIdDao depublishRecordIdDao, this.dpsClient = dpsClient; } - /** - * This method performs post processing after an individual workflow step. - * - * @param plugin The plugin that was successfully executed. - * @param datasetId The dataset ID to which the plugin belongs. - */ - void performPluginPostProcessing(AbstractExecutablePlugin plugin, String datasetId) - throws DpsException, InvalidIndexPluginException { - - final PluginType pluginType = plugin.getPluginType(); - LOGGER.info("Starting postprocessing of plugin {} in dataset {}.", pluginType, datasetId); - if (pluginType == PluginType.PREVIEW || pluginType == PluginType.PUBLISH) { - indexPostProcess(plugin, datasetId); - } else if (pluginType == PluginType.DEPUBLISH) { - depublishPostProcess((DepublishPlugin) plugin, datasetId); - } - LOGGER.info("Finished postprocessing of plugin {} in dataset {}.", pluginType, datasetId); - } - /** * Performs post-processing for indexing plugins * - * @param indexPlugin the index plugin - * @param datasetId the dataset id - * @throws DpsException if communication with ecloud dps failed + * @param indexPlugin The index plugin + * @param datasetId The dataset id + * @throws DpsException If communication with e-cloud dps failed + * @throws InvalidIndexPluginException If invalid type of plugin + * @throws BadContentException In case the records would violate the maximum number of de-published records that each + * dataset can have. */ private void indexPostProcess(AbstractExecutablePlugin indexPlugin, String datasetId) - throws DpsException, InvalidIndexPluginException { + throws DpsException, InvalidIndexPluginException, BadContentException { TargetIndexingDatabase targetIndexingDatabase; - TargetIndexingEnvironment targetIndexingEnvironment; if (indexPlugin instanceof IndexToPreviewPlugin) { targetIndexingDatabase = ((IndexToPreviewPlugin) indexPlugin).getTargetIndexingDatabase(); - targetIndexingEnvironment = ((IndexToPreviewPlugin) indexPlugin).getTargetIndexingEnvironment(); } else if (indexPlugin instanceof IndexToPublishPlugin) { targetIndexingDatabase = ((IndexToPublishPlugin) indexPlugin).getTargetIndexingDatabase(); - targetIndexingEnvironment = ((IndexToPublishPlugin) indexPlugin).getTargetIndexingEnvironment(); - //Reset depublish status - depublishRecordIdDao.markRecordIdsWithDepublicationStatus(datasetId, null, - DepublicationStatus.PENDING_DEPUBLICATION, null); + + final boolean isIncremental = ((IndexToPublishPlugin) indexPlugin).getPluginMetadata().isIncrementalIndexing(); + + if (isIncremental) { + // get all currently de-published records IDs from the database and create their full versions + final Set depublishedRecordIds = depublishRecordIdDao.getAllDepublishRecordIdsWithStatus( + datasetId, DepublishRecordIdSortField.DEPUBLICATION_STATE, SortDirection.ASCENDING, + DepublicationStatus.DEPUBLISHED); + final Map depublishedRecordIdsByFullId = depublishedRecordIds.stream() + .collect(Collectors.toMap(id -> DepublishRecordIdUtils.composeFullRecordId(datasetId, id), + Function.identity())); + + // Check which have been published by the index action - use full record IDs for eCloud. + if (!CollectionUtils.isEmpty(depublishedRecordIdsByFullId)) { + final List publishedRecordIds = dpsClient.searchPublishedDatasetRecords(datasetId, + new ArrayList<>(depublishedRecordIdsByFullId.keySet())); + + // Remove the 'depublished' status. Note: we need to check for an empty result (otherwise + // the DAO call will update all records). Use the simple record IDs again. + if (!CollectionUtils.isEmpty(publishedRecordIds)) { + depublishRecordIdDao.markRecordIdsWithDepublicationStatus(datasetId, + publishedRecordIds.stream().map(depublishedRecordIdsByFullId::get) + .collect(Collectors.toSet()), DepublicationStatus.PENDING_DEPUBLICATION, null); + } + } + } else { + // reset de-publish status, pass null, all records will be de-published + depublishRecordIdDao.markRecordIdsWithDepublicationStatus(datasetId, null, + DepublicationStatus.PENDING_DEPUBLICATION, null); + } } else { throw new InvalidIndexPluginException("Plugin is not of the types supported"); } final Integer databaseTotalRecords = retryableExternalRequestForNetworkExceptionsThrowing(() -> - (int) dpsClient.getTotalMetisDatabaseRecords(datasetId, targetIndexingDatabase, - targetIndexingEnvironment)); + (int) dpsClient.getTotalMetisDatabaseRecords(datasetId, targetIndexingDatabase)); indexPlugin.getExecutionProgress().setTotalDatabaseRecords(databaseTotalRecords); } /** - * Performs post processing for depublish plugins + * Performs post-processing for de-publish plugins * - * @param depublishPlugin the depublish plugin - * @param datasetId the dataset id - * @throws DpsException if communication with ecloud dps failed + * @param depublishPlugin The de-publish plugin + * @param datasetId The dataset id + * @throws DpsException If communication with e-cloud dps failed */ - private void depublishPostProcess(DepublishPlugin depublishPlugin, String datasetId) throws DpsException { + private void depublishPostProcess(DepublishPlugin depublishPlugin, String datasetId) + throws DpsException { if (depublishPlugin.getPluginMetadata().isDatasetDepublish()) { depublishDatasetPostProcess(datasetId); } else { @@ -132,12 +144,58 @@ private void depublishPostProcess(DepublishPlugin depublishPlugin, String datase } } + /** + * @param depublishPlugin The de-publish plugin + * @param datasetId The dataset id + * @throws DpsException If communication with e-cloud dps failed + */ + private void depublishRecordPostProcess(DepublishPlugin depublishPlugin, String datasetId) + throws DpsException { + + // Retrieve the successfully depublished records. + final long externalTaskId = Long.parseLong(depublishPlugin.getExternalTaskId()); + final List subTasks = new ArrayList<>(); + List subTasksBatch; + do { + subTasksBatch = retryableExternalRequestForNetworkExceptionsThrowing( + () -> dpsClient.getDetailedTaskReportBetweenChunks( + depublishPlugin.getTopologyName(), externalTaskId, subTasks.size(), + subTasks.size() + ECLOUD_REQUEST_BATCH_SIZE)); + subTasks.addAll(subTasksBatch); + } while (subTasksBatch.size() == ECLOUD_REQUEST_BATCH_SIZE); + + // Mark the records as DEPUBLISHED. + final Map> successfulRecords = subTasks.stream() + .filter(subTask -> + subTask.getRecordState() + == RecordState.SUCCESS) + .map(SubTaskInfo::getResource).map( + DepublishRecordIdUtils::decomposeFullRecordId) + .collect(Collectors.groupingBy( + Pair::getLeft, + Collectors.mapping( + Pair::getRight, + Collectors.toSet()))); + successfulRecords.forEach((dataset, records) -> + depublishRecordIdDao.markRecordIdsWithDepublicationStatus(dataset, records, + DepublicationStatus.DEPUBLISHED, new Date())); + + // Set publication fitness to PARTIALLY FIT (if not set to the more severe UNFIT). + final Dataset dataset = datasetDao.getDatasetByDatasetId(datasetId); + if (dataset.getPublicationFitness() != PublicationFitness.UNFIT) { + dataset.setPublicationFitness(PublicationFitness.PARTIALLY_FIT); + datasetDao.update(dataset); + } + } + + /** + * @param datasetId The dataset id + */ private void depublishDatasetPostProcess(String datasetId) { // Set all depublished records back to PENDING. depublishRecordIdDao.markRecordIdsWithDepublicationStatus(datasetId, null, DepublicationStatus.PENDING_DEPUBLICATION, null); - // Find latest PUBLISH Type Plugin and set dataStatus to DELETED. final PluginWithExecutionId latestSuccessfulPlugin = workflowExecutionDao .getLatestSuccessfulPlugin(datasetId, OrchestratorService.PUBLISH_TYPES); @@ -152,41 +210,32 @@ private void depublishDatasetPostProcess(String datasetId) { workflowExecutionDao.updateWorkflowPlugins(workflowExecutionToUpdate); } } - // Set publication fitness to UNFIT. final Dataset dataset = datasetDao.getDatasetByDatasetId(datasetId); dataset.setPublicationFitness(PublicationFitness.UNFIT); datasetDao.update(dataset); } - private void depublishRecordPostProcess(DepublishPlugin depublishPlugin, String datasetId) throws DpsException { - - // Retrieve the successfully depublished records. - final long externalTaskId = Long.parseLong(depublishPlugin.getExternalTaskId()); - final List subTasks = new ArrayList<>(); - List subTasksBatch; - do { - subTasksBatch = retryableExternalRequestForNetworkExceptionsThrowing(() -> dpsClient.getDetailedTaskReportBetweenChunks( - depublishPlugin.getTopologyName(), externalTaskId, subTasks.size(), - subTasks.size() + ECLOUD_REQUEST_BATCH_SIZE)); - subTasks.addAll(subTasksBatch); - } while (subTasksBatch.size() == ECLOUD_REQUEST_BATCH_SIZE); - - // Mark the records as DEPUBLISHED. - final Map> successfulRecords = subTasks.stream() - .filter(subTask -> subTask.getRecordState() == RecordState.SUCCESS) - .map(SubTaskInfo::getResource).map(DepublishRecordIdUtils::decomposeFullRecordId) - .collect(Collectors.groupingBy(Pair::getLeft, - Collectors.mapping(Pair::getRight, Collectors.toSet()))); - successfulRecords.forEach((dataset, records) -> - depublishRecordIdDao.markRecordIdsWithDepublicationStatus(dataset, records, - DepublicationStatus.DEPUBLISHED, new Date())); + /** + * This method performs post-processing after an individual workflow step. + * + * @param plugin The plugin that was successfully executed + * @param datasetId The dataset ID to which the plugin belongs + * @throws DpsException If communication with e-cloud dps failed + * @throws InvalidIndexPluginException If invalid type of plugin + * @throws BadContentException In case the records would violate the maximum number of de-published records that each dataset + * can have. + */ + void performPluginPostProcessing(AbstractExecutablePlugin plugin, String datasetId) + throws DpsException, InvalidIndexPluginException, BadContentException { - // Set publication fitness to PARTIALLY FIT (if not set to the more severe UNFIT). - final Dataset dataset = datasetDao.getDatasetByDatasetId(datasetId); - if (dataset.getPublicationFitness() != PublicationFitness.UNFIT) { - dataset.setPublicationFitness(PublicationFitness.PARTIALLY_FIT); - datasetDao.update(dataset); + final PluginType pluginType = plugin.getPluginType(); + LOGGER.info("Starting postprocessing of plugin {} in dataset {}.", pluginType, datasetId); + if (pluginType == PluginType.PREVIEW || pluginType == PluginType.PUBLISH) { + indexPostProcess(plugin, datasetId); + } else if (pluginType == PluginType.DEPUBLISH) { + depublishPostProcess((DepublishPlugin) plugin, datasetId); } + LOGGER.info("Finished postprocessing of plugin {} in dataset {}.", pluginType, datasetId); } -} +} \ No newline at end of file diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/OrchestratorService.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/OrchestratorService.java index 8a237a8a3..37447732f 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/OrchestratorService.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/OrchestratorService.java @@ -774,11 +774,9 @@ private void setPublishInformation(DatasetExecutionInformation executionInfo, final int depublishedRecordCount; if (datasetCurrentlyDepublished) { depublishedRecordCount = executionInfo.getLastPublishedRecords(); - } else if (depublishHappenedAfterLatestExecutablePublish) { + } else { depublishedRecordCount = (int) depublishRecordIdDao .countSuccessfullyDepublishedRecordIdsForDataset(datasetId); - } else { - depublishedRecordCount = 0; } //Compute more general information of the plugin diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/WorkflowExecutionFactory.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/WorkflowExecutionFactory.java index bac712f4a..d0f0c644f 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/WorkflowExecutionFactory.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/WorkflowExecutionFactory.java @@ -50,7 +50,6 @@ public class WorkflowExecutionFactory { private ValidationProperties validationExternalProperties; // Use getter and setter! private ValidationProperties validationInternalProperties; // Use getter and setter! - private boolean metisUseAlternativeIndexingEnvironment; // Use getter and setter for this field! private int defaultSamplingSizeForLinkChecking; // Use getter and setter for this field! /** @@ -112,24 +111,18 @@ private AbstractExecutablePlugin createWorkflowExecutionPlugin(Dataset datase this.setupValidationInternalForPluginMetadata( (ValidationInternalPluginMetadata) pluginMetadata, getValidationInternalProperties()); } else if (pluginMetadata instanceof IndexToPreviewPluginMetadata) { - ((IndexToPreviewPluginMetadata) pluginMetadata) - .setUseAlternativeIndexingEnvironment(isMetisUseAlternativeIndexingEnvironment()); ((IndexToPreviewPluginMetadata) pluginMetadata) .setDatasetIdsToRedirectFrom(dataset.getDatasetIdsToRedirectFrom()); boolean performRedirects = shouldRedirectsBePerformed(dataset, workflowPredecessor, ExecutablePluginType.PREVIEW, typesInWorkflowBeforeThisPlugin); ((IndexToPreviewPluginMetadata) pluginMetadata).setPerformRedirects(performRedirects); } else if (pluginMetadata instanceof IndexToPublishPluginMetadata) { - ((IndexToPublishPluginMetadata) pluginMetadata) - .setUseAlternativeIndexingEnvironment(isMetisUseAlternativeIndexingEnvironment()); ((IndexToPublishPluginMetadata) pluginMetadata) .setDatasetIdsToRedirectFrom(dataset.getDatasetIdsToRedirectFrom()); boolean performRedirects = shouldRedirectsBePerformed(dataset, workflowPredecessor, ExecutablePluginType.PUBLISH, typesInWorkflowBeforeThisPlugin); ((IndexToPublishPluginMetadata) pluginMetadata).setPerformRedirects(performRedirects); } else if (pluginMetadata instanceof DepublishPluginMetadata) { - ((DepublishPluginMetadata) pluginMetadata) - .setUseAlternativeIndexingEnvironment(isMetisUseAlternativeIndexingEnvironment()); setupDepublishPluginMetadata(dataset, ((DepublishPluginMetadata) pluginMetadata)); } else if (pluginMetadata instanceof LinkCheckingPluginMetadata) { ((LinkCheckingPluginMetadata) pluginMetadata) @@ -306,19 +299,6 @@ public void setValidationInternalProperties(ValidationProperties validationInter } } - private boolean isMetisUseAlternativeIndexingEnvironment() { - synchronized (this) { - return metisUseAlternativeIndexingEnvironment; - } - } - - public void setMetisUseAlternativeIndexingEnvironment( - boolean metisUseAlternativeIndexingEnvironment) { - synchronized (this) { - this.metisUseAlternativeIndexingEnvironment = metisUseAlternativeIndexingEnvironment; - } - } - private int getDefaultSamplingSizeForLinkChecking() { synchronized (this) { return defaultSamplingSizeForLinkChecking; diff --git a/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/service/TestDatasetService.java b/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/service/TestDatasetService.java index 678e3a3ff..74f6754e4 100644 --- a/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/service/TestDatasetService.java +++ b/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/service/TestDatasetService.java @@ -22,7 +22,6 @@ import static org.mockito.Mockito.when; import com.github.tomakehurst.wiremock.WireMockServer; -import eu.europeana.metis.utils.RestEndpoints; import eu.europeana.metis.authentication.user.MetisUserView; import eu.europeana.metis.core.dao.DatasetDao; import eu.europeana.metis.core.dao.DatasetXsltDao; @@ -41,6 +40,7 @@ import eu.europeana.metis.exception.GenericMetisException; import eu.europeana.metis.exception.UserUnauthorizedException; import eu.europeana.metis.network.NetworkUtil; +import eu.europeana.metis.utils.RestEndpoints; import java.io.IOException; import java.io.StringReader; import java.util.ArrayList; @@ -66,7 +66,7 @@ class TestDatasetService { static { try { - portForWireMock = NetworkUtil.getAvailableLocalPort(); + portForWireMock = new NetworkUtil().getAvailableLocalPort(); } catch (IOException e) { e.printStackTrace(); } diff --git a/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/service/TestProxiesService.java b/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/service/TestProxiesService.java index 0a004009b..c42e800a3 100644 --- a/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/service/TestProxiesService.java +++ b/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/service/TestProxiesService.java @@ -564,7 +564,7 @@ void testGetRecord() throws MCSException, ExternalTaskException { // Create representation final Representation representation = mock(Representation.class); - final String contentUri = "http://example.com"; + final String contentUri = "https://example.com"; final File file = new File(); file.setContentUri(URI.create(contentUri)); when(representation.getFiles()).thenReturn(Collections.singletonList(file)); diff --git a/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/utils/TestObjectFactory.java b/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/utils/TestObjectFactory.java index 17b7f9833..38d389e4b 100644 --- a/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/utils/TestObjectFactory.java +++ b/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/utils/TestObjectFactory.java @@ -16,8 +16,8 @@ import eu.europeana.metis.core.common.Language; import eu.europeana.metis.core.dao.WorkflowExecutionDao.ExecutionDatasetPair; import eu.europeana.metis.core.dataset.Dataset; -import eu.europeana.metis.core.dataset.DatasetXslt; import eu.europeana.metis.core.dataset.Dataset.PublicationFitness; +import eu.europeana.metis.core.dataset.DatasetXslt; import eu.europeana.metis.core.rest.Record; import eu.europeana.metis.core.workflow.ScheduleFrequence; import eu.europeana.metis.core.workflow.ScheduledWorkflow; @@ -134,8 +134,7 @@ private static WorkflowExecution createWorkflowExecutionObject(Dataset dataset) } /** - * Create a list of dummy workflow executions. The dataset name will have a suffix number for each - * dataset. + * Create a list of dummy workflow executions. The dataset name will have a suffix number for each dataset. * * @param size the number of dummy workflow executions to create * @return the created list @@ -146,8 +145,7 @@ public static List createListOfWorkflowExecutions(int size) { } /** - * Create a list of dummy execution overviews. The dataset name will have a suffix number for each - * dataset. + * Create a list of dummy execution overviews. The dataset name will have a suffix number for each dataset. * * @param size the number of dummy execution overviews to create * @return the created list @@ -180,8 +178,7 @@ public static ScheduledWorkflow createScheduledWorkflowObject() { } /** - * Create a list of dummy scheduled workflows. The dataset name will have a suffix number for each - * dataset. + * Create a list of dummy scheduled workflows. The dataset name will have a suffix number for each dataset. * * @param size the number of dummy scheduled workflows to create * @return the created list @@ -198,11 +195,11 @@ public static List createListOfScheduledWorkflows(int size) { } /** - * Create a list of dummy scheduled workflows with pointer date and frequency. The dataset name - * will have a suffix number for each dataset. + * Create a list of dummy scheduled workflows with pointer date and frequency. The dataset name will have a suffix number for + * each dataset. * - * @param size the number of dummy scheduled workflows to create - * @param date the pointer date + * @param size the number of dummy scheduled workflows to create + * @param date the pointer date * @param scheduleFrequence the schedule frequence * @return the created list */ @@ -270,17 +267,15 @@ public static MetisUserView createMetisUser(String email) { } /** - * Create a dummy sub task info + * Create a dummy subtask info * - * @return the created sub task info + * @return the created subtask info */ public static List createListOfSubTaskInfo() { - SubTaskInfo subTaskInfo1 = new SubTaskInfo(1, "some_resource_id1", RecordState.SUCCESS, "", - "Sensitive Information"); - final int resourceNum = 2; - SubTaskInfo subTaskInfo2 = new SubTaskInfo(resourceNum, "some_resource_id1", - RecordState.SUCCESS, "", - "Sensitive Information"); + SubTaskInfo subTaskInfo1 = new SubTaskInfo(1, "some_resource_id1", RecordState.SUCCESS, "info", + "additional info", "europeanaId", 0L); + SubTaskInfo subTaskInfo2 = new SubTaskInfo(2, "some_resource_id2", RecordState.SUCCESS, "info", + "additional info", "europeanaId", 0L); ArrayList subTaskInfos = new ArrayList<>(); subTaskInfos.add(subTaskInfo1); subTaskInfos.add(subTaskInfo2); @@ -304,8 +299,8 @@ public static TaskErrorsInfo createTaskErrorsInfoListWithoutIdentifiers(int numb } /** - * Create a task errors info object, which contains a list of {@link TaskErrorInfo} objects. These - * will also contain a list of {@link ErrorDetails} that in turn contain dummy identifiers. + * Create a task errors info object, which contains a list of {@link TaskErrorInfo} objects. These will also contain a list of + * {@link ErrorDetails} that in turn contain dummy identifiers. * * @param numberOfErrorTypes the number of dummy error types * @return the created task errors info @@ -325,11 +320,11 @@ public static TaskErrorsInfo createTaskErrorsInfoListWithIdentifiers(int numberO } /** - * Create a task errors info object, which contains a list of {@link TaskErrorInfo} objects. These - * will also contain a list of {@link ErrorDetails} that in turn contain dummy identifiers. + * Create a task errors info object, which contains a list of {@link TaskErrorInfo} objects. These will also contain a list of + * {@link ErrorDetails} that in turn contain dummy identifiers. * * @param errorType the error type to be used for the internal {@link TaskErrorInfo} - * @param message the message type to be used for the internal {@link TaskErrorInfo} + * @param message the message type to be used for the internal {@link TaskErrorInfo} * @return the created task errors info */ public static TaskErrorsInfo createTaskErrorsInfoWithIdentifiers(String errorType, diff --git a/metis-core/pom.xml b/metis-core/pom.xml index ffba399b3..44736dfb9 100644 --- a/metis-core/pom.xml +++ b/metis-core/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 6 + 7 metis-core pom diff --git a/metis-dereference/metis-dereference-common/pom.xml b/metis-dereference/metis-dereference-common/pom.xml index e485103b5..b99c31323 100644 --- a/metis-dereference/metis-dereference-common/pom.xml +++ b/metis-dereference/metis-dereference-common/pom.xml @@ -4,7 +4,7 @@ metis-dereference eu.europeana.metis - 6 + 7 metis-dereference-common diff --git a/metis-dereference/metis-dereference-common/src/main/java/eu/europeana/metis/dereference/IncomingRecordToEdmConverter.java b/metis-dereference/metis-dereference-common/src/main/java/eu/europeana/metis/dereference/IncomingRecordToEdmConverter.java deleted file mode 100644 index ed4170896..000000000 --- a/metis-dereference/metis-dereference-common/src/main/java/eu/europeana/metis/dereference/IncomingRecordToEdmConverter.java +++ /dev/null @@ -1,94 +0,0 @@ -package eu.europeana.metis.dereference; - -import java.io.StringReader; -import java.io.StringWriter; -import java.nio.charset.StandardCharsets; -import java.util.regex.Pattern; -import javax.xml.XMLConstants; -import javax.xml.transform.OutputKeys; -import javax.xml.transform.Source; -import javax.xml.transform.Templates; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.stream.StreamResult; -import javax.xml.transform.stream.StreamSource; -import net.sf.saxon.BasicTransformerFactory; - -/** - * Convert an incoming record to EDM. - */ -public class IncomingRecordToEdmConverter { - - private static final String EMPTY_XML_REGEX = "\\A(<\\?.*?\\?>||\\s)*\\Z"; - private static final Pattern EMPTY_XML_CHECKER = Pattern.compile(EMPTY_XML_REGEX, Pattern.DOTALL); - - /** Vocabulary XSLs require the resource ID as a parameter. This is the parameter name. **/ - private static final String TARGET_ID_PARAMETER_NAME = "targetId"; - - private final Templates template; - - /** - * Create a converter for the given vocabulary. - * - * @param vocabulary The vocabulary for which to perform the conversion. - * @throws TransformerException In case the input could not be parsed or the conversion could not - * be set up. - */ - public IncomingRecordToEdmConverter(Vocabulary vocabulary) throws TransformerException { - this(vocabulary.getXslt()); - } - - /** - * Create a converter for the transformation. - * - * @param xslt The xslt representing the conversion to perform. - * @throws TransformerException In case the input could not be parsed or the conversion could not - * be set up. - */ - public IncomingRecordToEdmConverter(String xslt) throws TransformerException { - final Source xsltSource = new StreamSource(new StringReader(xslt)); - // Ensure that the Saxon library is used by choosing the right transformer factory. - final TransformerFactory factory = new BasicTransformerFactory(); - factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); - this.template = factory.newTemplates(xsltSource); - } - - /** - * Convert the given record. - * - * @param record The incoming record (that comes from the vocabulary). - * @param recordId The record ID of the incoming record. - * @return The EDM record, or null if the record couldn't be transformed. - * @throws TransformerException In case there is a problem performing the transformation. - */ - public String convert(String record, String recordId) throws TransformerException { - - // Set up the transformer - final Source source = new StreamSource(new StringReader(record)); - final StringWriter stringWriter = new StringWriter(); - final Transformer transformer = template.newTransformer(); - transformer.setParameter(TARGET_ID_PARAMETER_NAME, recordId); - transformer.setOutputProperty(OutputKeys.INDENT, "yes"); - transformer.setOutputProperty(OutputKeys.ENCODING, StandardCharsets.UTF_8.name()); - - // Perform the transformation. - transformer.transform(source, new StreamResult(stringWriter)); - final String result = stringWriter.toString(); - - // Check whether there is a result (any tag in the file). - return isEmptyXml(result) ? null : result; - } - - /** - * This method analyzes the XML file and decides whether or not it has any content. Excluded are - * space characters, the XML header and XML comments. Note: if this method returns true, the input - * is not technically a valid XML as it doesn't have a root node. - * - * @param file The input XML. - * @return Whether the XML has any content. - */ - static boolean isEmptyXml(String file) { - return EMPTY_XML_CHECKER.matcher(file).matches(); - } -} diff --git a/metis-dereference/metis-dereference-common/src/main/java/eu/europeana/metis/dereference/IncomingRecordToEdmTransformer.java b/metis-dereference/metis-dereference-common/src/main/java/eu/europeana/metis/dereference/IncomingRecordToEdmTransformer.java new file mode 100644 index 000000000..8d2677b9a --- /dev/null +++ b/metis-dereference/metis-dereference-common/src/main/java/eu/europeana/metis/dereference/IncomingRecordToEdmTransformer.java @@ -0,0 +1,148 @@ +package eu.europeana.metis.dereference; + +import static eu.europeana.metis.utils.CommonStringValues.CRLF_PATTERN; + +import eu.europeana.metis.exception.BadContentException; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import java.util.regex.Pattern; +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Source; +import javax.xml.transform.Templates; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.stream.StreamResult; +import javax.xml.transform.stream.StreamSource; +import net.sf.saxon.BasicTransformerFactory; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.xml.sax.SAXException; + +/** + * Convert an incoming record to EDM. + */ +public class IncomingRecordToEdmTransformer { + + private static final Logger LOGGER = LoggerFactory.getLogger(IncomingRecordToEdmTransformer.class); + private static final Pattern XML_DECLARATION_CHECKER = Pattern.compile("\\A<\\?[^?]*\\?>\\s*\\z"); + + /** + * Vocabulary XSLs require the resource ID as a parameter. This is the parameter name. + **/ + private static final String TARGET_ID_PARAMETER_NAME = "targetId"; + + private final Templates template; + private final DocumentBuilderFactory documentBuilderFactory; + + /** + * Create a converter for the transformation. + * + * @param xslt The xslt representing the conversion to perform. + * @throws TransformerException if the transformer could not be initialized + * @throws ParserConfigurationException if the xml builder could not be initialized + */ + public IncomingRecordToEdmTransformer(String xslt) throws TransformerException, ParserConfigurationException { + final Source xsltSource = new StreamSource(new StringReader(xslt)); + // Ensure that the Saxon library is used by choosing the right transformer factory. + final TransformerFactory transformerFactory = new BasicTransformerFactory(); + transformerFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + this.template = transformerFactory.newTemplates(xsltSource); + + documentBuilderFactory = DocumentBuilderFactory.newInstance(); + documentBuilderFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + documentBuilderFactory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + documentBuilderFactory.setNamespaceAware(true); + } + + /** + * Transform the given xmlRecord. + * + * @param xmlRecord The incoming xmlRecord (that comes from the vocabulary). + * @param resourceId The xmlRecord ID of the incoming xmlRecord. + * @return The EDM xmlRecord, or null if the xmlRecord couldn't be transformed. + * @throws BadContentException if there was a problem performing the transformation. + */ + public Optional transform(String xmlRecord, String resourceId) throws BadContentException { + // Set up the transformer + final Source source = new StreamSource(new StringReader(xmlRecord)); + final StringWriter transformedXmlWriter = new StringWriter(); + final Transformer transformer; + try { + transformer = template.newTransformer(); + transformer.setParameter(TARGET_ID_PARAMETER_NAME, resourceId); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + transformer.setOutputProperty(OutputKeys.ENCODING, StandardCharsets.UTF_8.name()); + + // Perform the transformation. + transformer.transform(source, new StreamResult(transformedXmlWriter)); + } catch (TransformerException e) { + throw new BadContentException("Transformation failure", e); + } + return getValidatedXml(resourceId, transformedXmlWriter.toString()); + } + + /** + * Returns an optional which is empty if the provided xml is a validated empty xml or contains the xml itself if it's a valid + * parsable xml. + * + * @param resourceId the resource id + * @param xml the xml + * @return the optional being empty or with the xml contents + * @throws BadContentException if the xml parsing failed + */ + @NotNull + private Optional getValidatedXml(String resourceId, String xml) throws BadContentException { + final Optional xmlResponse; + if (isEmptyXml(xml)) { + xmlResponse = Optional.empty(); + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Transformed entity {} results to an empty XML.", + CRLF_PATTERN.matcher(resourceId).replaceAll("")); + } + } else { + try { + assertXmlValidity(xml); + xmlResponse = Optional.of(xml); + } catch (ParserConfigurationException | IOException | SAXException e) { + throw new BadContentException("Transformed xml is not valid", e); + } + } + + return xmlResponse; + } + + /** + * Asserts if the provided xml is valid and can be parsed. + * + * @param xml the xml string + * @throws ParserConfigurationException if xml parsing failed + * @throws IOException if xml parsing failed + * @throws SAXException if xml parsing failed + */ + private void assertXmlValidity(String xml) throws ParserConfigurationException, IOException, SAXException { + documentBuilderFactory.newDocumentBuilder().parse(new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))); + } + + /** + * Checks if the provided xml is empty. + *

    + * Emptiness is verifying if the only the xml header declaration is present. Note: if this method returns true, the input is not + * technically a valid XML as it doesn't have a root node. + *

    + * + * @param xml the input XML. + * @return true if xml is empty + */ + private boolean isEmptyXml(String xml) { + return XML_DECLARATION_CHECKER.matcher(xml).matches(); + } +} diff --git a/metis-dereference/metis-dereference-common/src/test/java/eu/europeana/metis/dereference/IncomingRecordToEdmConverterTest.java b/metis-dereference/metis-dereference-common/src/test/java/eu/europeana/metis/dereference/IncomingRecordToEdmConverterTest.java deleted file mode 100644 index 5e6d2ce78..000000000 --- a/metis-dereference/metis-dereference-common/src/test/java/eu/europeana/metis/dereference/IncomingRecordToEdmConverterTest.java +++ /dev/null @@ -1,35 +0,0 @@ -package eu.europeana.metis.dereference; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import eu.europeana.metis.dereference.IncomingRecordToEdmConverter; -import org.junit.jupiter.api.Test; - -class IncomingRecordToEdmConverterTest { - - - @Test - void testIsEmptyXml() { - - assertTrue(IncomingRecordToEdmConverter.isEmptyXml("")); - assertTrue(IncomingRecordToEdmConverter.isEmptyXml("")); - assertTrue( - IncomingRecordToEdmConverter.isEmptyXml("")); - assertTrue( - IncomingRecordToEdmConverter.isEmptyXml(" ")); - assertTrue(IncomingRecordToEdmConverter - .isEmptyXml("\n")); - assertTrue(IncomingRecordToEdmConverter - .isEmptyXml("\n")); - assertTrue(IncomingRecordToEdmConverter - .isEmptyXml(" \n ")); - - assertFalse(IncomingRecordToEdmConverter.isEmptyXml("A")); - assertFalse(IncomingRecordToEdmConverter.isEmptyXml( - " \n \n \n ")); - assertFalse(IncomingRecordToEdmConverter - .isEmptyXml("")); - - } -} diff --git a/metis-dereference/metis-dereference-common/src/test/java/eu/europeana/metis/dereference/IncomingRecordToEdmTransformerTest.java b/metis-dereference/metis-dereference-common/src/test/java/eu/europeana/metis/dereference/IncomingRecordToEdmTransformerTest.java new file mode 100644 index 000000000..0438de292 --- /dev/null +++ b/metis-dereference/metis-dereference-common/src/test/java/eu/europeana/metis/dereference/IncomingRecordToEdmTransformerTest.java @@ -0,0 +1,81 @@ +package eu.europeana.metis.dereference; + + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import eu.europeana.metis.exception.BadContentException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Objects; +import java.util.Optional; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +class IncomingRecordToEdmTransformerTest { + + private static final String copyXmlXsltFileName = "copy_xml.xslt"; + private static final String produceEmptyXsltFileName = "produce_empty.xslt"; + private static final String produceInvalidXmlXsltFileName = "produce_invalid_xml.xslt"; + private static final String ysoP105069FileName = "yso_p105069.xml"; + private static final String invalidXmlFileName = "invalid_xml.xml"; + + private static String copyXmlXsltString; + private static String produceEmptyXsltString; + private static String produceInvalidXmlXsltString; + private static String ysoP105069String; + private static String invalidXmlString; + + @BeforeAll + static void setUp() throws Exception { + ClassLoader classLoader = IncomingRecordToEdmTransformerTest.class.getClassLoader(); + Path path = Paths.get(Objects.requireNonNull(classLoader.getResource(copyXmlXsltFileName)).toURI()); + copyXmlXsltString = Files.readString(path, StandardCharsets.UTF_8); + + path = Paths.get(Objects.requireNonNull(classLoader.getResource(produceEmptyXsltFileName)).toURI()); + produceEmptyXsltString = Files.readString(path, StandardCharsets.UTF_8); + + path = Paths.get(Objects.requireNonNull(classLoader.getResource(produceInvalidXmlXsltFileName)).toURI()); + produceInvalidXmlXsltString = Files.readString(path, StandardCharsets.UTF_8); + + path = Paths.get(Objects.requireNonNull(classLoader.getResource(ysoP105069FileName)).toURI()); + ysoP105069String = Files.readString(path, StandardCharsets.UTF_8); + + path = Paths.get(Objects.requireNonNull(classLoader.getResource(invalidXmlFileName)).toURI()); + invalidXmlString = Files.readString(path, StandardCharsets.UTF_8); + } + + @Test + void transform() throws Exception { + IncomingRecordToEdmTransformer incomingRecordToEdmTransformer = new IncomingRecordToEdmTransformer(copyXmlXsltString); + final Optional transformedOptional = incomingRecordToEdmTransformer.transform(ysoP105069String, + "http://www.yso.fi/onto/yso/p105069"); + assertTrue(transformedOptional.isPresent()); + } + + @Test + void transform_EmptyXslt() throws Exception { + IncomingRecordToEdmTransformer incomingRecordToEdmTransformer = new IncomingRecordToEdmTransformer(produceEmptyXsltString); + final Optional transformedOptional = incomingRecordToEdmTransformer.transform(ysoP105069String, + "http://www.yso.fi/onto/yso/p105069"); + assertTrue(transformedOptional.isEmpty()); + } + + @Test + void transform_InvalidSourceXml_BadContentException() throws Exception { + IncomingRecordToEdmTransformer incomingRecordToEdmTransformer = new IncomingRecordToEdmTransformer(copyXmlXsltString); + assertThrows(BadContentException.class, () -> incomingRecordToEdmTransformer.transform(invalidXmlString, + "http://www.yso.fi/onto/yso/p105069")); + } + + @Test + void transform_InvalidXml_BadContentException() throws Exception { + IncomingRecordToEdmTransformer incomingRecordToEdmTransformer = new IncomingRecordToEdmTransformer( + produceInvalidXmlXsltString); + assertThrows(BadContentException.class, () -> incomingRecordToEdmTransformer.transform(ysoP105069String, + "http://www.yso.fi/onto/yso/p105069")); + } +} + diff --git a/metis-dereference/metis-dereference-common/src/test/resources/copy_xml.xslt b/metis-dereference/metis-dereference-common/src/test/resources/copy_xml.xslt new file mode 100644 index 000000000..28082a2d9 --- /dev/null +++ b/metis-dereference/metis-dereference-common/src/test/resources/copy_xml.xslt @@ -0,0 +1,11 @@ + + + + + + + + + + \ No newline at end of file diff --git a/metis-dereference/metis-dereference-common/src/test/resources/invalid_xml.xml b/metis-dereference/metis-dereference-common/src/test/resources/invalid_xml.xml new file mode 100644 index 000000000..7f7e28205 --- /dev/null +++ b/metis-dereference/metis-dereference-common/src/test/resources/invalid_xml.xml @@ -0,0 +1,3 @@ + + + diff --git a/metis-dereference/metis-dereference-common/src/test/resources/produce_empty.xslt b/metis-dereference/metis-dereference-common/src/test/resources/produce_empty.xslt new file mode 100644 index 000000000..777a7600d --- /dev/null +++ b/metis-dereference/metis-dereference-common/src/test/resources/produce_empty.xslt @@ -0,0 +1,4 @@ + + + \ No newline at end of file diff --git a/metis-dereference/metis-dereference-common/src/test/resources/produce_invalid_xml.xslt b/metis-dereference/metis-dereference-common/src/test/resources/produce_invalid_xml.xslt new file mode 100644 index 000000000..e81c022af --- /dev/null +++ b/metis-dereference/metis-dereference-common/src/test/resources/produce_invalid_xml.xslt @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/metis-dereference/metis-dereference-common/src/test/resources/yso_p105069.xml b/metis-dereference/metis-dereference-common/src/test/resources/yso_p105069.xml new file mode 100644 index 000000000..8998f86a4 --- /dev/null +++ b/metis-dereference/metis-dereference-common/src/test/resources/yso_p105069.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + Tjeckien + Tšekki + Czech Republic + + + + Källa för positionsinformation: Wikidata. + Location information source: Wikidata. + Sijaintitietojen lähde: Wikidata. + 1990-06-18 + 2016-05-23T16:13:36+03:00 + + + + + + + + + + Praha + Prague + Prag + 50.08333 + 14.41667 + + + + diff --git a/metis-dereference/metis-dereference-import/pom.xml b/metis-dereference/metis-dereference-import/pom.xml index 6201b2a4c..ecd559383 100644 --- a/metis-dereference/metis-dereference-import/pom.xml +++ b/metis-dereference/metis-dereference-import/pom.xml @@ -3,7 +3,7 @@ metis-dereference eu.europeana.metis - 6 + 7 4.0.0 metis-dereference-import diff --git a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactory.java b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactory.java index 2f491bb68..97eeea888 100644 --- a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactory.java +++ b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactory.java @@ -1,9 +1,13 @@ package eu.europeana.metis.dereference.vocimport; import eu.europeana.metis.dereference.vocimport.model.Location; +import eu.europeana.metis.exception.BadContentException; import java.io.IOException; import java.io.InputStream; +import java.net.MalformedURLException; import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; @@ -13,15 +17,14 @@ public class VocabularyCollectionImporterFactory { /** - * Create a vocabulary importer for remote web addresses, indicated by instances of {@link URI}. - * Note that this method can only be used for locations that are also a valid {@link - * java.net.URL}. + * Create a vocabulary importer for remote web addresses, indicated by instances of {@link URI}. Note that this method can only + * be used for locations that are also a valid {@link java.net.URL}. * * @param directoryLocation The location of the directory to import. * @return A vocabulary importer. */ - public VocabularyCollectionImporter createImporter(URI directoryLocation) { - return new VocabularyCollectionImporterImpl(new UriLocation(directoryLocation)); + public VocabularyCollectionImporter createImporter(URL directoryLocation) { + return new VocabularyCollectionImporterImpl(new UrlLocation(directoryLocation)); } /** @@ -35,9 +38,8 @@ public VocabularyCollectionImporter createImporter(Path directoryLocation) { } /** - * Create a vocabulary importer for local files, indicated by instances of {@link Path}. This - * method provides a way to set a base directory that will be assumed known (so that output and - * logs will only include the relative location). + * Create a vocabulary importer for local files, indicated by instances of {@link Path}. This method provides a way to set a + * base directory that will be assumed known (so that output and logs will only include the relative location). * * @param baseDirectory The base directory of the project or collection. Can be null. * @param directoryLocation The full location of the directory file to import. @@ -47,27 +49,32 @@ public VocabularyCollectionImporter createImporter(Path baseDirectory, Path dire return new VocabularyCollectionImporterImpl(new PathLocation(baseDirectory, directoryLocation)); } - private static final class UriLocation implements Location { + private static final class UrlLocation implements Location { - private final URI uri; + private final URL url; - UriLocation(URI uri) { - this.uri = uri; + UrlLocation(URL url) { + this.url = url; } @Override public InputStream read() throws IOException { - return uri.toURL().openStream(); + return url.openStream(); } @Override - public Location resolve(String relativeLocation) { - return new UriLocation(uri.resolve(relativeLocation)); + public Location resolve(String relativeLocation) throws BadContentException { + try { + return new UrlLocation(url.toURI().resolve(relativeLocation).toURL()); + } catch (URISyntaxException | MalformedURLException e) { + throw new BadContentException( + String.format("Provided url '%s' and relative location %s, failed to parse.", url, relativeLocation), e); + } } @Override public String toString() { - return uri.toString(); + return url.toString(); } } diff --git a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterImpl.java b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterImpl.java index 02bbcb5dc..733305b98 100644 --- a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterImpl.java +++ b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterImpl.java @@ -9,6 +9,7 @@ import eu.europeana.metis.dereference.vocimport.model.VocabularyDirectoryEntry; import eu.europeana.metis.dereference.vocimport.model.VocabularyLoader; import eu.europeana.metis.dereference.vocimport.model.VocabularyMetadata; +import eu.europeana.metis.exception.BadContentException; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -21,7 +22,7 @@ */ final class VocabularyCollectionImporterImpl implements VocabularyCollectionImporter { - private Location directoryLocation; + private final Location directoryLocation; VocabularyCollectionImporterImpl(Location directoryLocation) { this.directoryLocation = directoryLocation; @@ -33,18 +34,27 @@ public Iterable importVocabularies() throws VocabularyImportEx // Obtain the directory entries. final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); final VocabularyDirectoryEntry[] directoryEntries; + try (final InputStream input = directoryLocation.read()) { directoryEntries = mapper.readValue(input, VocabularyDirectoryEntry[].class); } catch (IOException e) { throw new VocabularyImportException( - "Could not read vocabulary directory at [" + directoryLocation + "].", e); + "Could not read vocabulary directory at [" + directoryLocation + "].", e); } // Compile the vocabulary loaders final List result = new ArrayList<>(directoryEntries.length); for (VocabularyDirectoryEntry entry : directoryEntries) { - final Location metadataLocation = directoryLocation.resolve(entry.getMetadata()); - final Location mappingLocation = directoryLocation.resolve(entry.getMapping()); + final Location metadataLocation; + final Location mappingLocation; + try { + metadataLocation = directoryLocation.resolve(entry.getMetadata()); + mappingLocation = directoryLocation.resolve(entry.getMapping()); + } catch (BadContentException e) { + throw new VocabularyImportException( + String.format("Could not read vocabulary directory at [%s] and entry metadata [%s], entry mapping [%s].", + directoryLocation, entry.getMetadata(), entry.getMapping()), e); + } result.add(() -> loadVocabulary(metadataLocation, mappingLocation, mapper)); } @@ -53,7 +63,7 @@ public Iterable importVocabularies() throws VocabularyImportEx } private Vocabulary loadVocabulary(Location metadataLocation, Location mappingLocation, - ObjectMapper mapper) throws VocabularyImportException { + ObjectMapper mapper) throws VocabularyImportException { // Read the metadata file. final VocabularyMetadata metadata; @@ -61,7 +71,7 @@ private Vocabulary loadVocabulary(Location metadataLocation, Location mappingLoc metadata = mapper.readValue(input, VocabularyMetadata.class); } catch (IOException e) { throw new VocabularyImportException( - "Could not read vocabulary metadata at [" + metadataLocation + "].", e); + "Could not read vocabulary metadata at [" + metadataLocation + "].", e); } // Read the mapping file. @@ -70,22 +80,22 @@ private Vocabulary loadVocabulary(Location metadataLocation, Location mappingLoc mapping = IOUtils.toString(input, StandardCharsets.UTF_8); } catch (IOException e) { throw new VocabularyImportException( - "Could not read vocabulary mapping at [" + mappingLocation + "].", e); + "Could not read vocabulary mapping at [" + mappingLocation + "].", e); } // Compile the vocabulary. return Vocabulary.builder() - .setName(metadata.getName()) - .setTypes(metadata.getTypes()) - .setPaths(metadata.getPaths()) - .setParentIterations(metadata.getParentIterations()) - .setSuffix(metadata.getSuffix()) - .setExamples(metadata.getExamples()) - .setCounterExamples(metadata.getCounterExamples()) - .setTransformation(mapping) - .setReadableMetadataLocation(metadataLocation.toString()) - .setReadableMappingLocation(mappingLocation.toString()) - .build(); + .setName(metadata.getName()) + .setTypes(metadata.getTypes()) + .setPaths(metadata.getPaths()) + .setParentIterations(metadata.getParentIterations()) + .setSuffix(metadata.getSuffix()) + .setExamples(metadata.getExamples()) + .setCounterExamples(metadata.getCounterExamples()) + .setTransformation(mapping) + .setReadableMetadataLocation(metadataLocation.toString()) + .setReadableMappingLocation(mappingLocation.toString()) + .build(); } public Location getDirectoryLocation() { diff --git a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionMavenRule.java b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionMavenRule.java index de4bbd747..68195cf59 100644 --- a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionMavenRule.java +++ b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionMavenRule.java @@ -8,6 +8,7 @@ import org.apache.maven.plugin.logging.Log; import org.apache.maven.project.MavenProject; import org.codehaus.plexus.component.repository.exception.ComponentLookupException; +import org.springframework.stereotype.Component; /** * This is a Maven-enabled enforcer rule that can be used in a maven project. For an example of how @@ -44,6 +45,7 @@ * * } */ +@Component public class VocabularyCollectionMavenRule implements EnforcerRule { /** @@ -69,6 +71,8 @@ public class VocabularyCollectionMavenRule implements EnforcerRule { */ private String vocabularyDirectoryFile = null; + private final VocabularyCollectionImporterFactory vocabularyCollectionImporterFactory = new VocabularyCollectionImporterFactory(); + /** * No-arguments constructor, required for maven instantiation. */ @@ -113,8 +117,9 @@ public void execute(EnforcerRuleHelper enforcerRuleHelper) throws EnforcerRuleEx final Path baseDirectory = project.getBasedir().toPath(); final Path vocabularyDirectory = baseDirectory.resolve(vocabularyDirectoryFile); + try { // Prepare validation - final VocabularyCollectionImporter importer = new VocabularyCollectionImporterFactory() + final VocabularyCollectionImporter importer = vocabularyCollectionImporterFactory .createImporter(baseDirectory, vocabularyDirectory); final VocabularyCollectionValidatorImpl validator = new VocabularyCollectionValidatorImpl( importer, lenientOnLackOfExamples, lenientOnMappingTestFailures, @@ -123,7 +128,7 @@ public void execute(EnforcerRuleHelper enforcerRuleHelper) throws EnforcerRuleEx log.info("Validating vocabulary collection: " + importer.getDirectoryLocation().toString()); // Perform validation - try { + validator.validate(vocabulary -> log.info(" Vocabulary found: " + vocabulary.getName()), log::warn); } catch (VocabularyImportException e) { diff --git a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionValidatorImpl.java b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionValidatorImpl.java index dfa92b86b..36b4ef706 100644 --- a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionValidatorImpl.java +++ b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionValidatorImpl.java @@ -1,22 +1,27 @@ package eu.europeana.metis.dereference.vocimport; import eu.europeana.enrichment.utils.EnrichmentBaseConverter; -import eu.europeana.metis.dereference.IncomingRecordToEdmConverter; +import eu.europeana.metis.dereference.IncomingRecordToEdmTransformer; import eu.europeana.metis.dereference.RdfRetriever; import eu.europeana.metis.dereference.vocimport.exception.VocabularyImportException; import eu.europeana.metis.dereference.vocimport.model.Vocabulary; import eu.europeana.metis.dereference.vocimport.model.VocabularyLoader; import eu.europeana.metis.dereference.vocimport.utils.NonCollidingPathVocabularyTrie; +import eu.europeana.metis.exception.BadContentException; import java.io.IOException; import java.net.URISyntaxException; import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.function.Consumer; import javax.xml.bind.JAXBException; +import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; -import org.apache.commons.lang3.StringUtils; +/** + * Class that contains functionality to validate vocabularies using a {@link VocabularyCollectionImporter}. + */ public class VocabularyCollectionValidatorImpl implements VocabularyCollectionValidator { private final VocabularyCollectionImporter importer; @@ -28,16 +33,15 @@ public class VocabularyCollectionValidatorImpl implements VocabularyCollectionVa * Constructor. * * @param importer Vocabulary importer. - * @param lenientOnLackOfExamples Whether the the validator is lenient on vocabulary mappings - * without examples. - * @param lenientOnMappingTestFailures Whether the validator is lenient on errors and unmet - * expectations when applying the mapping to the example and counterexample values. - * @param lenientOnExampleRetrievalFailures Whether the validator is lenient on example or - * counterexample retrieval (download) issues. + * @param lenientOnLackOfExamples Whether the the validator is lenient on vocabulary mappings without examples. + * @param lenientOnMappingTestFailures Whether the validator is lenient on errors and unmet expectations when applying the + * mapping to the example and counterexample values. + * @param lenientOnExampleRetrievalFailures Whether the validator is lenient on example or counterexample retrieval (download) + * issues. */ public VocabularyCollectionValidatorImpl(VocabularyCollectionImporter importer, - boolean lenientOnLackOfExamples, boolean lenientOnMappingTestFailures, - boolean lenientOnExampleRetrievalFailures) { + boolean lenientOnLackOfExamples, boolean lenientOnMappingTestFailures, + boolean lenientOnExampleRetrievalFailures) { this.importer = importer; this.lenientOnLackOfExamples = lenientOnLackOfExamples; this.lenientOnMappingTestFailures = lenientOnMappingTestFailures; @@ -46,25 +50,23 @@ public VocabularyCollectionValidatorImpl(VocabularyCollectionImporter importer, @Override public void validate(Consumer vocabularyReceiver, Consumer warningReceiver) - throws VocabularyImportException { + throws VocabularyImportException { validateInternal(vocabularyReceiver, warningReceiver, true); } @Override - public void validateVocabularyOnly(Consumer vocabularyReceiver) - throws VocabularyImportException { + public void validateVocabularyOnly(Consumer vocabularyReceiver) throws VocabularyImportException { validateInternal(vocabularyReceiver, null, false); } private void validateInternal(Consumer vocabularyReceiver, - Consumer warningReceiver, boolean validateExamples) - throws VocabularyImportException { + Consumer warningReceiver, boolean validateExamples) throws VocabularyImportException { final DuplicationChecker duplicationChecker = new DuplicationChecker(); final Iterable vocabularyLoaders = importer.importVocabularies(); for (VocabularyLoader loader : vocabularyLoaders) { final Vocabulary vocabulary = loader.load(); - final IncomingRecordToEdmConverter converter = validateVocabulary(vocabulary, - duplicationChecker); + final IncomingRecordToEdmTransformer converter = validateVocabulary(vocabulary, + duplicationChecker); if (validateExamples) { validateExamples(vocabulary, warningReceiver, converter); } @@ -72,29 +74,29 @@ private void validateInternal(Consumer vocabularyReceiver, } } - private IncomingRecordToEdmConverter validateVocabulary(Vocabulary vocabulary, - DuplicationChecker duplicationChecker) throws VocabularyImportException { + private IncomingRecordToEdmTransformer validateVocabulary(Vocabulary vocabulary, + DuplicationChecker duplicationChecker) throws VocabularyImportException { // Check the presence of the required fields. if (vocabulary.getName() == null) { throw new VocabularyImportException( - String.format("No vocabulary name given in metadata at [%s].", - vocabulary.getReadableMetadataLocation())); + String.format("No vocabulary name given in metadata at [%s].", + vocabulary.getReadableMetadataLocation())); } if (vocabulary.getTypes().isEmpty()) { throw new VocabularyImportException( - String.format("No vocabulary type(s) given in metadata at [%s].", - vocabulary.getReadableMetadataLocation())); + String.format("No vocabulary type(s) given in metadata at [%s].", + vocabulary.getReadableMetadataLocation())); } if (vocabulary.getPaths().isEmpty()) { throw new VocabularyImportException( - String.format("No vocabulary path(s) given in metadata at [%s].", - vocabulary.getReadableMetadataLocation())); + String.format("No vocabulary path(s) given in metadata at [%s].", + vocabulary.getReadableMetadataLocation())); } if (vocabulary.getTransformation() == null) { throw new VocabularyImportException( - String.format("No transformation given in mapping at [%s].", - vocabulary.getReadableMappingLocation())); + String.format("No transformation given in mapping at [%s].", + vocabulary.getReadableMappingLocation())); } // Check whether name and links are unique. @@ -102,21 +104,21 @@ private IncomingRecordToEdmConverter validateVocabulary(Vocabulary vocabulary, // Verifying the xslt - compile it. try { - return new IncomingRecordToEdmConverter(vocabulary.getTransformation()); - } catch (TransformerException e) { + return new IncomingRecordToEdmTransformer(vocabulary.getTransformation()); + } catch (TransformerException | ParserConfigurationException e) { throw new VocabularyImportException( - String.format("Error in the transformation given in mapping at [%s].", - vocabulary.getReadableMappingLocation()), e); + String.format("Error in the transformation given in mapping at [%s].", + vocabulary.getReadableMappingLocation()), e); } } private void validateExamples(Vocabulary vocabulary, Consumer warningReceiver, - IncomingRecordToEdmConverter converter) throws VocabularyImportException { + IncomingRecordToEdmTransformer converter) throws VocabularyImportException { // Testing the examples (if there are any - otherwise issue warning). if (vocabulary.getExamples().isEmpty()) { final String message = String.format("No examples specified for metadata at [%s].", - vocabulary.getReadableMetadataLocation()); + vocabulary.getReadableMetadataLocation()); if (lenientOnLackOfExamples) { warningReceiver.accept(message); } else { @@ -125,26 +127,26 @@ private void validateExamples(Vocabulary vocabulary, Consumer warningRec } for (String example : vocabulary.getExamples()) { testExample(converter, example, vocabulary.getSuffix(), false, - vocabulary.getReadableMetadataLocation(), warningReceiver); + vocabulary.getReadableMetadataLocation(), warningReceiver); } // Testing the counter examples (if there are any). for (String example : vocabulary.getCounterExamples()) { testExample(converter, example, vocabulary.getSuffix(), true, - vocabulary.getReadableMetadataLocation(), warningReceiver); + vocabulary.getReadableMetadataLocation(), warningReceiver); } } private String getTestErrorMessage(String example, boolean isCounterExample, - String readableMetadataLocation, String sentenceContinuation, Exception exception) { + String readableMetadataLocation, String sentenceContinuation, Exception exception) { final String sentence = String.format("%s '%s' in metadata at [%s] %s.", - isCounterExample ? "Counterexample" : "Example", example, readableMetadataLocation, - sentenceContinuation); - return sentence + (exception == null ? "" : " Error: " + exception.getMessage()); + isCounterExample ? "Counterexample" : "Example", example, readableMetadataLocation, + sentenceContinuation); + return sentence + (exception == null ? "" : String.format(" Error: %s", exception.getMessage())); } private void processTestError(String message, boolean isWarning, Consumer warningReceiver, - Exception originalException) throws VocabularyImportException { + Exception originalException) throws VocabularyImportException { if (isWarning) { warningReceiver.accept(message); } else { @@ -152,9 +154,9 @@ private void processTestError(String message, boolean isWarning, Consumer warningReceiver) throws VocabularyImportException { + private void testExample(IncomingRecordToEdmTransformer incomingRecordToEdmTransformer, String example, String suffix, + boolean isCounterExample, String readableMetadataLocation, + Consumer warningReceiver) throws VocabularyImportException { // Retrieve the example - is not null. final String exampleContent; @@ -162,40 +164,40 @@ private void testExample(IncomingRecordToEdmConverter converter, String example, exampleContent = new RdfRetriever().retrieve(example, suffix); } catch (IOException | URISyntaxException e) { final String message = getTestErrorMessage(example, isCounterExample, - readableMetadataLocation, "could not be retrieved", e); + readableMetadataLocation, "could not be retrieved", e); processTestError(message, lenientOnExampleRetrievalFailures, warningReceiver, e); return; } // Convert the example - final String result; + final Optional result; try { - result = converter.convert(exampleContent, example); - } catch (TransformerException e) { + result = incomingRecordToEdmTransformer.transform(exampleContent, example); + } catch (BadContentException e) { final String message = getTestErrorMessage(example, isCounterExample, - readableMetadataLocation, "could not be mapped", e); + readableMetadataLocation, "could not be mapped", e); processTestError(message, lenientOnMappingTestFailures, warningReceiver, e); return; } // Check whether the example yielded a mapped entity or not - if (StringUtils.isNotBlank(result) && isCounterExample) { + if (result.isPresent() && isCounterExample) { final String message = getTestErrorMessage(example, isCounterExample, - readableMetadataLocation, "yielded a mapped result, but is expected not to", null); + readableMetadataLocation, "yielded a mapped result, but is expected not to", null); processTestError(message, lenientOnMappingTestFailures, warningReceiver, null); - } else if (StringUtils.isBlank(result) && !isCounterExample) { + } else if (result.isEmpty() && !isCounterExample) { final String message = getTestErrorMessage(example, isCounterExample, - readableMetadataLocation, "did not yield a mapped result, but is expected to", null); + readableMetadataLocation, "did not yield a mapped result, but is expected to", null); processTestError(message, lenientOnMappingTestFailures, warningReceiver, null); } // Check whether the example yielded valid XML - if (StringUtils.isNotBlank(result)) { + if (result.isPresent()) { try { - EnrichmentBaseConverter.convertToEnrichmentBase(result); + EnrichmentBaseConverter.convertToEnrichmentBase(result.get()); } catch (JAXBException e) { final String message = getTestErrorMessage(example, isCounterExample, - readableMetadataLocation, "did not yield a valid XML", e); + readableMetadataLocation, "did not yield a valid XML", e); throw new VocabularyImportException(message, e); } } @@ -213,11 +215,11 @@ void checkAndRegister(Vocabulary vocabulary) throws VocabularyImportException { // Handle the name uniqueness final String nameToCheck = vocabulary.getName().trim().replaceAll("\\s", " ") - .toLowerCase(Locale.ENGLISH); + .toLowerCase(Locale.ENGLISH); if (knownNames.containsKey(nameToCheck)) { final String message = String.format("Duplicate name '%s' detected in metadata at [%s]:" - + " metadata at [%s] contains a name that is similar.", vocabulary.getName(), - vocabulary.getReadableMetadataLocation(), knownNames.get(nameToCheck)); + + " metadata at [%s] contains a name that is similar.", vocabulary.getName(), + vocabulary.getReadableMetadataLocation(), knownNames.get(nameToCheck)); throw new VocabularyImportException(message); } knownNames.put(nameToCheck, vocabulary.getReadableMetadataLocation()); diff --git a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/model/Location.java b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/model/Location.java index f6970584b..f2a1039da 100644 --- a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/model/Location.java +++ b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/model/Location.java @@ -1,5 +1,6 @@ package eu.europeana.metis.dereference.vocimport.model; +import eu.europeana.metis.exception.BadContentException; import java.io.IOException; import java.io.InputStream; @@ -14,14 +15,14 @@ public interface Location { InputStream read() throws IOException; /** - * Resolve a relative location against the given location. The given location can be assumed to be - * a file (as opposed to a path/directory) so that essentially the relative location is resolved - * against the parent of the given location. + * Resolve a relative location against the given location. The given location can be assumed to be a file (as opposed to a + * path/directory) so that essentially the relative location is resolved against the parent of the given location. * * @param relativeLocation The relative location to resolve. * @return The resolved location. + * @throws BadContentException if the resolve did not succeed */ - Location resolve(String relativeLocation); + Location resolve(String relativeLocation) throws BadContentException; /** * @return A human-readable representation of the location. diff --git a/metis-dereference/metis-dereference-rest/pom.xml b/metis-dereference/metis-dereference-rest/pom.xml index 436e5bed8..0d67b48d9 100644 --- a/metis-dereference/metis-dereference-rest/pom.xml +++ b/metis-dereference/metis-dereference-rest/pom.xml @@ -4,7 +4,7 @@ metis-dereference eu.europeana.metis - 6 + 7 metis-dereference-rest war @@ -69,6 +69,17 @@ springfox-swagger-ui ${version.swagger}     + + + org.springframework.boot + spring-boot-autoconfigure + ${version.spring-boot-autoconfigure} + com.jayway.jsonpath json-path-assert diff --git a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/DereferencingManagementController.java b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/DereferencingManagementController.java index 6d45846b3..3a8e4f426 100644 --- a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/DereferencingManagementController.java +++ b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/DereferencingManagementController.java @@ -1,17 +1,23 @@ package eu.europeana.metis.dereference.rest; -import eu.europeana.metis.utils.RestEndpoints; import eu.europeana.metis.dereference.Vocabulary; import eu.europeana.metis.dereference.service.DereferencingManagementService; import eu.europeana.metis.dereference.vocimport.exception.VocabularyImportException; +import eu.europeana.metis.exception.BadContentException; +import eu.europeana.metis.utils.RestEndpoints; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import io.swagger.annotations.ApiResponse; import io.swagger.annotations.ApiResponses; +import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; +import java.net.URL; +import java.util.HashSet; import java.util.List; +import java.util.Optional; +import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -34,10 +40,16 @@ public class DereferencingManagementController { private static final Logger LOGGER = LoggerFactory.getLogger(DereferencingManagementController.class); private final DereferencingManagementService service; + private final Set allowedUrlDomains; + /** + * @param service the dereferencing management service + * @param allowedUrlDomains the allowed valid url prefixes + */ @Autowired - public DereferencingManagementController(DereferencingManagementService service) { + public DereferencingManagementController(DereferencingManagementService service, Set allowedUrlDomains) { this.service = service; + this.allowedUrlDomains = new HashSet<>(allowedUrlDomains); } /** @@ -53,9 +65,8 @@ public List getAllVocabularies() { } /** - * Empty Cache. This will remove ALL entries in the cache (Redis). If the same redis - * instance/cluster is used for multiple services then the cache for other services is cleared as - * well. + * Empty Cache. This will remove ALL entries in the cache (Redis). If the same redis instance/cluster is used for multiple + * services then the cache for other services is cleared as well. */ @DeleteMapping(value = RestEndpoints.CACHE_EMPTY) @ResponseBody @@ -64,24 +75,65 @@ public void emptyCache() { service.emptyCache(); } + /** + * Empty the cache for all Resources without an XML representation + * */ + @DeleteMapping(value = RestEndpoints.CACHE_EMPTY_XML) + @ResponseBody + @ApiOperation(value = "Empty the cache without XML representations") + public void emptyCacheByEmptyXml() { + service.purgeByNullOrEmptyXml(); + } + + /** + * Empty the cache for a specific resource + * @param resourceId The resourceId to empty the cache for + * */ + @DeleteMapping(value = RestEndpoints.CACHE_EMPTY_RESOURCE) + @ResponseBody + @ApiOperation(value = "Empty the cache by resource Id") + public void emptyCacheByResourceId( + @ApiParam(value = "Id (URI) of resource to clear cache", required = true) @RequestParam(value = "resourceId") String resourceId) { + service.purgeByResourceId(resourceId); + } + + /** + * Empty the cache for a specific vocabulary, with all associated entities + * @param vocabularyId The vocabularyId to empty the cache for + * */ + @DeleteMapping(value = RestEndpoints.CACHE_EMPTY_VOCABULARY) + @ResponseBody + @ApiOperation(value = "Empty the cache by vocabulary Id") + public void emptyCacheByVocabularyId( + @ApiParam(value = "Id of vocabulary to clear cache", required = true) @RequestParam(value = "vocabularyId") String vocabularyId) { + service.purgeByVocabularyId(vocabularyId); + } + + /** * Load the vocabularies from an online source. This does NOT purge the cache. * - * @param directoryUrl The online location of the vocabulary directory. + * @param directoryUrl The online location of the vocabulary directory + * @return sting containing an error message otherwise empty */ @PostMapping(value = RestEndpoints.LOAD_VOCABULARIES) @ResponseBody @ApiOperation(value = "Load and replace the vocabularies listed by the given vocabulary directory. Does NOT purge the cache.") @ApiResponses(value = { - @ApiResponse(code = 200, message = "Vocabularies loaded successfully."), - @ApiResponse(code = 400, message = "Bad request parameters."), - @ApiResponse(code = 502, message = "Problem accessing vocabulary repository.") - }) public ResponseEntity loadVocabularies( - @ApiParam("directory_url") @RequestParam("directory_url") String directoryUrl) { + @ApiResponse(code = 200, message = "Vocabularies loaded successfully."), + @ApiResponse(code = 400, message = "Bad request parameters."), + @ApiResponse(code = 502, message = "Problem accessing vocabulary repository.") + }) + public ResponseEntity loadVocabularies( + @ApiParam("directory_url") @RequestParam("directory_url") String directoryUrl) { try { - service.loadVocabularies(new URI(directoryUrl)); - return ResponseEntity.ok().build(); - } catch (URISyntaxException e) { + final Optional validatedLocationUrl = getValidatedLocationUrl(directoryUrl); + if (validatedLocationUrl.isPresent()) { + service.loadVocabularies(validatedLocationUrl.get()); + return ResponseEntity.ok().build(); + } + return ResponseEntity.badRequest().body("The url of the directory to import is not valid."); + } catch (BadContentException e) { LOGGER.warn("Could not load vocabularies", e); return ResponseEntity.badRequest().body(e.getMessage()); } catch (VocabularyImportException e) { @@ -89,4 +141,34 @@ public void emptyCache() { return ResponseEntity.status(HttpStatus.BAD_GATEWAY).body(e.getMessage()); } } + + /** + * Validates a String representation of a URL. + *

    The method will check that the url is: + *

      + *
    • valid according to the protocol
      • + *
    • of https scheme
      • + *
    • part of the allowed domains
      • + *
    + * domain for the application to further access it.

    + * + * @param directoryUrl the url to validate + * @return the validated URL class + * @throws BadContentException if the url failed during parsing + */ + private Optional getValidatedLocationUrl(String directoryUrl) throws BadContentException { + try { + URI uri = new URI(directoryUrl); + String scheme = uri.getScheme(); + String remoteHost = uri.getHost(); + + if ("https".equals(scheme) && allowedUrlDomains.contains(remoteHost)) { + return Optional.of(uri.toURL()); + } + } catch (URISyntaxException | MalformedURLException e) { + throw new BadContentException(String.format("Provided directoryUrl '%s', failed to parse.", directoryUrl), e); + } + + return Optional.empty(); + } } diff --git a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java index 819a647bd..653edd953 100644 --- a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java +++ b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java @@ -7,6 +7,7 @@ import eu.europeana.metis.mongo.connection.MongoClientProvider; import eu.europeana.metis.mongo.connection.MongoProperties; import java.util.Collections; +import java.util.Set; import javax.annotation.PreDestroy; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Value; @@ -15,6 +16,8 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.web.servlet.config.annotation.EnableWebMvc; import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; import org.springframework.web.servlet.config.annotation.ViewControllerRegistry; @@ -31,6 +34,7 @@ * Spring configuration class Created by ymamakis on 12-2-16. */ @Configuration +@EnableScheduling @ComponentScan(basePackages = {"eu.europeana.metis.dereference.rest", "eu.europeana.metis.dereference.rest.exceptions"}) @PropertySource("classpath:dereferencing.properties") @@ -66,6 +70,9 @@ public class Application implements WebMvcConfigurer, InitializingBean { @Value("${vocabulary.db}") private String vocabularyDb; + //Valid directories list + @Value("${allowed.url.domains}") + private String[] allowedUrlDomains; private MongoClient mongoClientEntity; private MongoClient mongoClientVocabulary; @@ -116,11 +123,34 @@ VocabularyDao getVocabularyDao() { return new VocabularyDao(getVocabularyMongoClient(), vocabularyDb); } + @Bean + Set getAllowedUrlDomains() { + return Set.of(allowedUrlDomains); + } + @Bean public static PropertySourcesPlaceholderConfigurer propertySourcesPlaceholderConfigurer() { return new PropertySourcesPlaceholderConfigurer(); } + /** + * Empty Cache with XML entries null or empty. + * This will remove entries with null or empty XML in the cache (Redis). If the same redis instance/cluster is used for multiple + * services then the cache for other services is cleared as well. + * This task is scheduled by a cron expression. + */ + + @Scheduled(cron = "${dereference.purge.emptyxml.frequency}") + public void dereferenceCacheNullOrEmpty(){ getProcessedEntityDao().purgeByNullOrEmptyXml(); } + + /** + * Empty Cache. This will remove ALL entries in the cache (Redis). If the same redis instance/cluster is used for multiple + * services then the cache for other services is cleared as well. + * This task is scheduled by a cron expression. + */ + @Scheduled(cron = "${dereference.purge.all.frequency}") + public void dereferenceCachePurgeAll(){ getProcessedEntityDao().purgeAll(); } + /** * Closes any connections previous acquired. */ diff --git a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/ServletInitializer.java b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/ServletInitializer.java index 20d9dc309..22baa6a3a 100644 --- a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/ServletInitializer.java +++ b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/ServletInitializer.java @@ -1,8 +1,9 @@ package eu.europeana.metis.dereference.rest.config; +import eu.europeana.metis.dereference.RdfRetriever; import eu.europeana.metis.dereference.service.MongoDereferenceService; import eu.europeana.metis.dereference.service.MongoDereferencingManagementService; -import eu.europeana.metis.dereference.RdfRetriever; +import eu.europeana.metis.dereference.vocimport.VocabularyCollectionImporterFactory; import org.springframework.util.ClassUtils; import org.springframework.web.context.WebApplicationContext; import org.springframework.web.context.support.AnnotationConfigWebApplicationContext; @@ -19,6 +20,7 @@ protected WebApplicationContext createServletApplicationContext() { context.scan(ClassUtils.getPackageName(getClass())); context.register(MongoDereferenceService.class); context.register(MongoDereferencingManagementService.class); + context.register(VocabularyCollectionImporterFactory.class); context.register(RdfRetriever.class); return context; diff --git a/metis-dereference/metis-dereference-rest/src/main/resources/dereferencing.properties.example b/metis-dereference/metis-dereference-rest/src/main/resources/dereferencing.properties.example index 3ee045635..e4be549ec 100644 --- a/metis-dereference/metis-dereference-rest/src/main/resources/dereferencing.properties.example +++ b/metis-dereference/metis-dereference-rest/src/main/resources/dereferencing.properties.example @@ -12,4 +12,13 @@ mongo.username= mongo.password= mongo.application.name= entity.db= -vocabulary.db= \ No newline at end of file +vocabulary.db= + +#The allowed domains for vocabularies loading without the scheme(always validated against https). e.g. raw.githubusercontent.com +allowed.url.domains= + +# Dereferencing cache cron expressions, +# refer to Spring Framework CronExpression for documentation +dereference.purge.all.frequency=@monthly +# purge empty xml +dereference.purge.emptyxml.frequency=@daily \ No newline at end of file diff --git a/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingControllerTest.java b/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingControllerTest.java index ec784c9f1..1e3d37c88 100644 --- a/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingControllerTest.java +++ b/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingControllerTest.java @@ -4,6 +4,7 @@ import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.xpath; @@ -12,6 +13,8 @@ import eu.europeana.enrichment.api.external.model.Label; import eu.europeana.metis.dereference.rest.exceptions.RestResponseExceptionHandler; import eu.europeana.metis.dereference.service.DereferenceService; +import eu.europeana.metis.utils.RestEndpoints; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -24,6 +27,9 @@ import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.setup.MockMvcBuilders; +/** + * Unit test {@link DereferencingController} Class + */ class DereferencingControllerTest { private DereferenceService dereferenceServiceMock; @@ -35,62 +41,74 @@ void setUp() { dereferenceServiceMock = mock(DereferenceService.class); namespaceMap = getNamespaceMap(); - DereferencingController dereferenceController = new DereferencingController( - dereferenceServiceMock); + DereferencingController dereferenceController = new DereferencingController(dereferenceServiceMock); dereferencingControllerMock = MockMvcBuilders.standaloneSetup(dereferenceController) - .setControllerAdvice(new RestResponseExceptionHandler()).build(); + .setControllerAdvice(new RestResponseExceptionHandler()).build(); } @Test - void dereferenceGet_outputXML() throws Exception { - when(dereferenceServiceMock.dereference("http://www.example.com")) - .thenReturn(Collections.singletonList(getAgent("http://www.example.com"))); + void dereferenceGet_outputXML_expectSuccess() throws Exception { + when(dereferenceServiceMock.dereference("http://www.example.com")).thenReturn( + Collections.singletonList(getAgent("http://www.example.com"))); dereferencingControllerMock.perform( - get("/dereference/?uri=http://www.example.com").accept(MediaType.APPLICATION_XML_VALUE)) - .andExpect(status().is(200)) - // .andExpect(content().string("")) - .andExpect(xpath("metis:results/metis:result/edm:Agent/@rdf:about", namespaceMap) - .string("http://www.example.com")).andExpect(xpath( - "metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='en']", - namespaceMap).string("labelEn")).andExpect(xpath( - "metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='nl']", - namespaceMap).string("labelNl")).andExpect(xpath( - "metis:results/metis:result/edm:Agent/rdaGr2:dateOfBirth[@xml:lang='en']", - namespaceMap).string("10-10-10")); + get(RestEndpoints.DEREFERENCE + "/?uri=http://www.example.com").accept(MediaType.APPLICATION_XML_VALUE)) + .andExpect(status().is(200)).andExpect( + xpath("metis:results/metis:result/edm:Agent/@rdf:about", namespaceMap).string("http://www.example.com")).andExpect( + xpath("metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='en']", namespaceMap).string("labelEn")).andExpect( + xpath("metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='nl']", namespaceMap).string("labelNl")).andExpect( + xpath("metis:results/metis:result/edm:Agent/rdaGr2:dateOfBirth[@xml:lang='en']", namespaceMap).string("10-10-10")); } @Test - void dereferencePost_outputXML() throws Exception { - when(dereferenceServiceMock.dereference("http://www.example.com")) - .thenReturn(Collections.singletonList(getAgent("http://www.example.com"))); - - dereferencingControllerMock.perform(post("/dereference").accept(MediaType.APPLICATION_XML_VALUE) - .contentType(MediaType.APPLICATION_JSON).content("[ \"http://www.example.com\" ]")) - .andExpect(status().is(200)) - // .andExpect(content().string("")) - .andExpect(xpath("metis:results/metis:result/edm:Agent/@rdf:about", - namespaceMap).string("http://www.example.com")) - .andExpect(xpath( - "metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='en']", - namespaceMap).string("labelEn")) - .andExpect(xpath( - "metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='nl']", - namespaceMap).string("labelNl")) - .andExpect(xpath( - "metis:results/metis:result/edm:Agent/rdaGr2:dateOfBirth[@xml:lang='en']", - namespaceMap).string("10-10-10")); + void dereferenceGet_outputXML_expectInternalServerError() throws Exception { + when(dereferenceServiceMock.dereference("http://www.example.com")).thenThrow( + new URISyntaxException("URI Error", "Error reason")); + + dereferencingControllerMock.perform( + get(RestEndpoints.DEREFERENCE + "/?uri=http://www.example.com").accept(MediaType.APPLICATION_XML_VALUE)).andDo(print()) + .andExpect(status().is(500)).andExpect(xpath("//error").exists()) + .andExpect(xpath("//error/errorMessage").exists()).andExpect(xpath("//error/errorMessage").string( + "Dereferencing failed for uri: http://www.example.com with root cause: Error reason: URI Error")); + } + + @Test + void dereferencePost_outputXML_expectSuccess() throws Exception { + when(dereferenceServiceMock.dereference("http://www.example.com")).thenReturn( + Collections.singletonList(getAgent("http://www.example.com"))); + + dereferencingControllerMock.perform( + post(RestEndpoints.DEREFERENCE).accept(MediaType.APPLICATION_XML_VALUE).contentType(MediaType.APPLICATION_JSON) + .content("[ \"http://www.example.com\" ]")).andDo(print()).andExpect(status().is(200)) + .andExpect(xpath("metis:results/metis:result/edm:Agent/@rdf:about", namespaceMap).string( + "http://www.example.com")).andExpect( + xpath("metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='en']", namespaceMap).string("labelEn")).andExpect( + xpath("metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='nl']", namespaceMap).string("labelNl")).andExpect( + xpath("metis:results/metis:result/edm:Agent/rdaGr2:dateOfBirth[@xml:lang='en']", namespaceMap).string("10-10-10")); + } + + @Test + void dereferencePost_outputXML_expectEmptyList() throws Exception { + when(dereferenceServiceMock.dereference("http://www.example.com")).thenThrow( + new URISyntaxException("URI Error", "Error reason")); + + dereferencingControllerMock.perform( + post(RestEndpoints.DEREFERENCE).accept(MediaType.APPLICATION_XML_VALUE).contentType(MediaType.APPLICATION_JSON) + .content("[ \"http://www.example.com\" ]")).andDo(print()).andExpect(status().is(200)) + .andExpect(xpath("//metis:results", namespaceMap).exists()) + .andExpect(xpath("//metis:results", namespaceMap).nodeCount(1)) + .andExpect(xpath("//metis:results/metis:result", namespaceMap).exists()) + .andExpect(xpath("//metis:results/metis:result[*]", namespaceMap).nodeCount(0)); } @Test void exceptionHandling() throws Exception { - when(dereferenceServiceMock.dereference("http://www.example.com")) - .thenThrow(new TransformerException("myException")); + when(dereferenceServiceMock.dereference("http://www.example.com")).thenThrow(new TransformerException("myException")); dereferencingControllerMock.perform( - post("/dereference").content("[ \"http://www.example.com\" ]") - .accept(MediaType.APPLICATION_JSON).contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().is(500)).andExpect(content().string( - "{\"errorMessage\":\"Dereferencing failed for uri: http://www.example.com with root cause: myException\"}")); + post(RestEndpoints.DEREFERENCE).content("[ \"http://www.example.com\" ]").accept(MediaType.APPLICATION_JSON) + .contentType(MediaType.APPLICATION_JSON)).andExpect(status().is(500)).andExpect( + content().string( + "{\"errorMessage\":\"Dereferencing failed for uri: http://www.example.com with root cause: myException\"}")); } private Agent getAgent(String uri) { @@ -125,4 +143,4 @@ private Map getNamespaceMap() { namespaceMap.put("rdaGr2", "http://rdvocab.info/ElementsGr2/"); return namespaceMap; } -} \ No newline at end of file +} diff --git a/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingManagementControllerTest.java b/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingManagementControllerTest.java index 96f2f4589..c12ef839d 100644 --- a/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingManagementControllerTest.java +++ b/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingManagementControllerTest.java @@ -2,19 +2,31 @@ import static org.hamcrest.core.Is.is; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import eu.europeana.metis.dereference.Vocabulary; import eu.europeana.metis.dereference.rest.exceptions.RestResponseExceptionHandler; import eu.europeana.metis.dereference.service.DereferencingManagementService; +import eu.europeana.metis.dereference.vocimport.exception.VocabularyImportException; +import eu.europeana.metis.utils.RestEndpoints; +import java.net.URL; import java.util.ArrayList; import java.util.Collections; +import java.util.Set; import org.bson.types.ObjectId; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -22,23 +34,24 @@ import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.setup.MockMvcBuilders; +/** + * Unit tests {@link DereferencingController} class + */ class DereferencingManagementControllerTest { - private DereferencingManagementService dereferencingManagementServiceMock; - private MockMvc dereferencingManagementControllerMock; + private DereferencingManagementService deRefManagementServiceMock; + private MockMvc deRefManagementControllerMock; private String testEmptyCacheResult = ""; @BeforeEach void setUp() { - dereferencingManagementServiceMock = mock(DereferencingManagementService.class); + deRefManagementServiceMock = mock(DereferencingManagementService.class); DereferencingManagementController dereferencingManagementController = new DereferencingManagementController( - dereferencingManagementServiceMock); + deRefManagementServiceMock, Set.of("valid.domain.com")); - dereferencingManagementControllerMock = MockMvcBuilders - .standaloneSetup(dereferencingManagementController) - .setControllerAdvice(new RestResponseExceptionHandler()) - .build(); + deRefManagementControllerMock = MockMvcBuilders.standaloneSetup(dereferencingManagementController) + .setControllerAdvice(new RestResponseExceptionHandler()).build(); } @Test @@ -46,23 +59,39 @@ void testGetAllVocabularies() throws Exception { Vocabulary dummyVocab1 = new Vocabulary(); dummyVocab1.setId(new ObjectId()); dummyVocab1.setName("Dummy1"); - dummyVocab1.setUris(Collections.singleton("http://dummy1.org/path1")); + dummyVocab1.setUris(Collections.singleton("https://dummy1.org/path1")); Vocabulary dummyVocab2 = new Vocabulary(); dummyVocab2.setId(new ObjectId()); dummyVocab2.setName("Dummy2"); - dummyVocab2.setUris(Collections.singleton("http://dummy2.org/path2")); + dummyVocab2.setUris(Collections.singleton("https://dummy2.org/path2")); ArrayList dummyVocabList = new ArrayList<>(); dummyVocabList.add(dummyVocab1); dummyVocabList.add(dummyVocab2); - when(dereferencingManagementServiceMock.getAllVocabularies()).thenReturn(dummyVocabList); + when(deRefManagementServiceMock.getAllVocabularies()).thenReturn(dummyVocabList); - dereferencingManagementControllerMock.perform(get("/vocabularies")) - .andExpect(jsonPath("$[0].uris[0]", is("http://dummy1.org/path1"))) - .andExpect(jsonPath("$[1].uris[0]", is("http://dummy2.org/path2"))) - .andExpect(status().is(200)); + deRefManagementControllerMock.perform(get(RestEndpoints.VOCABULARIES)) + .andExpect(jsonPath("$[0].uris[0]", is("https://dummy1.org/path1"))) + .andExpect(jsonPath("$[1].uris[0]", is("https://dummy2.org/path2"))) + .andExpect(status().is(200)); + } + + @Test + void testLoadVocabularies_validDomain_expectSuccess() throws Exception { + doNothing().when(deRefManagementServiceMock).loadVocabularies(any(URL.class)); + deRefManagementControllerMock.perform(post(RestEndpoints.LOAD_VOCABULARIES) + .param("url", "http://valid.domain.com/path/to/vocab.rdf") + .param("directory_url", "https://valid.domain.com/test/call")) + .andExpect(status().is(200)); + } + + @Test + void testLoadVocabularies_invalidDomain_expectFail() throws Exception { + deRefManagementControllerMock.perform(post(RestEndpoints.LOAD_VOCABULARIES) + .param("directory_url", "https://invalid.domain.com")) + .andExpect(status().is(400)); } @Test @@ -70,11 +99,100 @@ void testEmptyCache() throws Exception { doAnswer((Answer) invocationOnMock -> { testEmptyCacheResult = "OK"; return null; - }).when(dereferencingManagementServiceMock).emptyCache(); + }).when(deRefManagementServiceMock).emptyCache(); + + deRefManagementControllerMock.perform(delete(RestEndpoints.CACHE_EMPTY)) + .andExpect(status().is(200)); + + assertEquals("OK", testEmptyCacheResult); + } + + @Test + void testEmptyCacheByEmptyXml() throws Exception { + doAnswer((Answer) invocationOnMock -> { + testEmptyCacheResult = "OK"; + return null; + }).when(deRefManagementServiceMock).purgeByNullOrEmptyXml(); + + deRefManagementControllerMock.perform(delete(RestEndpoints.CACHE_EMPTY_XML)) + .andExpect(status().is(200)); + + assertEquals("OK", testEmptyCacheResult); + } + + @Test + void testEmptyNullOrEmptyXML() throws Exception { + doAnswer((Answer) invocationOnMock -> { + testEmptyCacheResult = "OK"; + return null; + }).when(deRefManagementServiceMock).purgeByNullOrEmptyXml(); + + deRefManagementControllerMock.perform(delete(RestEndpoints.CACHE_EMPTY_XML)).andExpect(status().is(200)); + + assertEquals("OK", testEmptyCacheResult); + } + + + @Test + void testEmptyCacheByResourceId() throws Exception { + + doAnswer((Answer) invocationOnMock -> { + testEmptyCacheResult = "OK"; + return null; + }).when(deRefManagementServiceMock).purgeByResourceId(any(String.class)); - dereferencingManagementControllerMock.perform(delete("/cache")) - .andExpect(status().is(200)); + deRefManagementControllerMock.perform(delete(RestEndpoints.CACHE_EMPTY_RESOURCE) + .param("resourceId", "resourceId") + .param("resourceId", "12345")) + .andExpect(status().is(200)); assertEquals("OK", testEmptyCacheResult); } -} \ No newline at end of file + + @Test + void testEmptyCacheByVocabularyId() throws Exception { + + doAnswer((Answer) invocationOnMock -> { + testEmptyCacheResult = "OK"; + return null; + }).when(deRefManagementServiceMock).purgeByVocabularyId(any(String.class)); + + deRefManagementControllerMock.perform(delete(RestEndpoints.CACHE_EMPTY_VOCABULARY) + .param("vocabularyId", "12345")) + .andExpect(status().is(200)); + + assertEquals("OK", testEmptyCacheResult); + } + + @Test + void testLoadVocabularies_expectBadRequest() throws Exception { + deRefManagementControllerMock.perform(post(RestEndpoints.LOAD_VOCABULARIES) + .param("directory_url", "directory")) + .andDo(print()) + .andExpect(status().is(400)) + .andExpect(content().string("The url of the directory to import is not valid.")); + } + + @Test + void testLoadVocabularies_expectBadContent() throws Exception { + doThrow(VocabularyImportException.class).when(deRefManagementServiceMock).loadVocabularies(any(URL.class)); + deRefManagementControllerMock.perform(post(RestEndpoints.LOAD_VOCABULARIES) + .param("directory_url", "\\/tttp://test")) + .andDo(print()) + .andExpect(status().is(400)) + .andExpect(content().string("Provided directoryUrl '\\/tttp://test', failed to parse.")); + verify(deRefManagementServiceMock, times(0)).loadVocabularies(any(URL.class)); + } + + @Test + void testLoadVocabularies_expectBadVocabulary() throws Exception { + doThrow(new VocabularyImportException("Cannot load vocabulary")) + .when(deRefManagementServiceMock).loadVocabularies(any(URL.class)); + deRefManagementControllerMock.perform(post(RestEndpoints.LOAD_VOCABULARIES) + .param("directory_url", "https://valid.domain.com/test/call")) + .andDo(print()) + .andExpect(status().is(502)) + .andExpect(content().string("Cannot load vocabulary")); + verify(deRefManagementServiceMock, times(1)).loadVocabularies(any(URL.class)); + } +} diff --git a/metis-dereference/metis-dereference-service/pom.xml b/metis-dereference/metis-dereference-service/pom.xml index 5875c1722..343232172 100644 --- a/metis-dereference/metis-dereference-service/pom.xml +++ b/metis-dereference/metis-dereference-service/pom.xml @@ -4,7 +4,7 @@ metis-dereference eu.europeana.metis - 6 + 7 metis-dereference-service diff --git a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/DereferencingManagementService.java b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/DereferencingManagementService.java index 7248f6a63..e588df540 100644 --- a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/DereferencingManagementService.java +++ b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/DereferencingManagementService.java @@ -2,7 +2,7 @@ import eu.europeana.metis.dereference.Vocabulary; import eu.europeana.metis.dereference.vocimport.exception.VocabularyImportException; -import java.net.URI; +import java.net.URL; import java.util.List; /** @@ -22,12 +22,28 @@ public interface DereferencingManagementService { */ void emptyCache(); + /** + * + * purge all ProcessedEntities with empty XML + */ + void purgeByNullOrEmptyXml(); + + /** + * Empty the cache by resource ID(URI) + * @param resourceId The resourceId (URI) of the resource to be purged from the cache + */ + void purgeByResourceId(String resourceId); + /** + * Empty the cache by vocabulary ID + * @param vocabularyId the vocabulary ID to be purged from the cache, with all associated resources + */ + void purgeByVocabularyId(String vocabularyId); + /** * Load the vocabularies from an online source. This does NOT purge the cache. * * @param directoryUrl The online location of the vocabulary directory. - * @throws VocabularyImportException In case some issue occurred while importing the - * vocabularies. + * @throws VocabularyImportException In case some issue occurred while importing the vocabularies. */ - void loadVocabularies(URI directoryUrl) throws VocabularyImportException; + void loadVocabularies(URL directoryUrl) throws VocabularyImportException; } diff --git a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferenceService.java b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferenceService.java index 5d8f88ae0..247646a49 100644 --- a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferenceService.java +++ b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferenceService.java @@ -9,7 +9,7 @@ import eu.europeana.enrichment.api.external.model.Resource; import eu.europeana.enrichment.api.external.model.TimeSpan; import eu.europeana.enrichment.utils.EnrichmentBaseConverter; -import eu.europeana.metis.dereference.IncomingRecordToEdmConverter; +import eu.europeana.metis.dereference.IncomingRecordToEdmTransformer; import eu.europeana.metis.dereference.ProcessedEntity; import eu.europeana.metis.dereference.RdfRetriever; import eu.europeana.metis.dereference.Vocabulary; @@ -17,6 +17,7 @@ import eu.europeana.metis.dereference.service.dao.VocabularyDao; import eu.europeana.metis.dereference.service.utils.GraphUtils; import eu.europeana.metis.dereference.service.utils.VocabularyCandidates; +import eu.europeana.metis.exception.BadContentException; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -32,6 +33,7 @@ import java.util.function.Function; import java.util.stream.Stream; import javax.xml.bind.JAXBException; +import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -108,7 +110,7 @@ public List dereference(String resourceId) * @return A collection of dereferenced resources. Is not null, but could be empty. */ private Collection dereferenceResource(String resourceId) - throws JAXBException, TransformerException, URISyntaxException { + throws JAXBException, URISyntaxException { // Get the main object to dereference. If null, we are done. final Pair resource = computeEnrichmentBaseVocabularyPair( @@ -123,7 +125,7 @@ private Collection dereferenceResource(String resourceId) try { result = computeEnrichmentBaseVocabularyPair(key); return result == null ? null : result.getLeft(); - } catch (JAXBException | TransformerException | URISyntaxException e) { + } catch (JAXBException | URISyntaxException e) { LOGGER.warn(String.format("Problem occurred while dereferencing broader resource %s.", key), e); return null; @@ -166,25 +168,6 @@ private static Stream getStream(Collection collection) { return collection == null ? Stream.empty() : collection.stream(); } - Pair computeEnrichmentBaseVocabularyPair(String resourceId) - throws JAXBException, TransformerException, URISyntaxException { - - // Try to get the entity and its vocabulary from the cache. - final ProcessedEntity cachedEntity = processedEntityDao.get(resourceId); - final Pair entityVocabularyPair = computeEntityVocabularyPair(resourceId, - cachedEntity); - - // Parse the entity. - final Pair enrichmentBaseVocabularyPair; - if (entityVocabularyPair.getLeft() == null || entityVocabularyPair.getRight() == null) { - enrichmentBaseVocabularyPair = null; - } else { - enrichmentBaseVocabularyPair = convertToEnrichmentBaseVocabularyPair( - entityVocabularyPair.getLeft(), entityVocabularyPair.getRight()); - } - return enrichmentBaseVocabularyPair; - } - /** * Computes the entity and vocabulary. *

    It will use the cache if it's still valid, otherwise it will retrieve(if applicable) the @@ -208,7 +191,7 @@ Pair computeEnrichmentBaseVocabularyPair(String reso * @throws TransformerException if an exception occurred during transformation of the original entity */ private Pair computeEntityVocabularyPair(String resourceId, - ProcessedEntity cachedEntity) throws URISyntaxException, TransformerException { + ProcessedEntity cachedEntity) throws URISyntaxException { final Pair transformedEntityVocabularyPair; @@ -233,35 +216,7 @@ private Pair computeEntityVocabularyPair(String resourceId, return transformedEntityVocabularyPair; } - private void saveEntity(String resourceId, ProcessedEntity cachedEntity, - Pair transformedEntityAndVocabularyPair) { - - final String entityXml = transformedEntityAndVocabularyPair.getLeft(); - final Vocabulary vocabulary = transformedEntityAndVocabularyPair.getRight(); - final String vocabularyIdString = Optional.ofNullable(vocabulary).map(Vocabulary::getId) - .map(ObjectId::toString).orElse(null); - //Save entity - ProcessedEntity entityToCache = (cachedEntity == null) ? new ProcessedEntity() : cachedEntity; - entityToCache.setResourceId(resourceId); - entityToCache.setXml(entityXml); - entityToCache.setVocabularyId(vocabularyIdString); - processedEntityDao.save(entityToCache); - } - - private Pair convertToEnrichmentBaseVocabularyPair(String entityXml, - Vocabulary entityVocabulary) throws JAXBException { - final Pair result; - if (entityXml == null || entityVocabulary == null) { - result = null; - } else { - result = new ImmutablePair<>(EnrichmentBaseConverter.convertToEnrichmentBase(entityXml), - entityVocabulary); - } - return result; - } - - private Pair retrieveAndTransformEntity(String resourceId) - throws TransformerException, URISyntaxException { + private Pair retrieveAndTransformEntity(String resourceId) throws URISyntaxException { final VocabularyCandidates vocabularyCandidates = VocabularyCandidates .findVocabulariesForUrl(resourceId, vocabularyDao::getByUriSearch); @@ -296,6 +251,47 @@ private Pair retrieveAndTransformEntity(String resourceId) return entityVocabularyPair; } + private void saveEntity(String resourceId, ProcessedEntity cachedEntity, + Pair transformedEntityAndVocabularyPair) { + + final String entityXml = transformedEntityAndVocabularyPair.getLeft(); + final Vocabulary vocabulary = transformedEntityAndVocabularyPair.getRight(); + final String vocabularyIdString = Optional.ofNullable(vocabulary).map(Vocabulary::getId) + .map(ObjectId::toString).orElse(null); + //Save entity + ProcessedEntity entityToCache = (cachedEntity == null) ? new ProcessedEntity() : cachedEntity; + entityToCache.setResourceId(resourceId); + entityToCache.setXml(entityXml); + entityToCache.setVocabularyId(vocabularyIdString); + processedEntityDao.save(entityToCache); + } + + private Pair convertToEnrichmentBaseVocabularyPair(String entityXml, + Vocabulary entityVocabulary) throws JAXBException { + final Pair result; + if (entityXml == null || entityVocabulary == null) { + result = null; + } else { + result = new ImmutablePair<>(EnrichmentBaseConverter.convertToEnrichmentBase(entityXml), + entityVocabulary); + } + return result; + } + + private String transformEntity(Vocabulary vocabulary, String originalEntity, String resourceId) { + Optional result; + try { + final IncomingRecordToEdmTransformer incomingRecordToEdmTransformer = new IncomingRecordToEdmTransformer( + vocabulary.getXslt()); + result = incomingRecordToEdmTransformer.transform(originalEntity, resourceId); + } catch (TransformerException | BadContentException | ParserConfigurationException e) { + LOGGER.warn("Error transforming entity: {} with message: {}", resourceId, e.getMessage()); + LOGGER.debug("Transformation issue: ", e); + result = Optional.empty(); + } + return result.orElse(null); + } + private String retrieveOriginalEntity(String resourceId, VocabularyCandidates candidates) throws URISyntaxException { @@ -323,21 +319,22 @@ private String retrieveOriginalEntity(String resourceId, VocabularyCandidates ca return originalEntity; } - private String transformEntity(Vocabulary vocabulary, String originalEntity, String resourceId) - throws TransformerException { - final IncomingRecordToEdmConverter converter = new IncomingRecordToEdmConverter(vocabulary); - final String result; - try { - result = converter.convert(originalEntity, resourceId); - if (result == null && LOGGER.isInfoEnabled()) { - LOGGER.info("Could not transform entity {} as it results is an empty XML.", - CRLF_PATTERN.matcher(resourceId).replaceAll("")); - } - } catch (TransformerException e) { - LOGGER.warn("Error transforming entity: {} with message: {}", resourceId, e.getMessage()); - LOGGER.debug("Transformation issue: ", e); - return null; + Pair computeEnrichmentBaseVocabularyPair(String resourceId) + throws JAXBException, URISyntaxException { + + // Try to get the entity and its vocabulary from the cache. + final ProcessedEntity cachedEntity = processedEntityDao.getByResourceId(resourceId); + final Pair entityVocabularyPair = computeEntityVocabularyPair(resourceId, + cachedEntity); + + // Parse the entity. + final Pair enrichmentBaseVocabularyPair; + if (entityVocabularyPair.getLeft() == null || entityVocabularyPair.getRight() == null) { + enrichmentBaseVocabularyPair = null; + } else { + enrichmentBaseVocabularyPair = convertToEnrichmentBaseVocabularyPair( + entityVocabularyPair.getLeft(), entityVocabularyPair.getRight()); } - return result; + return enrichmentBaseVocabularyPair; } } diff --git a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementService.java b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementService.java index 19fd99634..824cc4c9f 100644 --- a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementService.java +++ b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementService.java @@ -8,7 +8,7 @@ import eu.europeana.metis.dereference.vocimport.VocabularyCollectionValidator; import eu.europeana.metis.dereference.vocimport.VocabularyCollectionValidatorImpl; import eu.europeana.metis.dereference.vocimport.exception.VocabularyImportException; -import java.net.URI; +import java.net.URL; import java.util.ArrayList; import java.util.List; import org.springframework.beans.factory.annotation.Autowired; @@ -22,6 +22,7 @@ public class MongoDereferencingManagementService implements DereferencingManagem private final VocabularyDao vocabularyDao; private final ProcessedEntityDao processedEntityDao; + private final VocabularyCollectionImporterFactory vocabularyCollectionImporterFactory; /** * Constructor. @@ -30,9 +31,10 @@ public class MongoDereferencingManagementService implements DereferencingManagem */ @Autowired public MongoDereferencingManagementService(VocabularyDao vocabularyDao, - ProcessedEntityDao processedEntityDao) { + ProcessedEntityDao processedEntityDao, VocabularyCollectionImporterFactory vocabularyCollectionImporterFactory) { this.vocabularyDao = vocabularyDao; this.processedEntityDao = processedEntityDao; + this.vocabularyCollectionImporterFactory = vocabularyCollectionImporterFactory; } @Override @@ -42,26 +44,43 @@ public List getAllVocabularies() { @Override public void emptyCache() { - this.processedEntityDao.purgeAll(); + processedEntityDao.purgeAll(); } @Override - public void loadVocabularies(URI directoryUrl) throws VocabularyImportException { + public void purgeByNullOrEmptyXml() { processedEntityDao.purgeByNullOrEmptyXml(); } - // Import and validate the vocabularies - final List vocabularies = new ArrayList<>(); - final VocabularyCollectionImporter importer = new VocabularyCollectionImporterFactory() - .createImporter(directoryUrl); - final VocabularyCollectionValidator validator = new VocabularyCollectionValidatorImpl(importer, - true, true, true); - validator.validateVocabularyOnly(vocabulary -> vocabularies.add(convertVocabulary(vocabulary))); + @Override + public void purgeByResourceId(String resourceId) { + processedEntityDao.purgeByResourceId(resourceId); + } + + @Override + public void purgeByVocabularyId(String vocabularyId) { + processedEntityDao.purgeByVocabularyId(vocabularyId); + } + + @Override + public void loadVocabularies(URL directoryUrl) throws VocabularyImportException { + + try { + // Import and validate the vocabularies + final List vocabularies = new ArrayList<>(); + final VocabularyCollectionImporter importer = vocabularyCollectionImporterFactory + .createImporter(directoryUrl); + final VocabularyCollectionValidator validator = new VocabularyCollectionValidatorImpl(importer, + true, true, true); + validator.validateVocabularyOnly(vocabulary -> vocabularies.add(convertVocabulary(vocabulary))); - // All vocabularies are loaded well. Now we replace the vocabularies. - vocabularyDao.replaceAll(vocabularies); + // All vocabularies are loaded well. Now we replace the vocabularies. + vocabularyDao.replaceAll(vocabularies); + } catch (VocabularyImportException e) { + throw new VocabularyImportException("An error as occurred while loading the vocabularies", e); + } } private static Vocabulary convertVocabulary( - eu.europeana.metis.dereference.vocimport.model.Vocabulary input) { + eu.europeana.metis.dereference.vocimport.model.Vocabulary input) { final Vocabulary vocabulary = new Vocabulary(); vocabulary.setName(input.getName()); vocabulary.setUris(input.getPaths()); diff --git a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/dao/ProcessedEntityDao.java b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/dao/ProcessedEntityDao.java index de1fb8aa2..e9a6a6c0c 100644 --- a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/dao/ProcessedEntityDao.java +++ b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/dao/ProcessedEntityDao.java @@ -34,8 +34,8 @@ public class ProcessedEntityDao { */ public ProcessedEntityDao(MongoClient mongo, String databaseName) { final MapperOptions mapperOptions = MapperOptions.builder().discriminatorKey("className") - .discriminator(DiscriminatorFunction.className()) - .collectionNaming(NamingStrategy.identity()).build(); + .discriminator(DiscriminatorFunction.className()) + .collectionNaming(NamingStrategy.identity()).build(); this.datastore = Morphia.createDatastore(mongo, databaseName, mapperOptions); this.datastore.getMapper().map(ProcessedEntity.class); } @@ -46,12 +46,25 @@ public ProcessedEntityDao(MongoClient mongo, String databaseName) { * @param resourceId The resource ID (URI) to retrieve * @return The entity with the given resource ID. */ - public ProcessedEntity get(String resourceId) { + public ProcessedEntity getByResourceId(String resourceId) { return retryableExternalRequestForNetworkExceptions( () -> datastore.find(ProcessedEntity.class).filter(Filters.eq("resourceId", resourceId)) - .first()); + .first()); } + /** + * Get an entity by vocabulary ID. + * + * @param vocabularyId The vocabuylaryDi to retrieve + * @return The entity with the given vocabulary ID. + */ + public ProcessedEntity getByVocabularyId(String vocabularyId) { + return retryableExternalRequestForNetworkExceptions( + () -> datastore.find(ProcessedEntity.class).filter(Filters.eq("vocabularyId", vocabularyId)) + .first()); + } + + /** * Save an entity. * @@ -60,7 +73,7 @@ public ProcessedEntity get(String resourceId) { public void save(ProcessedEntity processedEntity) { try { final ObjectId objectId = Optional.ofNullable(processedEntity.getId()) - .orElseGet(ObjectId::new); + .orElseGet(ObjectId::new); processedEntity.setId(objectId); retryableExternalRequestForNetworkExceptions(() -> datastore.save(processedEntity)); } catch (DuplicateKeyException e) { @@ -70,6 +83,40 @@ public void save(ProcessedEntity processedEntity) { } } + /** + * Delete an entity with no description in XML resources. Empty or Null + **/ + public void purgeByNullOrEmptyXml() { + retryableExternalRequestForNetworkExceptions(() -> + datastore.find(ProcessedEntity.class) + .filter(Filters.eq("xml", null)) + .delete(new DeleteOptions().multi(true))); + } + + /** + * Delete an entity by resource ID. + * + * @param resourceId The resource ID (URI) to delete + **/ + public void purgeByResourceId(String resourceId) { + retryableExternalRequestForNetworkExceptions(() -> + datastore.find(ProcessedEntity.class) + .filter(Filters.eq("resourceId", resourceId)) + .delete(new DeleteOptions())); + } + + /** + * Delete the entity based on its vocabulary ID. + * + * @param vocabularyId The ID of the vocabulary to delete. + **/ + public void purgeByVocabularyId(String vocabularyId) { + retryableExternalRequestForNetworkExceptions(() -> + datastore.find(ProcessedEntity.class) + .filter(Filters.eq("vocabularyId", vocabularyId)) + .delete(new DeleteOptions().multi(true))); + } + /** * Remove all entities. */ @@ -77,4 +124,14 @@ public void purgeAll() { retryableExternalRequestForNetworkExceptions( () -> datastore.find(ProcessedEntity.class).delete(new DeleteOptions().multi(true))); } + + /** + * Size of Processed entities + * + * @return amount of documents in db + */ + protected long size() { + return retryableExternalRequestForNetworkExceptions( + () -> datastore.find(ProcessedEntity.class).stream().count()); + } } diff --git a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/dao/VocabularyDao.java b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/dao/VocabularyDao.java index ade518f21..52bdb7d9e 100644 --- a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/dao/VocabularyDao.java +++ b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/dao/VocabularyDao.java @@ -89,4 +89,13 @@ public void replaceAll(List vocabularies) { protected Datastore getDatastore() { return datastore; } + + /** + * Amount of documents + * + * @return amount of documents in db + */ + protected long size() { + return retryableExternalRequestForNetworkExceptions(() -> datastore.find(Vocabulary.class).stream().count()); + } } diff --git a/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementServiceTest.java b/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementServiceTest.java index f77c0a64a..0bbf94c13 100644 --- a/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementServiceTest.java +++ b/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementServiceTest.java @@ -1,15 +1,30 @@ package eu.europeana.metis.dereference.service; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; import dev.morphia.Datastore; +import eu.europeana.metis.dereference.ProcessedEntity; import eu.europeana.metis.dereference.Vocabulary; import eu.europeana.metis.dereference.service.dao.ProcessedEntityDao; import eu.europeana.metis.dereference.service.dao.VocabularyDao; +import eu.europeana.metis.dereference.vocimport.VocabularyCollectionImporter; +import eu.europeana.metis.dereference.vocimport.VocabularyCollectionImporterFactory; +import eu.europeana.metis.dereference.vocimport.exception.VocabularyImportException; import eu.europeana.metis.mongo.embedded.EmbeddedLocalhostMongo; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.Collections; import java.util.List; import org.junit.jupiter.api.AfterEach; @@ -17,16 +32,19 @@ import org.junit.jupiter.api.Test; /** - * Created by ymamakis on 2/22/16. + * Unit tests for {@link MongoDereferencingManagementService} class */ class MongoDereferencingManagementServiceTest { private MongoDereferencingManagementService service; - private EmbeddedLocalhostMongo embeddedLocalhostMongo = new EmbeddedLocalhostMongo(); - private Datastore vocDaoDatastore; + private final EmbeddedLocalhostMongo embeddedLocalhostMongo = new EmbeddedLocalhostMongo(); + private Datastore vocabularyDaoDatastore; + + private VocabularyCollectionImporterFactory vocabularyCollectionImporterFactory; + private final ProcessedEntityDao processedEntityDao = mock(ProcessedEntityDao.class); @BeforeEach - void prepare() { + void prepare() throws IOException { embeddedLocalhostMongo.start(); String mongoHost = embeddedLocalhostMongo.getMongoHost(); int mongoPort = embeddedLocalhostMongo.getMongoPort(); @@ -36,11 +54,12 @@ void prepare() { VocabularyDao vocDao = new VocabularyDao(mongoClient, "voctest") { { - vocDaoDatastore = this.getDatastore(); + vocabularyDaoDatastore = this.getDatastore(); } }; - ProcessedEntityDao processedEntityDao = mock(ProcessedEntityDao.class); - service = new MongoDereferencingManagementService(vocDao, processedEntityDao); + vocabularyCollectionImporterFactory = mock(VocabularyCollectionImporterFactory.class); + + service = new MongoDereferencingManagementService(vocDao, processedEntityDao, vocabularyCollectionImporterFactory); } @Test @@ -50,11 +69,86 @@ void testGetAllVocabularies() { voc.setName("testName"); voc.setUris(Collections.singleton("http://www.test.uri/")); voc.setXslt("testXSLT"); - vocDaoDatastore.save(voc); + vocabularyDaoDatastore.save(voc); List retVoc = service.getAllVocabularies(); assertEquals(1, retVoc.size()); } + + @Test + void purgeAllCache() { + ProcessedEntity processedEntity = new ProcessedEntity(); + processedEntity.setResourceId("http://www.test.uri/"); + processedEntityDao.save(processedEntity); + service.emptyCache(); + ProcessedEntity ret = processedEntityDao.getByResourceId("http://www.test.uri/"); + assertNull(ret); + } + + @Test + void purgeCacheWithEmptyXML() { + ProcessedEntity processedEntity = new ProcessedEntity(); + processedEntity.setResourceId("http://www.test.uri/"); + processedEntity.setXml(null); + processedEntityDao.save(processedEntity); + service.purgeByNullOrEmptyXml(); + ProcessedEntity ret = processedEntityDao.getByResourceId("http://www.test.uri/"); + assertNull(ret); + } + + + @Test + void purgeCacheByResourceId() { + ProcessedEntity processedEntity = new ProcessedEntity(); + processedEntity.setResourceId("http://www.test.uri/"); + processedEntityDao.save(processedEntity); + service.purgeByResourceId("http://www.test.uri/"); + ProcessedEntity ret = processedEntityDao.getByResourceId("http://www.test.uri/"); + assertNull(ret); + } + + @Test + void purgeCacheByVocabularyId() { + ProcessedEntity processedEntity = new ProcessedEntity(); + processedEntity.setVocabularyId("vocabularyId"); + processedEntityDao.save(processedEntity); + service.purgeByVocabularyId("vocabularyId"); + ProcessedEntity ret = processedEntityDao.getByVocabularyId("vocabularyId"); + assertNull(ret); + } + + @Test + void loadVocabularies_expectSucess() throws VocabularyImportException, URISyntaxException, IOException { + final URL resourceLocation = this.getClass().getClassLoader().getResource("vocabulary.yml"); + final String expectedXslt = Files.readString(Paths.get(getClass() + .getClassLoader() + .getResource("vocabulary/voctest.xsl").toURI())).trim(); + + final VocabularyCollectionImporter importer = new VocabularyCollectionImporterFactory().createImporter(resourceLocation); + doReturn(importer).when(vocabularyCollectionImporterFactory).createImporter(any(URL.class)); + + service.loadVocabularies(resourceLocation); + Vocabulary vocabulary = vocabularyDaoDatastore.find(Vocabulary.class).first(); + + assertEquals("TestWikidata", vocabulary.getName()); + assertEquals(expectedXslt, vocabulary.getXslt()); + assertEquals("http://www.wikidata.org/entity/", vocabulary.getUris().stream().findFirst().get()); + verify(vocabularyCollectionImporterFactory, times(1)).createImporter(resourceLocation); + } + + @Test + void loadVocabularies_expectVocabularyImportError() throws VocabularyImportException, URISyntaxException, IOException { + final URL resourceLocation = this.getClass().getClassLoader().getResource("vocabulary-fault.yml"); + final VocabularyCollectionImporter importer = new VocabularyCollectionImporterFactory().createImporter(resourceLocation); + doReturn(importer).when(vocabularyCollectionImporterFactory).createImporter(any(URL.class)); + + VocabularyImportException expectedException = assertThrows(VocabularyImportException.class, + () -> service.loadVocabularies(resourceLocation)); + + assertEquals("An error as occurred while loading the vocabularies", expectedException.getMessage()); + verify(vocabularyCollectionImporterFactory, times(1)).createImporter(resourceLocation); + } + @AfterEach void destroy() { embeddedLocalhostMongo.stop(); diff --git a/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/dao/ProcessedEntityDaoTest.java b/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/dao/ProcessedEntityDaoTest.java new file mode 100644 index 000000000..0b42c5c44 --- /dev/null +++ b/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/dao/ProcessedEntityDaoTest.java @@ -0,0 +1,119 @@ +package eu.europeana.metis.dereference.service.dao; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import eu.europeana.metis.dereference.ProcessedEntity; +import eu.europeana.metis.dereference.service.dao.ProcessedEntityDao; +import eu.europeana.metis.mongo.embedded.EmbeddedLocalhostMongo; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link ProcessedEntityDao} class + */ +class ProcessedEntityDaoTest { + + private static EmbeddedLocalhostMongo embeddedLocalhostMongo; + + private static ProcessedEntityDao processedEntityDao; + + @BeforeAll + static void prepare() { + embeddedLocalhostMongo = new EmbeddedLocalhostMongo(); + + embeddedLocalhostMongo.start(); + final String mongoHost = embeddedLocalhostMongo.getMongoHost(); + final int mongoPort = embeddedLocalhostMongo.getMongoPort(); + + final MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s", mongoHost, mongoPort)); + processedEntityDao = new ProcessedEntityDao(mongoClient, "metis-dereference"); + } + + @BeforeEach + void setupDb() { + initDatabaseWithEntities(); + } + + @AfterEach + void tearDownDb() { + processedEntityDao.purgeAll(); + } + + @Test + void processDaoPurgeAll() { + assertEquals(5, processedEntityDao.size()); + + processedEntityDao.purgeAll(); + + for (int i = 1; i < 5; i++) { + assertNull(processedEntityDao.getByResourceId("http://www.test" + i + ".uri/")); + } + assertEquals(0, processedEntityDao.size()); + } + + @Test + void processDaoPurgeByNullOrEmptyXML() { + assertEquals(5, processedEntityDao.size()); + + processedEntityDao.purgeByNullOrEmptyXml(); + + assertNull(processedEntityDao.getByResourceId("http://www.test5.uri/")); + for (int i = 1; i < 4; i++) { + assertEquals("http://www.test" + i + ".uri/", + processedEntityDao.getByResourceId("http://www.test" + i + ".uri/").getResourceId()); + } + assertEquals(4, processedEntityDao.size()); + } + + @Test + void processDaoPurgeByResourceId() { + assertEquals(5, processedEntityDao.size()); + + processedEntityDao.purgeByResourceId("http://www.test1.uri/"); + + assertNull(processedEntityDao.getByResourceId("http://www.test1.uri/")); + assertEquals(4, processedEntityDao.size()); + } + + @Test + void processDaoPurgeByVocabulary() { + assertEquals(5, processedEntityDao.size()); + + processedEntityDao.purgeByVocabularyId("vocabularyId1"); + + assertNull(processedEntityDao.getByVocabularyId("vocabularyId1")); + for (int i = 2; i < 5; i++) { + assertEquals("vocabularyId" + i, + processedEntityDao.getByVocabularyId("vocabularyId" + i).getVocabularyId()); + } + assertEquals(4, processedEntityDao.size()); + } + + // side note flapdoodle embedded mongo + // doesn't support DuplicateExceptionKey Mongo. + + void initDatabaseWithEntities() { + for (int i = 1; i <= 5; i++) { + final ProcessedEntity processedEntity = new ProcessedEntity(); + processedEntity.setResourceId("http://www.test" + i + ".uri/"); + processedEntity.setVocabularyId("vocabularyId" + i); + if (i == 5) { + processedEntity.setXml(null); + } else { + processedEntity.setXml("value"); + } + processedEntityDao.save(processedEntity); + } + } + + @AfterAll + static void destroy() { + embeddedLocalhostMongo.stop(); + } +} diff --git a/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/dao/VocabularyDaoTest.java b/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/dao/VocabularyDaoTest.java new file mode 100644 index 000000000..bb1012803 --- /dev/null +++ b/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/dao/VocabularyDaoTest.java @@ -0,0 +1,146 @@ +package eu.europeana.metis.dereference.service.dao; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import dev.morphia.DeleteOptions; +import eu.europeana.metis.dereference.Vocabulary; +import eu.europeana.metis.mongo.embedded.EmbeddedLocalhostMongo; +import java.util.List; +import java.util.Optional; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + + +/** + * Unit tests for {@link VocabularyDao} class + */ +class VocabularyDaoTest { + + private static EmbeddedLocalhostMongo embeddedLocalhostMongo; + + private static VocabularyDao vocabularyDao; + + @BeforeAll + static void prepare() { + embeddedLocalhostMongo = new EmbeddedLocalhostMongo(); + + embeddedLocalhostMongo.start(); + final String mongoHost = embeddedLocalhostMongo.getMongoHost(); + final int mongoPort = embeddedLocalhostMongo.getMongoPort(); + + final MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s", mongoHost, mongoPort)); + vocabularyDao = new VocabularyDao(mongoClient, "metis-dereference"); + } + + @AfterAll + static void destroy() { + embeddedLocalhostMongo.stop(); + } + + @BeforeEach + void setupDb() { + initDatabaseWithEntities(); + } + + @AfterEach + void tearDownDb() { + vocabularyDao.getDatastore().find(Vocabulary.class).delete(new DeleteOptions().multi(true)); + } + + @Test + void getByUriSearch() { + assertEquals(5, vocabularyDao.size()); + + List vocabularies = vocabularyDao.getByUriSearch("http://domain2.uri"); + + assertEquals(2, vocabularies.size()); + assertNotEquals(Optional.empty(), + vocabularies.stream().filter(vocabulary -> vocabulary.getName().equals("vocabulary2")).findFirst()); + assertNotEquals(Optional.empty(), + vocabularies.stream().filter(vocabulary -> vocabulary.getName().equals("vocabulary4")).findFirst()); + assertEquals(Optional.empty(), + vocabularies.stream().filter(vocabulary -> vocabulary.getName().equals("vocabulary1")).findFirst()); + assertEquals(Optional.empty(), + vocabularies.stream().filter(vocabulary -> vocabulary.getName().equals("vocabulary3")).findFirst()); + assertEquals(Optional.empty(), + vocabularies.stream().filter(vocabulary -> vocabulary.getName().equals("vocabulary5")).findFirst()); + } + + @Test + void getAll() { + assertEquals(5, vocabularyDao.size()); + + List vocabularies = vocabularyDao.getAll(); + + assertEquals(5, vocabularies.size()); + assertNotEquals(Optional.empty(), + vocabularies.stream().filter(vocabulary -> vocabulary.getName().equals("vocabulary1")).findFirst()); + assertNotEquals(Optional.empty(), + vocabularies.stream().filter(vocabulary -> vocabulary.getName().equals("vocabulary2")).findFirst()); + assertNotEquals(Optional.empty(), + vocabularies.stream().filter(vocabulary -> vocabulary.getName().equals("vocabulary3")).findFirst()); + assertNotEquals(Optional.empty(), + vocabularies.stream().filter(vocabulary -> vocabulary.getName().equals("vocabulary4")).findFirst()); + assertNotEquals(Optional.empty(), + vocabularies.stream().filter(vocabulary -> vocabulary.getName().equals("vocabulary5")).findFirst()); + } + + @Test + void get() { + assertEquals(5, vocabularyDao.size()); + + Vocabulary expectedVocabulary = vocabularyDao.getDatastore().find(Vocabulary.class).first(); + + Vocabulary vocabulary = vocabularyDao.get(expectedVocabulary.getId().toString()); + + assertEquals(expectedVocabulary.getName(), vocabulary.getName()); + assertEquals(expectedVocabulary.getSuffix(), vocabulary.getSuffix()); + assertEquals(expectedVocabulary.getXslt(), vocabulary.getXslt()); + assertEquals(expectedVocabulary.getUris().stream().findFirst().get(), vocabulary.getUris().stream().findFirst().get()); + assertEquals(expectedVocabulary.getIterations(), vocabulary.getIterations()); + } + + @Test + void replaceAll() { + assertEquals(5, vocabularyDao.size()); + + Vocabulary vocabulary = new Vocabulary(); + vocabulary.setXslt("xlst"); + vocabulary.setSuffix("suffix"); + vocabulary.setUris(List.of("uri")); + vocabulary.setIterations(0); + vocabulary.setName("vocabularyName"); + + vocabularyDao.replaceAll(List.of(vocabulary)); + + assertEquals(1, vocabularyDao.size()); + } + + @Test + void getDatastore() { + assertNotNull(vocabularyDao.getDatastore()); + } + + void initDatabaseWithEntities() { + for (int i = 1; i <= 5; i++) { + Vocabulary vocabulary = new Vocabulary(); + vocabulary.setName("vocabulary" + i); + vocabulary.setSuffix("suffix" + i); + if (i % 2 == 0) { + vocabulary.setUris(List.of("http://domain2.uri")); + } else { + vocabulary.setUris(List.of("http://domain1.uri")); + } + vocabulary.setXslt("xlst" + i); + vocabulary.setIterations(0); + vocabularyDao.getDatastore().save(vocabulary); + } + } +} diff --git a/metis-dereference/metis-dereference-service/src/test/resources/vocabulary-fault.yml b/metis-dereference/metis-dereference-service/src/test/resources/vocabulary-fault.yml new file mode 100644 index 000000000..07d1d8dcb --- /dev/null +++ b/metis-dereference/metis-dereference-service/src/test/resources/vocabulary-fault.yml @@ -0,0 +1,3 @@ +# test +- metadata: vocabulary/voctest1.yml + mapping: vocabulary/voctest1.xsl diff --git a/metis-dereference/metis-dereference-service/src/test/resources/vocabulary.yml b/metis-dereference/metis-dereference-service/src/test/resources/vocabulary.yml new file mode 100644 index 000000000..24bf65236 --- /dev/null +++ b/metis-dereference/metis-dereference-service/src/test/resources/vocabulary.yml @@ -0,0 +1,3 @@ +# test +- metadata: vocabulary/voctest.yml + mapping: vocabulary/voctest.xsl diff --git a/metis-dereference/metis-dereference-service/src/test/resources/vocabulary/voctest.xsl b/metis-dereference/metis-dereference-service/src/test/resources/vocabulary/voctest.xsl new file mode 100644 index 000000000..6042f2139 --- /dev/null +++ b/metis-dereference/metis-dereference-service/src/test/resources/vocabulary/voctest.xsl @@ -0,0 +1,960 @@ + + + + + + + + + + + + + + + + + en,pl,de,nl,fr,it,da,sv,el,fi,hu,cs,sl,et,pt,es,lt,lv,bg,ro,sk,hr,ga,mt,no,ca,ru + + + + http://viaf.org/viaf/$1 + http://d-nb.info/gnd/$1 + http://id.loc.gov/authorities/names/$1 + http://vocab.getty.edu/ulan/$1 + http://data.bnf.fr/ark:/12148/cb$1 + http://www.idref.fr/$1/id + http://id.ndl.go.jp/auth/ndlna/$1 + http://id.nlm.nih.gov/mesh/$1 + http://purl.org/bncf/tid/$1 + https://www.freebase.com$1 + https://g.co/kg$1 + http://openlibrary.org/works/$1 + http://id.nlm.nih.gov/mesh/$1 + http://libris.kb.se/resource/auth/$1 + http://datos.bne.es/resource/$1 + http://data.bibliotheken.nl/id/thes/p$1 + http://vocab.getty.edu/aat/$1 + https://livedata.bibsys.no/authority/$1 + http://dewey.info/class/$1/ + http://iconclass.org/$1 + http://kulturarvsdata.se/$1 + http://ta.sandrart.net/-person-$1 + https://sws.geonames.org/$1/ + http://pleiades.stoa.org/places/$1/rdf + http://vocab.getty.edu/tgn/$1 + urn:uuid:$1 + http://dare.ht.lu.se/places/$1 + http://id.worldcat.org/fast/$1 + http://www.yso.fi/onto/yso/p$1 + http://www.geonames.org/ontology#$1 + http://babelnet.org/rdf/s$1 + http://g.co/kg$1 + http://data.cervantesvirtual.com/person/$1 + http://nomisma.org/id/$1 + http://data.ordnancesurvey.co.uk/id/$1 + http://nlg.okfn.gr/resource/authority/record$1 + http://thesaurus.europeanafashion.eu/thesaurus/$1 + http://zbw.eu/stw/descriptor/$1 + http://vocabularies.unesco.org/thesaurus/$1 + http://data.carnegiehall.org/names/$1 + https://id.erfgoed.net/thesauri/erfgoedtypes/$1 + http://id.loc.gov/authorities/genreForms/$1 + http://lod.nl.go.kr/resource/$1 + http://cv.iptc.org/newscodes/$1 + https://libris.kb.se/$1 + http://uri.gbv.de/terminology/bk/$1 + http://www.yso.fi/onto/ysa/$1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/metis-dereference/metis-dereference-service/src/test/resources/vocabulary/voctest.yml b/metis-dereference/metis-dereference-service/src/test/resources/vocabulary/voctest.yml new file mode 100644 index 000000000..5905cfb3d --- /dev/null +++ b/metis-dereference/metis-dereference-service/src/test/resources/vocabulary/voctest.yml @@ -0,0 +1,18 @@ +name: TestWikidata +types: + - AGENT + - CONCEPT + - PLACE + - TIMESTAMP +paths: + - http://www.wikidata.org/entity/ +examples: + - http://www.wikidata.org/entity/Q3930 + - http://www.wikidata.org/entity/Q604667 + - http://www.wikidata.org/entity/Q79007 + - http://www.wikidata.org/entity/Q1261026 + - http://www.wikidata.org/entity/Q36422 + - http://www.wikidata.org/entity/Q6927 + - http://www.wikidata.org/entity/Q90 + - http://www.wikidata.org/entity/Q187843 + - http://www.wikidata.org/entity/Q145 diff --git a/metis-dereference/pom.xml b/metis-dereference/pom.xml index f7a9345af..e339a7ee9 100644 --- a/metis-dereference/pom.xml +++ b/metis-dereference/pom.xml @@ -3,7 +3,7 @@ 4.0.0 eu.europeana.metis - 6 + 7 metis-framework diff --git a/metis-enrichment/metis-enrichment-client/.gitignore b/metis-enrichment/metis-enrichment-client/.gitignore deleted file mode 100644 index e8b90183b..000000000 --- a/metis-enrichment/metis-enrichment-client/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -##Add to ignore to not commit by mistake - diff --git a/metis-enrichment/metis-enrichment-client/pom.xml b/metis-enrichment/metis-enrichment-client/pom.xml index 55ff01ac5..fab5664bf 100644 --- a/metis-enrichment/metis-enrichment-client/pom.xml +++ b/metis-enrichment/metis-enrichment-client/pom.xml @@ -4,7 +4,7 @@ metis-enrichment eu.europeana.metis - 6 + 7 metis-enrichment-client jar @@ -42,7 +42,6 @@ eu.europeana.metis metis-schema - ${project.version} com.fasterxml.jackson.core diff --git a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/EnrichmentWorkerImpl.java b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/EnrichmentWorkerImpl.java index 7f944fd97..316eabc5f 100644 --- a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/EnrichmentWorkerImpl.java +++ b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/EnrichmentWorkerImpl.java @@ -20,6 +20,7 @@ public class EnrichmentWorkerImpl implements EnrichmentWorker { private static final Logger LOGGER = LoggerFactory.getLogger(EnrichmentWorkerImpl.class); + private static final RdfConversionUtils rdfConversionUtils = new RdfConversionUtils(); private final Enricher enricher; private final Dereferencer dereferencer; @@ -169,18 +170,18 @@ private String convertRdfToStringForLogging(final RDF rdf) { String convertRdfToString(RDF rdf) throws SerializationException { - return RdfConversionUtils.convertRdfToString(rdf); + return rdfConversionUtils.convertRdfToString(rdf); } byte[] convertRdfToBytes(RDF rdf) throws SerializationException { - return RdfConversionUtils.convertRdfToBytes(rdf); + return rdfConversionUtils.convertRdfToBytes(rdf); } RDF convertStringToRdf(String xml) throws SerializationException { - return RdfConversionUtils.convertStringToRdf(xml); + return rdfConversionUtils.convertStringToRdf(xml); } RDF convertInputStreamToRdf(InputStream xml) throws SerializationException { - return RdfConversionUtils.convertInputStreamToRdf(xml); + return rdfConversionUtils.convertInputStreamToRdf(xml); } } diff --git a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/dereference/DereferencerProvider.java b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/dereference/DereferencerProvider.java index 4f1c6f82c..22677591b 100644 --- a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/dereference/DereferencerProvider.java +++ b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/dereference/DereferencerProvider.java @@ -1,7 +1,7 @@ package eu.europeana.enrichment.rest.client.dereference; +import eu.europeana.enrichment.api.external.impl.RemoteEntityResolver; import eu.europeana.enrichment.rest.client.ConnectionProvider; -import eu.europeana.enrichment.rest.client.enrichment.RemoteEntityResolver; import eu.europeana.enrichment.rest.client.exceptions.DereferenceException; import eu.europeana.enrichment.utils.EntityMergeEngine; import java.net.MalformedURLException; diff --git a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/EnricherImpl.java b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/EnricherImpl.java index c84c0abb3..029ac61af 100644 --- a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/EnricherImpl.java +++ b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/EnricherImpl.java @@ -1,5 +1,8 @@ package eu.europeana.enrichment.rest.client.enrichment; +import static eu.europeana.enrichment.api.internal.EntityResolver.europeanaLinkPattern; +import static eu.europeana.enrichment.api.internal.EntityResolver.semiumLinkPattern; + import eu.europeana.enrichment.api.external.model.EnrichmentBase; import eu.europeana.enrichment.api.internal.EntityResolver; import eu.europeana.enrichment.api.internal.ProxyFieldType; @@ -19,7 +22,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; -import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; @@ -31,13 +33,17 @@ public class EnricherImpl implements Enricher { private static final Logger LOGGER = LoggerFactory.getLogger(EnricherImpl.class); - private static final Pattern europeanaLinkPattern = Pattern - .compile("^https?://data.europeana.eu.*$"); - private final RecordParser recordParser; private final EntityResolver entityResolver; private final EntityMergeEngine entityMergeEngine; + /** + * Constructor with required parameters. + * + * @param recordParser the record parser + * @param entityResolver the entity resolver + * @param entityMergeEngine the entity merge engine + */ public EnricherImpl(RecordParser recordParser, EntityResolver entityResolver, EntityMergeEngine entityMergeEngine) { this.recordParser = recordParser; @@ -112,10 +118,11 @@ public Map> enrichReferences( public void cleanupPreviousEnrichmentEntities(RDF rdf) { final ProxyType europeanaProxy = RdfEntityUtils.getEuropeanaProxy(rdf); //Find the correct links - final Set europeanaLinks = Arrays.stream(ProxyFieldType.values()) - .map(proxyFieldType -> proxyFieldType.extractFieldLinksForEnrichment(europeanaProxy)) - .flatMap(Collection::stream).filter(europeanaLinkPattern.asPredicate()) - .collect(Collectors.toSet()); - RdfEntityUtils.removeMatchingEntities(rdf, europeanaLinks); + final Set matchingLinks = Arrays.stream(ProxyFieldType.values()) + .map(proxyFieldType -> proxyFieldType.extractFieldLinksForEnrichment(europeanaProxy)) + .flatMap(Collection::stream) + .filter(europeanaLinkPattern.asPredicate().or(semiumLinkPattern.asPredicate())) + .collect(Collectors.toSet()); + RdfEntityUtils.removeMatchingEntities(rdf, matchingLinks); } } diff --git a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/EnricherProvider.java b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/EnricherProvider.java index f0aff04aa..ed1c82318 100644 --- a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/EnricherProvider.java +++ b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/EnricherProvider.java @@ -1,5 +1,6 @@ package eu.europeana.enrichment.rest.client.enrichment; +import eu.europeana.enrichment.api.external.impl.RemoteEntityResolver; import eu.europeana.enrichment.api.internal.EntityResolver; import eu.europeana.enrichment.api.internal.RecordParser; import eu.europeana.enrichment.rest.client.ConnectionProvider; diff --git a/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/dereference/DereferencerImplTest.java b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/dereference/DereferencerImplTest.java index 332bed182..76abf28c0 100644 --- a/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/dereference/DereferencerImplTest.java +++ b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/dereference/DereferencerImplTest.java @@ -10,6 +10,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import eu.europeana.enrichment.api.external.impl.RemoteEntityResolver; import eu.europeana.enrichment.api.external.model.Agent; import eu.europeana.enrichment.api.external.model.EnrichmentBase; import eu.europeana.enrichment.api.external.model.EnrichmentResultBaseWrapper; @@ -18,7 +19,6 @@ import eu.europeana.enrichment.api.external.model.TimeSpan; import eu.europeana.enrichment.api.internal.SearchTerm; import eu.europeana.enrichment.api.internal.SearchTermImpl; -import eu.europeana.enrichment.rest.client.enrichment.RemoteEntityResolver; import eu.europeana.enrichment.rest.client.exceptions.DereferenceException; import eu.europeana.enrichment.utils.EntityMergeEngine; import eu.europeana.enrichment.utils.EntityType; diff --git a/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/EnricherImplTest.java b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/EnricherImplTest.java index b90fc5ec3..7c25212b3 100644 --- a/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/EnricherImplTest.java +++ b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/EnricherImplTest.java @@ -11,6 +11,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import eu.europeana.enrichment.api.external.impl.RemoteEntityResolver; import eu.europeana.enrichment.api.external.model.EnrichmentBase; import eu.europeana.enrichment.api.external.model.Place; import eu.europeana.enrichment.api.internal.ProxyFieldType; diff --git a/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/MetisRecordParserTest.java b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/MetisRecordParserTest.java index d83e6874c..6cf493987 100644 --- a/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/MetisRecordParserTest.java +++ b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/MetisRecordParserTest.java @@ -24,6 +24,7 @@ import eu.europeana.metis.schema.jibx.Subject; import eu.europeana.metis.schema.jibx.Temporal; import eu.europeana.metis.schema.jibx.Type; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Set; import org.apache.commons.io.IOUtils; @@ -31,6 +32,8 @@ public class MetisRecordParserTest { + private static final RdfConversionUtils rdfConversionUtils = new RdfConversionUtils(); + @Test public void testExtractedFieldValuesForEnrichment() { RDF rdf = new RDF(); @@ -177,8 +180,8 @@ public void testExtractedFieldValuesForEnrichment() { @Test public void testSetAdditionalData() throws Exception { String xml = IOUtils - .toString(getClass().getClassLoader().getResourceAsStream("sample_completeness.rdf"), "UTF-8"); - RDF rdf = RdfConversionUtils.convertStringToRdf(xml); + .toString(getClass().getClassLoader().getResourceAsStream("sample_completeness.rdf"), StandardCharsets.UTF_8); + RDF rdf = rdfConversionUtils.convertStringToRdf(xml); EnrichmentUtils.setAdditionalData(rdf); EuropeanaAggregationType europeanaAggregationType = rdf.getEuropeanaAggregationList().stream() .findAny().orElse(null); diff --git a/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/RemoteEntityResolverTest.java b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/RemoteEntityResolverTest.java index b3be97e28..510285f29 100644 --- a/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/RemoteEntityResolverTest.java +++ b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/RemoteEntityResolverTest.java @@ -14,6 +14,7 @@ import static org.mockito.Mockito.verify; import eu.europeana.enrichment.api.exceptions.UnknownException; +import eu.europeana.enrichment.api.external.impl.RemoteEntityResolver; import eu.europeana.enrichment.api.external.model.Agent; import eu.europeana.enrichment.api.external.model.EnrichmentBase; import eu.europeana.enrichment.api.external.model.EnrichmentResultBaseWrapper; diff --git a/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/utils/EntityMergeEngineTest.java b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/utils/EntityMergeEngineTest.java index 1760a1a6a..3dc87169d 100644 --- a/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/utils/EntityMergeEngineTest.java +++ b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/utils/EntityMergeEngineTest.java @@ -44,6 +44,8 @@ public class EntityMergeEngineTest { + private static final RdfConversionUtils rdfConversionUtils = new RdfConversionUtils(); + private static Place createPlace() { Place place = new Place(); @@ -605,7 +607,7 @@ public void testMergePlace() throws SerializationException { verifyPlace((Place) inputList.get(2), rdf.getPlaceList().get(2)); // Convert RDF to string as extra test that everything is OK. - RdfConversionUtils.convertRdfToString(rdf); + rdfConversionUtils.convertRdfToString(rdf); } @Test @@ -645,7 +647,7 @@ public void testMergeOtherTypes() throws SerializationException { verifyOrganization((Organization) inputList.get(5), rdf.getOrganizationList().get(0)); // Convert RDF to string as extra test that everything is OK. - RdfConversionUtils.convertRdfToString(rdf); + rdfConversionUtils.convertRdfToString(rdf); } @Test diff --git a/metis-enrichment/metis-enrichment-common/pom.xml b/metis-enrichment/metis-enrichment-common/pom.xml index 222ef65b6..fa3be2982 100644 --- a/metis-enrichment/metis-enrichment-common/pom.xml +++ b/metis-enrichment/metis-enrichment-common/pom.xml @@ -4,15 +4,18 @@ metis-enrichment eu.europeana.metis - 6 + 7 metis-enrichment-common + + 2.0 + + eu.europeana.metis metis-schema - ${project.version} eu.europeana.metis @@ -35,7 +38,7 @@ javax.xml.bind - jaxb-api + jaxb-api org.glassfish.jaxb @@ -59,13 +62,43 @@ org.apache.commons commons-lang3 + + org.apache.commons + commons-collections4 + io.swagger swagger-annotations ${version.swagger.annotations} + + + eu.europeana.api + entity-api-client + ${version.entity-api-client} + + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + + + com.fasterxml.jackson.core + jackson-databind + + + org.slf4j + jcl-over-slf4j + + + org.apache.logging.log4j + * + + + + + + org.mockito + mockito-core + - - - diff --git a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/ClientEntityResolver.java b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/ClientEntityResolver.java new file mode 100644 index 000000000..5f505e9b2 --- /dev/null +++ b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/ClientEntityResolver.java @@ -0,0 +1,245 @@ +package eu.europeana.enrichment.api.external.impl; + +import static eu.europeana.metis.network.ExternalRequestUtil.retryableExternalRequestForNetworkExceptionsThrowing; +import static java.lang.String.format; +import static java.util.function.Predicate.not; +import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; +import static org.apache.commons.collections4.ListUtils.partition; + +import com.fasterxml.jackson.core.JsonProcessingException; +import eu.europeana.enrichment.api.exceptions.UnknownException; +import eu.europeana.enrichment.api.external.model.EnrichmentBase; +import eu.europeana.enrichment.api.internal.EntityResolver; +import eu.europeana.enrichment.api.internal.ReferenceTerm; +import eu.europeana.enrichment.api.internal.SearchTerm; +import eu.europeana.enrichment.utils.EnrichmentBaseConverter; +import eu.europeana.enrichment.utils.LanguageCodeConverter; +import eu.europeana.entity.client.web.EntityClientApi; +import eu.europeana.entitymanagement.definitions.model.Entity; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.commons.lang3.StringUtils; + +/** + * An entity resolver that works by accessing a service via Entity Client API and obtains entities from Entity Management API + * + * @author Srishti.singh@europeana.eu + */ +public class ClientEntityResolver implements EntityResolver { + + private final int batchSize; + private final LanguageCodeConverter languageCodeConverter; + private final EntityClientApi entityClientApi; + + + /** + * Constructor with required parameters. + * + * @param entityClientApi the entity client api + * @param batchSize the batch size + */ + public ClientEntityResolver(EntityClientApi entityClientApi, int batchSize) { + this.batchSize = batchSize; + this.languageCodeConverter = new LanguageCodeConverter(); + this.entityClientApi = entityClientApi; + } + + @Override + public Map> resolveByText(Set searchTerms) { + return performInBatches(searchTerms); + } + + @Override + public Map> resolveByUri(Set referenceTerms) { + return performInBatches(referenceTerms, true); + } + + @Override + public Map resolveById(Set referenceTerms) { + final Map> batches = performInBatches(referenceTerms); + return convertToMapWithSingleValues(batches); + } + + private Map> performInBatches(Set inputValues) { + return performInBatches(inputValues, false); + } + + private HashMap convertToMapWithSingleValues( + Map> batches) { + return batches.entrySet().stream().collect(HashMap::new, (map, entry) -> map.put(entry.getKey(), + entry.getValue().stream().findFirst().orElse(null)), HashMap::putAll); + } + + /** + * Perform search in batches. + * + * @param the input value type. Can be OR + * @param inputValues the set of values for which enrichment will be performed + * @param uriSearch boolean indicating if it is an uri search or not(then it is an id search) + * @return the results mapped per input value + */ + private Map> performInBatches(Set inputValues, boolean uriSearch) { + final Map> result = new HashMap<>(); + for (List batch : partition(new ArrayList<>(inputValues), batchSize)) { + result.putAll(performBatch(uriSearch, batch)); + } + return result; + } + + private Map> performBatch(boolean uriSearch, List batch) { + final Map> result = new HashMap<>(); + // TODO: 02/06/2022 This is actually bypassing the batching.. This is the selected way to perform this for now. + for (I batchItem : batch) { + List enrichmentBaseList = performItem(batchItem, uriSearch); + result.put(batchItem, enrichmentBaseList); + } + return result; + } + + private List performItem(I batchItem, boolean uriSearch) { + List entities = resolveEntities(batchItem, uriSearch); + List enrichmentBases = new ArrayList<>(); + if (isNotEmpty(entities)) { + entities = extendEntitiesWithParents(entities); + enrichmentBases = convertToEnrichmentBase(entities); + } + return enrichmentBases; + } + + private List resolveEntities(I batchItem, boolean uriSearch) { + if (batchItem instanceof ReferenceTerm) { + return resolveReference((ReferenceTerm) batchItem, uriSearch); + } else { + return resolveTextSearch((SearchTerm) batchItem); + } + } + + /** + * Get entities based on a reference. + *

    We always check first if the reference resembles a euroepeana entity identifier and if so then we search by id..

    + *

    For invocations that are uri searches({@code uriSearch} equals true) then we also invoke the remote uri search.

    + *

    For uri searches, this resembles the metis implementation where the about search is invoked and if no result return then + * a second invocation on the owlSameAs is performed.

    + * + * @param referenceTerm the reference term + * @param uriSearch indicates if the search is an uri or an id search + * @return the list of entities + */ + private List resolveReference(ReferenceTerm referenceTerm, boolean uriSearch) { + final String referenceValue = referenceTerm.getReference().toString(); + + List result = new ArrayList<>(); + if (europeanaLinkPattern.matcher(referenceValue).matches()) { + result = Optional.ofNullable(retryableExternalRequestForNetworkExceptionsThrowing( + () -> entityClientApi.getEntityById(referenceValue))).map(List::of).orElse(Collections.emptyList()); + } else if (uriSearch) { + result = retryableExternalRequestForNetworkExceptionsThrowing( + () -> entityClientApi.getEntityByUri(referenceValue)); + } + return result; + } + + /** + * Get entities by text search. + *

    + * The result will always be a list of size 1. Internally the remote request might return more than one entities which in that + * case the return of this method will be an empty list. That is because the remote request would be ambiguous and therefore we + * do not know which of the entities is actually intended. + *

    + *

    + * ATTENTION: The described discarding of entities applies correctly in the case where the remote request does NOT + * contain parent entities and that the parent entities are fetched remotely i.e. {@link #extendEntitiesWithParents}. + *

    + * + * @param searchTerm the text search term + * @return the list of entities(at this point of size 0 or 1) + */ + private List resolveTextSearch(SearchTerm searchTerm) { + final String entityTypesConcatenated = searchTerm.getCandidateTypes().stream() + .map(entityType -> entityType.name().toLowerCase(Locale.US)) + .collect(Collectors.joining(",")); + final String language = languageCodeConverter.convertLanguageCode(searchTerm.getLanguage()); + final List entities; + try { + entities = retryableExternalRequestForNetworkExceptionsThrowing( + () -> entityClientApi.getEnrichment(searchTerm.getTextValue(), language, entityTypesConcatenated, null)); + return entities.size() == 1 ? entities : Collections.emptyList(); + } catch (JsonProcessingException e) { + throw new UnknownException( + format("SearchTerm request failed for textValue: %s, language: %s, entityTypes: %s.", searchTerm.getTextValue(), + searchTerm.getLanguage(), entityTypesConcatenated), e); + } + } + + /** + * Creates a copy list that is then extended with any parents found. + * + * @param entities the entities + * @return the extended entities + */ + private List extendEntitiesWithParents(List entities) { + //Copy list so that we can extend + final ArrayList copyEntities = new ArrayList<>(entities); + return findParentEntitiesRecursive(copyEntities, copyEntities); + } + + /** + * Converts the list of entities to a list of {@link EnrichmentBase}s. + * + * @param entities the entities + * @return the converted list + */ + private List convertToEnrichmentBase(List entities) { + return EnrichmentBaseConverter.convertEntitiesToEnrichmentBase(entities); + } + + /** + * Finds parent entities and extends recursively. + *

    For each recursion it will, iterate over {@link Entity#getIsPartOfArray} bypassing blank values and entities already + * encountered. Each recursion will extended list if more parents have been found. + *

    + * + * @param collectedEntities the collected entities + * @param children the children to check their parents for + * @return the extended list of entities + */ + private List findParentEntitiesRecursive(List collectedEntities, List children) { + List parentEntities = + Stream.ofNullable(children).flatMap(Collection::stream) + .map(Entity::getIsPartOfArray).filter(Objects::nonNull).flatMap(Collection::stream) + .filter(StringUtils::isNotBlank) + .filter(not(parentEntityId -> doesEntityExist(parentEntityId, collectedEntities))) + .map(parentEntityId -> retryableExternalRequestForNetworkExceptionsThrowing( + () -> entityClientApi.getEntityById(parentEntityId))) + .filter(Objects::nonNull) + .collect(Collectors.toCollection(ArrayList::new)); + + if (isNotEmpty(parentEntities)) { + collectedEntities.addAll(parentEntities); + //Now check again parents of parents + findParentEntitiesRecursive(collectedEntities, parentEntities); + } + return collectedEntities; + } + + /** + * Checks if an entity identifier matches an identifier of the entities provided. + * + * @param entityIdToCheck the entity identifier to check + * @param entities the entity list + * @return true if it matches otherwise false + */ + private static boolean doesEntityExist(String entityIdToCheck, List entities) { + return entities.stream().anyMatch(entity -> entity.getEntityId().equals(entityIdToCheck)); + } +} diff --git a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/EntityResolverType.java b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/EntityResolverType.java new file mode 100644 index 000000000..48dea0d79 --- /dev/null +++ b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/EntityResolverType.java @@ -0,0 +1,8 @@ +package eu.europeana.enrichment.api.external.impl; + +/** + * Entity resolver type for choosing the requested implementation + */ +public enum EntityResolverType { + PERSISTENT, ENTITY_CLIENT +} diff --git a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/RemoteEntityResolver.java b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/RemoteEntityResolver.java similarity index 80% rename from metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/RemoteEntityResolver.java rename to metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/RemoteEntityResolver.java index 10f8290a9..f8db213b8 100644 --- a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/RemoteEntityResolver.java +++ b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/RemoteEntityResolver.java @@ -1,9 +1,10 @@ -package eu.europeana.enrichment.rest.client.enrichment; +package eu.europeana.enrichment.api.external.impl; import static eu.europeana.metis.network.ExternalRequestUtil.retryableExternalRequestForNetworkExceptions; import static eu.europeana.metis.utils.RestEndpoints.ENRICH_ENTITY_EQUIVALENCE; import static eu.europeana.metis.utils.RestEndpoints.ENRICH_ENTITY_ID; import static eu.europeana.metis.utils.RestEndpoints.ENRICH_ENTITY_SEARCH; +import static org.apache.commons.collections4.ListUtils.partition; import eu.europeana.enrichment.api.exceptions.UnknownException; import eu.europeana.enrichment.api.external.EnrichmentReference; @@ -34,20 +35,24 @@ import org.springframework.http.MediaType; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; - /** - * An entity resolver that works by accessing a service through HTTP/REST and obtains entities from - * there. + * An entity resolver that works by accessing a service through HTTP/REST and obtains entities from there. */ public class RemoteEntityResolver implements EntityResolver { private final int batchSize; - private final RestTemplate template; + private final RestTemplate restTemplate; private final URL enrichmentServiceUrl; - public RemoteEntityResolver(URL enrichmentServiceUrl, int batchSize, RestTemplate template) { + /** + * Constructor with required parameters + * @param enrichmentServiceUrl the enrichment service url + * @param batchSize the batch size + * @param restTemplate the rest template + */ + public RemoteEntityResolver(URL enrichmentServiceUrl, int batchSize, RestTemplate restTemplate) { this.enrichmentServiceUrl = enrichmentServiceUrl; - this.template = template; + this.restTemplate = restTemplate; this.batchSize = batchSize; } @@ -55,7 +60,7 @@ public RemoteEntityResolver(URL enrichmentServiceUrl, int batchSize, RestTemplat public Map> resolveByText(Set searchTerms) { final Function, EnrichmentSearch> inputFunction = partition -> { final List searchValues = partition.stream() - .map(term -> new SearchValue(term.getTextValue(), term.getLanguage(), + .map(term -> new SearchValue(term.getTextValue(), term.getLanguage(), term.getCandidateTypes().toArray(EntityType[]::new))) .collect(Collectors.toList()); final EnrichmentSearch enrichmentSearch = new EnrichmentSearch(); @@ -97,33 +102,21 @@ private Map performInBatches(String endpointPath, Set inputVa try { final URI parentUri = enrichmentServiceUrl.toURI(); uri = new URI(parentUri.getScheme(), parentUri.getUserInfo(), parentUri.getHost(), - parentUri.getPort(), parentUri.getPath() + "/" + endpointPath, - parentUri.getQuery(), parentUri.getFragment()).normalize(); + parentUri.getPort(), parentUri.getPath() + "/" + endpointPath, + parentUri.getQuery(), parentUri.getFragment()).normalize(); } catch (URISyntaxException e) { throw new UnknownException( - "URL syntax issue with service url: " + enrichmentServiceUrl + ".", e); + "URL syntax issue with service url: " + enrichmentServiceUrl + ".", e); } - // Create partitions - final List> partitions = new ArrayList<>(); - partitions.add(new ArrayList<>()); - inputValues.forEach(item -> { - List currentPartition = partitions.get(partitions.size() - 1); - if (currentPartition.size() >= batchSize) { - currentPartition = new ArrayList<>(); - partitions.add(currentPartition); - } - currentPartition.add(item); - }); - - // Process partitions + final List> batches = partition(new ArrayList<>(inputValues), batchSize); final Map result = new HashMap<>(); - for (List partition : partitions) { - final EnrichmentResultList enrichmentResultList = executeRequest(uri, bodyCreator, partition); - for (int i = 0; i < partition.size(); i++) { - final I inputItem = partition.get(i); + for (List batch : batches) { + final EnrichmentResultList enrichmentResultList = executeRequest(uri, bodyCreator, batch); + for (int i = 0; i < batch.size(); i++) { + final I inputItem = batch.get(i); Optional.ofNullable(enrichmentResultList - .getEnrichmentBaseResultWrapperList().get(i)) + .getEnrichmentBaseResultWrapperList().get(i)) .map(EnrichmentResultBaseWrapper::getEnrichmentBaseList) .filter(list -> !list.isEmpty()) .map(resultParser).ifPresent(resultItem -> result.put(inputItem, resultItem)); @@ -135,10 +128,10 @@ private Map performInBatches(String endpointPath, Set inputVa } private EnrichmentResultList executeRequest(URI uri, Function, B> bodyCreator, - List partition) { + List batch) { // Create the request - final B body = bodyCreator.apply(partition); + final B body = bodyCreator.apply(batch); final HttpHeaders headers = new HttpHeaders(); if (body != null) { headers.setContentType(MediaType.APPLICATION_JSON); @@ -150,7 +143,7 @@ private EnrichmentResultList executeRequest(URI uri, Function, B> final EnrichmentResultList enrichmentResultList; try { enrichmentResultList = retryableExternalRequestForNetworkExceptions( - () -> template.postForObject(uri, httpEntity, EnrichmentResultList.class)); + () -> restTemplate.postForObject(uri, httpEntity, EnrichmentResultList.class)); } catch (RestClientException e) { throw new UnknownException("Enrichment client POST call failed: " + uri + ".", e); @@ -158,7 +151,7 @@ private EnrichmentResultList executeRequest(URI uri, Function, B> if (enrichmentResultList == null) { throw new UnknownException("Empty body from server (" + uri + ")."); } - if (enrichmentResultList.getEnrichmentBaseResultWrapperList().size() != partition.size()) { + if (enrichmentResultList.getEnrichmentBaseResultWrapperList().size() != batch.size()) { throw new UnknownException("Server returned unexpected number of results (" + uri + ")."); } diff --git a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/model/Agent.java b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/model/Agent.java index a28c4c5a6..83157d09a 100644 --- a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/model/Agent.java +++ b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/model/Agent.java @@ -11,4 +11,14 @@ @XmlAccessorType(XmlAccessType.FIELD) public class Agent extends AgentBase { + public Agent() {} + + public Agent(eu.europeana.entitymanagement.definitions.model.Agent entity) { + super(entity); + } + + public Agent(eu.europeana.entitymanagement.definitions.model.Organization entity) { + super(entity); + } + } diff --git a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/model/AgentBase.java b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/model/AgentBase.java index 158b86ef3..f43c432a2 100644 --- a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/model/AgentBase.java +++ b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/model/AgentBase.java @@ -1,5 +1,12 @@ package eu.europeana.enrichment.api.external.model; +import static eu.europeana.enrichment.utils.EntityValuesConverter.convertListToLabel; +import static eu.europeana.enrichment.utils.EntityValuesConverter.convertListToLabelResource; +import static eu.europeana.enrichment.utils.EntityValuesConverter.convertListToPart; +import static eu.europeana.enrichment.utils.EntityValuesConverter.convertListToResource; +import static eu.europeana.enrichment.utils.EntityValuesConverter.convertMapToLabels; +import static eu.europeana.enrichment.utils.EntityValuesConverter.convertResourceOrLiteral; + import java.util.ArrayList; import java.util.List; import javax.xml.bind.annotation.XmlAccessType; @@ -53,6 +60,19 @@ public abstract class AgentBase extends EnrichmentBase { @XmlElement(name = "sameAs", namespace = "http://www.w3.org/2002/07/owl#") private List sameAs = new ArrayList<>(); + protected AgentBase() { + } + + protected AgentBase(eu.europeana.entitymanagement.definitions.model.Organization organization) { + super(organization); + } + + // Used for creating XML entity from EM model class + protected AgentBase(eu.europeana.entitymanagement.definitions.model.Agent agent) { + super(agent); + init(agent); + } + public List     + + org.springframework + spring-webmvc + ${version.spring} + org.springframework spring-core @@ -74,6 +79,17 @@ springfox-swagger-ui ${version.swagger} + + + org.springframework.boot + spring-boot-autoconfigure + ${version.spring-boot-autoconfigure} + com.jayway.jsonpath json-path-assert @@ -93,11 +109,6 @@ javax.servlet-api ${version.servlet.api} - - org.springframework - spring-webmvc - ${version.spring} - org.springframework spring-test diff --git a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/EnrichmentController.java b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/EnrichmentController.java index e9490095f..87c771ea9 100644 --- a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/EnrichmentController.java +++ b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/EnrichmentController.java @@ -13,9 +13,8 @@ import io.swagger.annotations.ApiParam; import io.swagger.annotations.ApiResponse; import io.swagger.annotations.ApiResponses; -import java.util.Collections; -import java.util.List; -import java.util.Optional; + +import java.util.*; import java.util.stream.Collectors; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; diff --git a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/Application.java b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/Application.java index 171b8f1e4..ca30e0818 100644 --- a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/Application.java +++ b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/Application.java @@ -1,15 +1,23 @@ package eu.europeana.enrichment.rest.config; +import static java.lang.String.format; + import com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule; import com.mongodb.client.MongoClient; import eu.europeana.corelib.web.socks.SocksProxy; +import eu.europeana.enrichment.api.external.impl.ClientEntityResolver; +import eu.europeana.enrichment.api.external.impl.EntityResolverType; +import eu.europeana.enrichment.api.internal.EntityResolver; import eu.europeana.enrichment.service.EnrichmentService; import eu.europeana.enrichment.service.PersistentEntityResolver; import eu.europeana.enrichment.service.dao.EnrichmentDao; +import eu.europeana.entity.client.config.EntityClientConfiguration; +import eu.europeana.entity.client.web.EntityClientApiImpl; import eu.europeana.metis.mongo.connection.MongoClientProvider; import eu.europeana.metis.mongo.connection.MongoProperties; -import java.util.Collections; +import java.util.Properties; import javax.annotation.PreDestroy; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; @@ -19,24 +27,16 @@ import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder; import org.springframework.web.servlet.config.annotation.EnableWebMvc; -import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; -import org.springframework.web.servlet.config.annotation.ViewControllerRegistry; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; -import springfox.documentation.builders.PathSelectors; -import springfox.documentation.builders.RequestHandlerSelectors; -import springfox.documentation.service.ApiInfo; -import springfox.documentation.service.Contact; -import springfox.documentation.spi.DocumentationType; -import springfox.documentation.spring.web.plugins.Docket; -import springfox.documentation.swagger2.annotations.EnableSwagger2; +/** + * Main Spring Configuration class + */ @Configuration @ComponentScan(basePackages = {"eu.europeana.enrichment.rest", "eu.europeana.enrichment.rest.exception"}) @PropertySource("classpath:enrichment.properties") @EnableWebMvc -@EnableSwagger2 -public class Application implements WebMvcConfigurer, InitializingBean { +public class Application implements InitializingBean { //Socks proxy @Value("${socks.proxy.enabled}") @@ -59,6 +59,22 @@ public class Application implements WebMvcConfigurer, InitializingBean { @Value("${enrichment.mongo.application.name}") private String enrichmentMongoApplicationName; + @Value("${enrichment.batch.size:20}") + private int enrichmentBatchSize; + + @Value("${enrichment.entity.resolver.type:PERSISTENT}") + private EntityResolverType entityResolverType; + + @Value("${entity.management.url}") + private String entityManagementUrl; + + @Value("${entity.api.url}") + private String entityApiUrl; + + @Value("${entity.api.key}") + private String entityApiKey; + + private MongoClient mongoClient; /** @@ -71,21 +87,30 @@ public void afterPropertiesSet() { } } - @Override - public void addViewControllers(ViewControllerRegistry registry) { - registry.addRedirectViewController("/", "/swagger-ui/index.html"); - } - - @Override - public void addResourceHandlers(ResourceHandlerRegistry registry) { - registry.addResourceHandler("/swagger-ui/**") - .addResourceLocations("classpath:/META-INF/resources/webjars/springfox-swagger-ui/") - .resourceChain(false); + @Bean + EnrichmentService getEnrichmentService(EntityResolver entityResolver) { + return new EnrichmentService(entityResolver); } @Bean - EnrichmentService getEnrichmentService(EnrichmentDao enrichmentDao) { - return new EnrichmentService(new PersistentEntityResolver(enrichmentDao)); + EntityResolver getEntityResolver() { + final EntityResolver entityResolver; + if (entityResolverType == EntityResolverType.ENTITY_CLIENT) { + //Sanity check + if (StringUtils.isAnyBlank(entityManagementUrl, entityApiUrl, entityApiKey)) { + throw new IllegalArgumentException( + format("Requested %s resolver but configuration is missing", EntityResolverType.ENTITY_CLIENT)); + } + final Properties properties = new Properties(); + properties.put("entity.management.url", entityManagementUrl); + properties.put("entity.api.url", entityApiUrl); + properties.put("entity.api.key", entityApiKey); + entityResolver = new ClientEntityResolver(new EntityClientApiImpl(new EntityClientConfiguration(properties)), + enrichmentBatchSize); + } else { + entityResolver = new PersistentEntityResolver(new EnrichmentDao(mongoClient, enrichmentMongoDatabase)); + } + return entityResolver; } @Bean @@ -99,11 +124,6 @@ MongoClient getMongoClient() { return mongoClient; } - @Bean - EnrichmentDao getEnrichmentDao(MongoClient mongoClient) { - return new EnrichmentDao(mongoClient, enrichmentMongoDatabase); - } - @Bean public Jackson2ObjectMapperBuilder objectMapperBuilder() { Jackson2ObjectMapperBuilder builder = new Jackson2ObjectMapperBuilder(); @@ -126,29 +146,4 @@ public void close() { mongoClient.close(); } } - - @Bean - public Docket api() { - return new Docket(DocumentationType.SWAGGER_2) - .useDefaultResponseMessages(false) - .select() - .apis(RequestHandlerSelectors.any()) - .paths(PathSelectors.regex("/.*")) - .build() - .apiInfo(apiInfo()); - } - - private ApiInfo apiInfo() { - Contact contact = new Contact("Europeana", "http:\\www.europeana.eu", - "development@europeana.eu"); - - return new ApiInfo( - "Enrichment REST API", - "Enrichment REST API for Europeana", - "v1", - "API TOS", - contact, - "EUPL Licence v1.2", - "https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12", Collections.emptyList()); - } } diff --git a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/SwaggerConfig.java b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/SwaggerConfig.java new file mode 100644 index 000000000..50d8fe5da --- /dev/null +++ b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/SwaggerConfig.java @@ -0,0 +1,65 @@ +package eu.europeana.enrichment.rest.config; + +import java.util.Collections; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; +import org.springframework.web.servlet.config.annotation.ViewControllerRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; +import springfox.documentation.builders.PathSelectors; +import springfox.documentation.builders.RequestHandlerSelectors; +import springfox.documentation.service.ApiInfo; +import springfox.documentation.service.Contact; +import springfox.documentation.spi.DocumentationType; +import springfox.documentation.spring.web.plugins.Docket; +import springfox.documentation.swagger2.annotations.EnableSwagger2; + +/** + * Config for Swagger documentation. + */ +@Configuration +@EnableSwagger2 +public class SwaggerConfig implements WebMvcConfigurer { + + @Override + public void addViewControllers(ViewControllerRegistry registry) { + registry.addRedirectViewController("/", "/swagger-ui/index.html"); + } + + @Override + public void addResourceHandlers(ResourceHandlerRegistry registry) { + registry.addResourceHandler("/swagger-ui/**") + .addResourceLocations("classpath:/META-INF/resources/webjars/springfox-swagger-ui/") + .resourceChain(false); + } + + /** + * Initialize Swagger Documentation + * + * @return Swagger Docket for this API + */ + @Bean + public Docket api() { + return new Docket(DocumentationType.SWAGGER_2) + .useDefaultResponseMessages(false) + .select() + .apis(RequestHandlerSelectors.any()) + .paths(PathSelectors.regex("/.*")) + .build() + .apiInfo(apiInfo()); + } + + private ApiInfo apiInfo() { + Contact contact = new Contact("Europeana", "http:\\www.europeana.eu", + "development@europeana.eu"); + + return new ApiInfo( + "Enrichment REST API", + "Enrichment REST API for Europeana", + "v1", + "API TOS", + contact, + "EUPL Licence v1.2", + "https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12", Collections.emptyList()); + } +} diff --git a/metis-enrichment/metis-enrichment-rest/src/main/resources/enrichment.properties.example b/metis-enrichment/metis-enrichment-rest/src/main/resources/enrichment.properties.example index 68ad705a3..69b503833 100644 --- a/metis-enrichment/metis-enrichment-rest/src/main/resources/enrichment.properties.example +++ b/metis-enrichment/metis-enrichment-rest/src/main/resources/enrichment.properties.example @@ -9,4 +9,19 @@ socks.proxy.password= enrichment.mongo.host= enrichment.mongo.database= enrichment.mongo.port= -enrichment.mongo.application.name= \ No newline at end of file +enrichment.mongo.application.name= + + +#If not provided defaults to 20 +enrichment.batch.size= +#Options PERSISTENT, ENTITY_CLIENT. Defaults to PERSISTENT +enrichment.entity.resolver.type= + +#Entity Management Base Url for Entity Retrieval +entity.management.url= + +#Entity Api V2 url for search and suggest +entity.api.url= + +#Api key +entity.api.key= \ No newline at end of file diff --git a/metis-enrichment/metis-enrichment-rest/src/test/java/eu/europeana/enrichment/rest/exception/RestResponseExceptionHandlerTest.java b/metis-enrichment/metis-enrichment-rest/src/test/java/eu/europeana/enrichment/rest/exception/RestResponseExceptionHandlerTest.java index 7f4ae493a..a2eee4679 100644 --- a/metis-enrichment/metis-enrichment-rest/src/test/java/eu/europeana/enrichment/rest/exception/RestResponseExceptionHandlerTest.java +++ b/metis-enrichment/metis-enrichment-rest/src/test/java/eu/europeana/enrichment/rest/exception/RestResponseExceptionHandlerTest.java @@ -7,19 +7,18 @@ import eu.europeana.metis.exception.StructuredExceptionWrapper; import javax.servlet.http.HttpServletResponse; - import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import org.springframework.http.HttpStatus; -public class RestResponseExceptionHandlerTest { +class RestResponseExceptionHandlerTest { private static final RestResponseExceptionHandler REST_RESPONSE_EXCEPTION_HANDLER = new RestResponseExceptionHandler(); private static final String ERROR_MESSAGE = "error message"; ArgumentCaptor valueCaptor = ArgumentCaptor.forClass(Integer.class); @Test - void testHandleResponse(){ + void testHandleResponse() { HttpServletResponse response = mock(HttpServletResponse.class); Exception exception = new Exception(ERROR_MESSAGE); diff --git a/metis-enrichment/metis-enrichment-service/.gitignore b/metis-enrichment/metis-enrichment-service/.gitignore deleted file mode 100644 index 0b021c7ac..000000000 --- a/metis-enrichment/metis-enrichment-service/.gitignore +++ /dev/null @@ -1 +0,0 @@ -**.properties \ No newline at end of file diff --git a/metis-enrichment/metis-enrichment-service/pom.xml b/metis-enrichment/metis-enrichment-service/pom.xml index e7a20c1f0..60c468b0c 100644 --- a/metis-enrichment/metis-enrichment-service/pom.xml +++ b/metis-enrichment/metis-enrichment-service/pom.xml @@ -4,7 +4,7 @@ metis-enrichment eu.europeana.metis - 6 + 7 metis-enrichment-service jar diff --git a/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/EnrichmentService.java b/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/EnrichmentService.java index 7b6fb0805..7e5582a62 100644 --- a/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/EnrichmentService.java +++ b/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/EnrichmentService.java @@ -4,20 +4,18 @@ import eu.europeana.enrichment.api.external.SearchValue; import eu.europeana.enrichment.api.external.model.EnrichmentBase; import eu.europeana.enrichment.api.external.model.EnrichmentResultBaseWrapper; +import eu.europeana.enrichment.api.internal.EntityResolver; import eu.europeana.enrichment.api.internal.ReferenceTerm; import eu.europeana.enrichment.api.internal.ReferenceTermImpl; import eu.europeana.enrichment.api.internal.SearchTerm; import eu.europeana.enrichment.api.internal.SearchTermImpl; -import eu.europeana.enrichment.internal.model.OrganizationEnrichmentEntity; import eu.europeana.enrichment.service.dao.EnrichmentDao; import java.net.MalformedURLException; import java.net.URL; import java.util.Collections; -import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.slf4j.Logger; @@ -26,26 +24,26 @@ import org.springframework.stereotype.Service; /** - * Contains functionality for accessing entities from the enrichment database using {@link - * EnrichmentDao}. + * Contains functionality for accessing entities from the enrichment database using {@link EnrichmentDao}. * * @author Simon Tzanakis * @since 2020-07-16 */ @Service public class EnrichmentService { + private static final Logger LOGGER = LoggerFactory.getLogger(EnrichmentService.class); - private final PersistentEntityResolver persistentEntityResolver; + private final EntityResolver entityResolver; /** * Parameter constructor. * - * @param persistentEntityResolver the entity resolver + * @param entityResolver the entity resolver */ @Autowired - public EnrichmentService(PersistentEntityResolver persistentEntityResolver) { - this.persistentEntityResolver = persistentEntityResolver; + public EnrichmentService(EntityResolver entityResolver) { + this.entityResolver = entityResolver; } /** @@ -57,12 +55,12 @@ public EnrichmentService(PersistentEntityResolver persistentEntityResolver) { public List enrichByEnrichmentSearchValues( List searchValues) { final List orderedSearchTerms = searchValues.stream().map( - search -> new SearchTermImpl(search.getValue(), search.getLanguage(), - Set.copyOf(search.getEntityTypes()))).collect(Collectors.toList()); - final Map> result = persistentEntityResolver - .resolveByText(new HashSet<>(orderedSearchTerms)); + search -> new SearchTermImpl(search.getValue(), search.getLanguage(), + Set.copyOf(search.getEntityTypes()))).collect(Collectors.toList()); + final Map> result = entityResolver + .resolveByText(new HashSet<>(orderedSearchTerms)); return orderedSearchTerms.stream().map(result::get).map(EnrichmentResultBaseWrapper::new) - .collect(Collectors.toList()); + .collect(Collectors.toList()); } /** @@ -74,9 +72,9 @@ public List enrichByEnrichmentSearchValues( public List enrichByEquivalenceValues(ReferenceValue referenceValue) { try { final ReferenceTerm referenceTerm = new ReferenceTermImpl( - new URL(referenceValue.getReference()), Set.copyOf(referenceValue.getEntityTypes())); - return persistentEntityResolver.resolveByUri(Set.of(referenceTerm)) - .getOrDefault(referenceTerm, Collections.emptyList()); + new URL(referenceValue.getReference()), Set.copyOf(referenceValue.getEntityTypes())); + return entityResolver.resolveByUri(Set.of(referenceTerm)) + .getOrDefault(referenceTerm, Collections.emptyList()); } catch (MalformedURLException e) { LOGGER.debug("There was a problem converting the input to ReferenceTermType"); throw new IllegalArgumentException("The input values are invalid", e); @@ -92,73 +90,12 @@ public List enrichByEquivalenceValues(ReferenceValue referenceVa public EnrichmentBase enrichById(String entityAbout) { try { final ReferenceTerm referenceTerm = new ReferenceTermImpl(new URL(entityAbout), - new HashSet<>()); - return persistentEntityResolver.resolveById(Set.of(referenceTerm)).get(referenceTerm); + new HashSet<>()); + return entityResolver.resolveById(Set.of(referenceTerm)).get(referenceTerm); } catch (MalformedURLException e) { LOGGER.debug("There was a problem converting the input to ReferenceTermType"); throw new IllegalArgumentException("The input values are invalid", e); } } - /* --- Organization specific methods, used by the annotations api --- */ - - /** - * Save an organization to the database - * - * @param organizationEnrichmentEntity the organization to save - * @param created the created date to be used - * @param updated the updated date to be used - * @return the saved organization - */ - public OrganizationEnrichmentEntity saveOrganization( - OrganizationEnrichmentEntity organizationEnrichmentEntity, Date created, Date updated) { - return persistentEntityResolver.saveOrganization(organizationEnrichmentEntity, created, updated); - } - - /** - * Return the list of ids for existing organizations from database - * - * @param organizationIds The organization ids to check existence - * @return list of ids of existing organizations - */ - public List findExistingOrganizations(List organizationIds) { - return persistentEntityResolver.findExistingOrganizations(organizationIds); - } - - /** - * Get an organization by uri - * - * @param uri The EDM organization uri - * @return OrganizationImpl object - */ - public Optional getOrganizationByUri(String uri) { - return persistentEntityResolver.getOrganizationByUri(uri); - } - - /** - * Delete organizations from database by given organization ids - * - * @param organizationIds The organization ids - */ - public void deleteOrganizations(List organizationIds) { - persistentEntityResolver.deleteOrganizations(organizationIds); - } - - /** - * This method removes organization from database by given organization id. - * - * @param organizationId The organization id - */ - public void deleteOrganization(String organizationId) { - persistentEntityResolver.deleteOrganization(organizationId); - } - - /** - * Get the date of the latest updated organization. - * - * @return the date of the latest updated organization - */ - public Date getDateOfLastUpdatedOrganization() { - return persistentEntityResolver.getDateOfLastUpdatedOrganization(); - } } diff --git a/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/PersistentEntityResolver.java b/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/PersistentEntityResolver.java index 69d11039b..57d321668 100644 --- a/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/PersistentEntityResolver.java +++ b/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/PersistentEntityResolver.java @@ -5,19 +5,16 @@ import eu.europeana.enrichment.api.internal.ReferenceTerm; import eu.europeana.enrichment.api.internal.SearchTerm; import eu.europeana.enrichment.internal.model.EnrichmentTerm; -import eu.europeana.enrichment.internal.model.OrganizationEnrichmentEntity; import eu.europeana.enrichment.service.dao.EnrichmentDao; +import eu.europeana.enrichment.service.utils.EnrichmentTermsToEnrichmentBaseConverter; import eu.europeana.enrichment.utils.EntityType; +import eu.europeana.enrichment.utils.LanguageCodeConverter; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -26,7 +23,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; -import org.bson.types.ObjectId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,25 +32,11 @@ public class PersistentEntityResolver implements EntityResolver { private static final Logger LOGGER = LoggerFactory.getLogger(PersistentEntityResolver.class); - private static final Set ALL_2CODE_LANGUAGES; - private static final Map ALL_3CODE_TO_2CODE_LANGUAGES; private static final Pattern PATTERN_MATCHING_VERY_BROAD_TIMESPANS = Pattern .compile("http://semium.org/time/(ChronologicalPeriod$|Time$|(AD|BC)[1-9]x{3}$)"); - public static final int THREE_CHARACTER_LANGUAGE_LENGTH = 3; - public static final int TWO_CHARACTER_LANGUAGE_LENGTH = 2; - - static { - HashSet all2CodeLanguages = new HashSet<>(); - Map all3CodeLanguages = new HashMap<>(); - Arrays.stream(Locale.getISOLanguages()).map(Locale::new).forEach(locale -> { - all2CodeLanguages.add(locale.getLanguage()); - all3CodeLanguages.put(locale.getISO3Language(), locale.getLanguage()); - }); - ALL_2CODE_LANGUAGES = Collections.unmodifiableSet(all2CodeLanguages); - ALL_3CODE_TO_2CODE_LANGUAGES = Collections.unmodifiableMap(all3CodeLanguages); - } private final EnrichmentDao enrichmentDao; + private final LanguageCodeConverter languageCodeConverter; /** * Constructor with the persistence dao parameter. @@ -63,6 +45,7 @@ public class PersistentEntityResolver implements EntityResolver { */ public PersistentEntityResolver(EnrichmentDao enrichmentDao) { this.enrichmentDao = enrichmentDao; + languageCodeConverter = new LanguageCodeConverter(); } @Override @@ -141,17 +124,7 @@ private void findEnrichmentEntitiesBySearchTerm( if (!StringUtils.isBlank(value)) { final Set entityTypes = searchTerm.getCandidateTypes(); //Language has to be a valid 2 or 3 code, otherwise we do not use it - final String inputValueLanguage = searchTerm.getLanguage(); - final String language; - if (inputValueLanguage != null - && inputValueLanguage.length() == THREE_CHARACTER_LANGUAGE_LENGTH) { - language = ALL_3CODE_TO_2CODE_LANGUAGES.get(inputValueLanguage); - } else if (inputValueLanguage != null - && inputValueLanguage.length() == TWO_CHARACTER_LANGUAGE_LENGTH) { - language = ALL_2CODE_LANGUAGES.contains(inputValueLanguage) ? inputValueLanguage : null; - } else { - language = null; - } + final String language = languageCodeConverter.convertLanguageCode(searchTerm.getLanguage()); if (CollectionUtils.isEmpty(entityTypes)) { searchTermListMap.put(searchTerm, findEnrichmentTerms(null, value, language)); @@ -188,12 +161,13 @@ private List findEnrichmentTerms(EntityType entityType, String t final List enrichmentTerms = enrichmentDao .getAllEnrichmentTermsByFields(fieldNameMap); final List parentEnrichmentTerms = enrichmentTerms.stream() - .map(this::findParentEntities).flatMap(List::stream).collect(Collectors.toList()); + .map(this::findParentEntities).flatMap(List::stream) + .collect(Collectors.toList()); final List enrichmentBases = new ArrayList<>(); //Convert to EnrichmentBases - enrichmentBases.addAll(Converter.convert(enrichmentTerms)); - enrichmentBases.addAll(Converter.convert(parentEnrichmentTerms)); + enrichmentBases.addAll(EnrichmentTermsToEnrichmentBaseConverter.convert(enrichmentTerms)); + enrichmentBases.addAll(EnrichmentTermsToEnrichmentBaseConverter.convert(parentEnrichmentTerms)); return enrichmentBases; } @@ -250,7 +224,7 @@ private List searchBasesFirstAboutThenOwlSameAs(String reference private List getEnrichmentTermsAndConvert( List> fieldNamesAndValues) { final List enrichmentTerms = getEnrichmentTerms(fieldNamesAndValues); - return Converter.convert(enrichmentTerms); + return EnrichmentTermsToEnrichmentBaseConverter.convert(enrichmentTerms); } private List getEnrichmentTerms(List> fieldNamesAndValues) { @@ -259,87 +233,4 @@ private List getEnrichmentTerms(List> field return enrichmentDao.getAllEnrichmentTermsByFields(fieldNameMap); } - /* --- Organization specific methods, used by the annotations api --- */ - - /** - * Save an organization to the database - * - * @param organizationEnrichmentEntity the organization to save - * @param created the created date to be used - * @param updated the updated date to be used - * @return the saved organization - */ - public OrganizationEnrichmentEntity saveOrganization( - OrganizationEnrichmentEntity organizationEnrichmentEntity, Date created, Date updated) { - - final EnrichmentTerm enrichmentTerm = Converter - .organizationImplToEnrichmentTerm(organizationEnrichmentEntity, created, updated); - - final Optional objectId = enrichmentDao - .getEnrichmentTermObjectIdByField(EnrichmentDao.ENTITY_ABOUT_FIELD, - organizationEnrichmentEntity.getAbout()); - objectId.ifPresent(enrichmentTerm::setId); - - //Save term list - final String id = enrichmentDao.saveEnrichmentTerm(enrichmentTerm); - return enrichmentDao.getEnrichmentTermByField(EnrichmentDao.ID_FIELD, id) - .map(EnrichmentTerm::getEnrichmentEntity).map(OrganizationEnrichmentEntity.class::cast) - .orElse(null); - } - - /** - * Return the list of ids for existing organizations from database - * - * @param organizationIds The organization ids to check existence - * @return list of ids of existing organizations - */ - public List findExistingOrganizations(List organizationIds) { - List existingOrganizationIds = new ArrayList<>(); - for (String id : organizationIds) { - Optional organization = getOrganizationByUri(id); - organization.ifPresent(value -> existingOrganizationIds.add(value.getAbout())); - } - return existingOrganizationIds; - } - - /** - * Get an organization by uri - * - * @param uri The EDM organization uri - * @return OrganizationImpl object - */ - public Optional getOrganizationByUri(String uri) { - final List enrichmentTerm = getEnrichmentTerms( - Collections.singletonList(new ImmutablePair<>(EnrichmentDao.ENTITY_ABOUT_FIELD, uri))); - return enrichmentTerm.stream().findFirst().map(EnrichmentTerm::getEnrichmentEntity) - .map(OrganizationEnrichmentEntity.class::cast); - } - - /** - * Delete organizations from database by given organization ids - * - * @param organizationIds The organization ids - */ - public void deleteOrganizations(List organizationIds) { - enrichmentDao.deleteEnrichmentTerms(EntityType.ORGANIZATION, organizationIds); - } - - /** - * This method removes organization from database by given organization id. - * - * @param organizationId The organization id - */ - public void deleteOrganization(String organizationId) { - deleteOrganizations(Collections.singletonList(organizationId)); - } - - /** - * Get the date of the latest updated organization. - * - * @return the date of the latest updated organization - */ - public Date getDateOfLastUpdatedOrganization() { - return enrichmentDao.getDateOfLastUpdatedEnrichmentTerm(EntityType.ORGANIZATION); - } - } diff --git a/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/Converter.java b/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/utils/EnrichmentTermsToEnrichmentBaseConverter.java similarity index 66% rename from metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/Converter.java rename to metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/utils/EnrichmentTermsToEnrichmentBaseConverter.java index 54d429f11..37587b108 100644 --- a/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/Converter.java +++ b/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/utils/EnrichmentTermsToEnrichmentBaseConverter.java @@ -1,9 +1,14 @@ -package eu.europeana.enrichment.service; +package eu.europeana.enrichment.service.utils; + +import static eu.europeana.enrichment.utils.EntityValuesConverter.convertListToPart; +import static eu.europeana.enrichment.utils.EntityValuesConverter.convertMapToLabels; +import static eu.europeana.enrichment.utils.EntityValuesConverter.convertMultilingualMapToLabel; +import static eu.europeana.enrichment.utils.EntityValuesConverter.convertResourceOrLiteral; +import static eu.europeana.enrichment.utils.EntityValuesConverter.convertToResourceList; import eu.europeana.enrichment.api.external.model.Agent; import eu.europeana.enrichment.api.external.model.Concept; import eu.europeana.enrichment.api.external.model.EnrichmentBase; -import eu.europeana.enrichment.api.external.model.Label; import eu.europeana.enrichment.api.external.model.LabelInfo; import eu.europeana.enrichment.api.external.model.LabelResource; import eu.europeana.enrichment.api.external.model.Organization; @@ -22,16 +27,11 @@ import eu.europeana.enrichment.internal.model.PlaceEnrichmentEntity; import eu.europeana.enrichment.internal.model.TimespanEnrichmentEntity; import eu.europeana.enrichment.utils.EntityType; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.Stream; - import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; @@ -39,18 +39,19 @@ /** * Contains functionality for converting from an incoming Object to a different one. */ -public final class Converter { +public final class EnrichmentTermsToEnrichmentBaseConverter { - private Converter() { + private EnrichmentTermsToEnrichmentBaseConverter() { } /** * Converter from list of {@link EnrichmentTerm} to list of {@link EnrichmentBase}. + * * @param enrichmentTerms the enrichment terms to convert * @return the converted enrichment bases */ public static List convert(List enrichmentTerms) { - return enrichmentTerms.stream().map(Converter::convert).collect(Collectors.toList()); + return enrichmentTerms.stream().map(EnrichmentTermsToEnrichmentBaseConverter::convert).collect(Collectors.toList()); } /** @@ -93,14 +94,14 @@ private static TimeSpan convertTimespan(TimespanEnrichmentEntity timespanEnrichm TimeSpan output = new TimeSpan(); output.setAbout(timespanEnrichmentEntity.getAbout()); - output.setPrefLabelList(convert(timespanEnrichmentEntity.getPrefLabel())); - output.setAltLabelList(convert(timespanEnrichmentEntity.getAltLabel())); - output.setBegin(convert(timespanEnrichmentEntity.getBegin()).get(0)); - output.setEnd(convert(timespanEnrichmentEntity.getEnd()).get(0)); + output.setPrefLabelList(convertMultilingualMapToLabel(timespanEnrichmentEntity.getPrefLabel())); + output.setAltLabelList(convertMultilingualMapToLabel(timespanEnrichmentEntity.getAltLabel())); + output.setBegin(convertMultilingualMapToLabel(timespanEnrichmentEntity.getBegin()).get(0)); + output.setEnd(convertMultilingualMapToLabel(timespanEnrichmentEntity.getEnd()).get(0)); output.setHasPartsList(convertResourceOrLiteral(timespanEnrichmentEntity.getDctermsHasPart())); - output.setHiddenLabel(convert(timespanEnrichmentEntity.getHiddenLabel())); - output.setNotes(convert(timespanEnrichmentEntity.getNote())); - output.setSameAs(convertToPartsList(timespanEnrichmentEntity.getOwlSameAs())); + output.setHiddenLabel(convertMultilingualMapToLabel(timespanEnrichmentEntity.getHiddenLabel())); + output.setNotes(convertMultilingualMapToLabel(timespanEnrichmentEntity.getNote())); + output.setSameAs(convertListToPart(timespanEnrichmentEntity.getOwlSameAs())); if (StringUtils.isNotBlank(timespanEnrichmentEntity.getIsPartOf())) { output.setIsPartOf(List.of(new LabelResource(timespanEnrichmentEntity.getIsPartOf()))); @@ -117,11 +118,11 @@ private static Concept convertConcept(ConceptEnrichmentEntity conceptEnrichmentE Concept output = new Concept(); output.setAbout(conceptEnrichmentEntity.getAbout()); - output.setPrefLabelList(convert(conceptEnrichmentEntity.getPrefLabel())); - output.setAltLabelList(convert(conceptEnrichmentEntity.getAltLabel())); - output.setHiddenLabel(convert(conceptEnrichmentEntity.getHiddenLabel())); - output.setNotation(convert(conceptEnrichmentEntity.getNotation())); - output.setNotes(convert(conceptEnrichmentEntity.getNote())); + output.setPrefLabelList(convertMultilingualMapToLabel(conceptEnrichmentEntity.getPrefLabel())); + output.setAltLabelList(convertMultilingualMapToLabel(conceptEnrichmentEntity.getAltLabel())); + output.setHiddenLabel(convertMultilingualMapToLabel(conceptEnrichmentEntity.getHiddenLabel())); + output.setNotation(convertMultilingualMapToLabel(conceptEnrichmentEntity.getNotation())); + output.setNotes(convertMultilingualMapToLabel(conceptEnrichmentEntity.getNote())); output.setBroader(convertToResourceList(conceptEnrichmentEntity.getBroader())); output.setBroadMatch(convertToResourceList(conceptEnrichmentEntity.getBroadMatch())); output.setCloseMatch(convertToResourceList(conceptEnrichmentEntity.getCloseMatch())); @@ -141,12 +142,12 @@ private static Place convertPlace(PlaceEnrichmentEntity placeEnrichmentEntity) { Place output = new Place(); output.setAbout(placeEnrichmentEntity.getAbout()); - output.setPrefLabelList(convert(placeEnrichmentEntity.getPrefLabel())); - output.setAltLabelList(convert(placeEnrichmentEntity.getAltLabel())); + output.setPrefLabelList(convertMultilingualMapToLabel(placeEnrichmentEntity.getPrefLabel())); + output.setAltLabelList(convertMultilingualMapToLabel(placeEnrichmentEntity.getAltLabel())); output.setHasPartsList(convertResourceOrLiteral(placeEnrichmentEntity.getDcTermsHasPart())); - output.setNotes(convert(placeEnrichmentEntity.getNote())); - output.setSameAs(convertToPartsList(placeEnrichmentEntity.getOwlSameAs())); + output.setNotes(convertMultilingualMapToLabel(placeEnrichmentEntity.getNote())); + output.setSameAs(convertListToPart(placeEnrichmentEntity.getOwlSameAs())); if (StringUtils.isNotBlank(placeEnrichmentEntity.getIsPartOf())) { output.setIsPartOf(List.of(new LabelResource(placeEnrichmentEntity.getIsPartOf()))); @@ -169,33 +170,33 @@ private static Agent convertAgent(AgentEnrichmentEntity agentEntityEnrichment) { Agent output = new Agent(); output.setAbout(agentEntityEnrichment.getAbout()); - output.setPrefLabelList(convert(agentEntityEnrichment.getPrefLabel())); - output.setAltLabelList(convert(agentEntityEnrichment.getAltLabel())); - output.setHiddenLabel(convert(agentEntityEnrichment.getHiddenLabel())); - output.setFoafName(convert(agentEntityEnrichment.getFoafName())); - output.setNotes(convert(agentEntityEnrichment.getNote())); + output.setPrefLabelList(convertMultilingualMapToLabel(agentEntityEnrichment.getPrefLabel())); + output.setAltLabelList(convertMultilingualMapToLabel(agentEntityEnrichment.getAltLabel())); + output.setHiddenLabel(convertMultilingualMapToLabel(agentEntityEnrichment.getHiddenLabel())); + output.setFoafName(convertMultilingualMapToLabel(agentEntityEnrichment.getFoafName())); + output.setNotes(convertMultilingualMapToLabel(agentEntityEnrichment.getNote())); - output.setBeginList(convert(agentEntityEnrichment.getBegin())); - output.setEndList(convert(agentEntityEnrichment.getEnd())); + output.setBeginList(convertMultilingualMapToLabel(agentEntityEnrichment.getBegin())); + output.setEndList(convertMultilingualMapToLabel(agentEntityEnrichment.getEnd())); - output.setIdentifier(convert(agentEntityEnrichment.getDcIdentifier())); + output.setIdentifier(convertMultilingualMapToLabel(agentEntityEnrichment.getDcIdentifier())); output.setHasMet(convertToResourceList(agentEntityEnrichment.getEdmHasMet())); output.setBiographicalInformation( convertResourceOrLiteral(agentEntityEnrichment.getRdaGr2BiographicalInformation())); output.setPlaceOfBirth(convertResourceOrLiteral(agentEntityEnrichment.getRdaGr2PlaceOfBirth())); output.setPlaceOfDeath(convertResourceOrLiteral(agentEntityEnrichment.getRdaGr2PlaceOfDeath())); - output.setDateOfBirth(convert(agentEntityEnrichment.getRdaGr2DateOfBirth())); - output.setDateOfDeath(convert(agentEntityEnrichment.getRdaGr2DateOfDeath())); - output.setDateOfEstablishment(convert(agentEntityEnrichment.getRdaGr2DateOfEstablishment())); - output.setDateOfTermination(convert(agentEntityEnrichment.getRdaGr2DateOfTermination())); - output.setGender(convert(agentEntityEnrichment.getRdaGr2Gender())); + output.setDateOfBirth(convertMultilingualMapToLabel(agentEntityEnrichment.getRdaGr2DateOfBirth())); + output.setDateOfDeath(convertMultilingualMapToLabel(agentEntityEnrichment.getRdaGr2DateOfDeath())); + output.setDateOfEstablishment(convertMultilingualMapToLabel(agentEntityEnrichment.getRdaGr2DateOfEstablishment())); + output.setDateOfTermination(convertMultilingualMapToLabel(agentEntityEnrichment.getRdaGr2DateOfTermination())); + output.setGender(convertMultilingualMapToLabel(agentEntityEnrichment.getRdaGr2Gender())); output.setDate(convertResourceOrLiteral(agentEntityEnrichment.getDcDate())); output.setProfessionOrOccupation( convertResourceOrLiteral(agentEntityEnrichment.getRdaGr2ProfessionOrOccupation())); output.setWasPresentAt(convertToResourceList(agentEntityEnrichment.getEdmWasPresentAt())); - output.setSameAs(convertToPartsList(agentEntityEnrichment.getOwlSameAs())); + output.setSameAs(convertListToPart(agentEntityEnrichment.getOwlSameAs())); return output; } @@ -205,10 +206,10 @@ private static Organization convertOrganization( Organization output = new Organization(); output.setAbout(organizationEnrichmentEntity.getAbout()); - output.setPrefLabelList(convert(organizationEnrichmentEntity.getPrefLabel())); - output.setAltLabelList(convert(organizationEnrichmentEntity.getAltLabel())); - output.setNotes(convert(organizationEnrichmentEntity.getNote())); - output.setSameAs(convertToPartsList(organizationEnrichmentEntity.getOwlSameAs())); + output.setPrefLabelList(convertMultilingualMapToLabel(organizationEnrichmentEntity.getPrefLabel())); + output.setAltLabelList(convertMultilingualMapToLabel(organizationEnrichmentEntity.getAltLabel())); + output.setNotes(convertMultilingualMapToLabel(organizationEnrichmentEntity.getNote())); + output.setSameAs(convertListToPart(organizationEnrichmentEntity.getOwlSameAs())); if (MapUtils.isNotEmpty(organizationEnrichmentEntity.getEdmCountry())) { output.setCountry( organizationEnrichmentEntity.getEdmCountry().entrySet().iterator().next().getValue()); @@ -222,7 +223,7 @@ private static Organization convertOrganization( output.setHomepage(new Resource(organizationEnrichmentEntity.getFoafHomepage())); output.setLogo(new Resource(organizationEnrichmentEntity.getFoafLogo())); output.setDepiction(new Resource(organizationEnrichmentEntity.getFoafDepiction())); - output.setAcronyms(convert(organizationEnrichmentEntity.getEdmAcronym())); + output.setAcronyms(convertMultilingualMapToLabel(organizationEnrichmentEntity.getEdmAcronym())); output.setDescriptions(convertMapToLabels(organizationEnrichmentEntity.getDcDescription())); final Address address = organizationEnrichmentEntity.getAddress(); @@ -240,18 +241,6 @@ private static Organization convertOrganization( return output; } - static EnrichmentTerm organizationImplToEnrichmentTerm( - OrganizationEnrichmentEntity organizationEnrichmentEntity, Date created, Date updated) { - final EnrichmentTerm enrichmentTerm = new EnrichmentTerm(); - enrichmentTerm.setEnrichmentEntity(organizationEnrichmentEntity); - enrichmentTerm.setEntityType(EntityType.ORGANIZATION); - enrichmentTerm.setCreated(Objects.requireNonNullElseGet(created, Date::new)); - enrichmentTerm.setUpdated(updated); - enrichmentTerm.setLabelInfos(createLabelInfoList(organizationEnrichmentEntity)); - - return enrichmentTerm; - } - /** * Generates the list of {@link LabelInfo} values. * @param abstractEnrichmentEntity the entity to generate them for @@ -283,52 +272,4 @@ private static void copyToCombinedLabels(Map> combinedLabel }); } } - - private static List