Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

use MultiThreadedHttpConnectionManager when instantiating clients and #385

Merged
merged 1 commit into from
Jul 19, 2024
Merged

use MultiThreadedHttpConnectionManager when instantiating clients and #385

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions annotation-client/src/main/java/eu/europeana/annotation/client/http/HttpConnection.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@
package eu.europeana.annotation.client.http;

import java.io.IOException;

import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.methods.DeleteMethod;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
Expand Down Expand Up @@ -362,7 +362,7 @@ public String getURLContentWithBody(String url, String jsonParamValue,

private HttpClient getHttpClient(int connectionRetry, int conectionTimeout) {
if (this.httpClient == null) {
HttpClient client = new HttpClient();
HttpClient client = new HttpClient(new MultiThreadedHttpConnectionManager());

//configure retry handler
client.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
Expand Down
153 changes: 81 additions & 72 deletions ...ntegration-test/java/eu/europeana/annotation/tests/eval/TranscriptionsValidationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,83 +21,92 @@
@SpringBootTest
@AutoConfigureMockMvc
public class TranscriptionsValidationTest {

Logger log = LogManager.getLogger(getClass());

HttpClient httpClient = new HttpClient();
File outputFile = new File("/tmp/transcriptionseval.csv");

//commented out until the input files are provided
//@Test
public void validateTranscriptionsTest() throws Exception{

//read file
File transcriptionsFile = new File(getClass().getResource("/transcriptions.csv").getFile());
outputFile.getParentFile().mkdirs();
List<String> entries = FileUtils.readLines(transcriptionsFile, "UTF8");
String[] values;
String csvLine;
int count = 0;
writeToOutputFile("anno_uri,target_uri.source, sourceHttpCode, target_uri.scope, scopeHttpCode");
for (String entry : entries) {
values = entry.split(",");
if(count == 0){
//ignore header
count++;
continue;
}
csvLine = validateValues(values);
writeToOutputFile(csvLine);
count++;
if(count % 100 == 0) {
log.debug("Processed items: " + count);
}
}
Logger log = LogManager.getLogger(getClass());

HttpClient httpClient = new HttpClient();
File outputFile = new File("/tmp/transcriptionseval.csv");

// commented out until the input files are provided
// @Test
public void validateTranscriptionsTest() throws Exception {

// read file
File transcriptionsFile = new File(getClass().getResource("/transcriptions.csv").getFile());
outputFile.getParentFile().mkdirs();
List<String> entries = FileUtils.readLines(transcriptionsFile, "UTF8");
String[] values;
String csvLine;
int count = 0;
writeToOutputFile(
"anno_uri,target_uri.source, sourceHttpCode, target_uri.scope, scopeHttpCode");
for (String entry : entries) {
values = entry.split(",");
if (count == 0) {
// ignore header
count++;
continue;
}
csvLine = validateValues(values);
writeToOutputFile(csvLine);
count++;
if (count % 100 == 0) {
log.debug("Processed items: " + count);
}
}
}

private void writeToOutputFile(String csvLine) throws IOException {
// TODO Auto-generated method stub
FileUtils.writeStringToFile(outputFile, csvLine+"\n", StandardCharsets.UTF_8, true);
private void writeToOutputFile(String csvLine) throws IOException {
// TODO Auto-generated method stub
FileUtils.writeStringToFile(outputFile, csvLine + "\n", StandardCharsets.UTF_8, true);
}

private String validateValues(String[] values) throws Exception {
String annoId = values[0];
String scope = "";// europeana item
String source = "";// image
int sourceHttpCode = -1;
int scopeHttpCode = -1;
if (values.length > 2) {
source = values[1];
scope = values[2];
// scope = scope.replace("http:", "https:");
} else if (values.length == 2) {
scope = values[1];
// scope = scope.replace("http:", "https:");
}

private String validateValues(String[] values) throws Exception {
String annoId = values[0];
String scope="";//europeana item
String source="";//image
int sourceHttpCode=-1;
int scopeHttpCode=-1;
if(values.length > 2) {
source = values[1];
scope = values[2];
// scope = scope.replace("http:", "https:");
}else if(values.length == 2){
scope = values[1];
// scope = scope.replace("http:", "https:");
}

assertTrue(scope.startsWith(AnnotationTestsConfiguration.getInstance().getPropAnnotationItemDataEndpoint()) || scope.startsWith("https://www.europeana.eu/"));
assertFalse(source.startsWith(AnnotationTestsConfiguration.getInstance().getPropAnnotationItemDataEndpoint()));

sourceHttpCode = getHttpCode(source, false);
scopeHttpCode = getHttpCode(scope, true);

//return CSV line
return annoId + "," + source +"," + sourceHttpCode+ "," + scope + "," + scopeHttpCode;
assertTrue(scope
.startsWith(AnnotationTestsConfiguration.getInstance().getPropAnnotationItemDataEndpoint())
|| scope.startsWith("https://www.europeana.eu/"));
assertFalse(source.startsWith(
AnnotationTestsConfiguration.getInstance().getPropAnnotationItemDataEndpoint()));

sourceHttpCode = getHttpCode(source, false);
scopeHttpCode = getHttpCode(scope, true);

// return CSV line
return annoId + "," + source + "," + sourceHttpCode + "," + scope + "," + scopeHttpCode;
}

private int getHttpCode(String url, boolean addAccept) throws HttpException, IOException {
if (StringUtils.isEmpty(url)) {
return 404;
}
HeadMethod method = new HeadMethod(url);
method.setFollowRedirects(true);
if (addAccept) {
method.addRequestHeader(new Header("Accept", "application/json"));
}
int code = -1;
try {
code = httpClient.executeMethod(method);
} finally {
method.releaseConnection();
}

private int getHttpCode(String url, boolean addAccept) throws HttpException, IOException {
if(StringUtils.isEmpty(url)) {
return 404;
}
HeadMethod method = new HeadMethod(url);
method.setFollowRedirects(true);
if(addAccept) {
method.addRequestHeader(new Header("Accept", "application/json"));
}
int code = httpClient.executeMethod(method);

// String redirection = method.getResponseHeader("Location");
// log.debug("redirection: " + redirection);
return code;
}
// String redirection = method.getResponseHeader("Location");
// log.debug("redirection: " + redirection);
return code;
}
}
5 changes: 5 additions & 0 deletions ...s/src/integration-test/java/eu/europeana/annotation/tests/utils/EuropeanaOauthClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,12 @@
import org.apache.commons.lang3.StringUtils;
import org.codehaus.jettison.json.JSONObject;
import org.springframework.http.HttpHeaders;
import eu.europeana.annotation.utils.HttpConnection;

@Deprecated
/**
* @deprecated not used anymore, use {@link HttpConnection} instead
*/
public class EuropeanaOauthClient {

public EuropeanaOauthClient() {
Expand Down
72 changes: 44 additions & 28 deletions annotation-utils/src/main/java/eu/europeana/annotation/utils/HttpConnection.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@

import java.io.IOException;
import java.io.InputStream;

import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.StringRequestEntity;
Expand All @@ -28,7 +28,7 @@ public class HttpConnection {
private static final int STATUS_OK_START = 200;
private static final int STATUS_OK_END = 299;
private HttpClient httpClient;
private int connectionRetries;
private int connectionRetries;
private int connectionTimeout;
Logger logger = LogManager.getLogger(getClass().getName());

Expand Down Expand Up @@ -56,7 +56,8 @@ public HttpConnection() {
}

/**
* retrieve the response as String
* retrieve the response as String
*
* @param url - url of the web resource
* @param headerName - headers to be set (e.g. Accept)
* @param headerValue - value to be set for the indicated header
Expand All @@ -69,20 +70,26 @@ public String getURLContentAsString(String url, String headerName, String header
GetMethod get = new GetMethod(url);
get.setRequestHeader(headerName, headerValue);

client.executeMethod(get);
if (get.getStatusCode() >= STATUS_OK_START && get.getStatusCode() <= STATUS_OK_END) {
return get.getResponseBodyAsString();
} else {
if(logger.isWarnEnabled()) {
logger.warn("Received Status Code: {}, and Response Body: {}, for the url: {}",
get.getStatusCode(), get.getResponseBodyAsString(), url);
}
return null;
try {
client.executeMethod(get);
if (get.getStatusCode() >= STATUS_OK_START && get.getStatusCode() <= STATUS_OK_END) {
return get.getResponseBodyAsString();
} else {
if (logger.isWarnEnabled()) {
logger.warn("Received Status Code: {}, and Response Body: {}, for the url: {}",
get.getStatusCode(), get.getResponseBodyAsString(), url);
}
return null;
}
} finally {
get.releaseConnection();
}
}

/**
* retrieve the response as stream to be used for parsing to specific type. The Accept header is set to application/xml
* retrieve the response as stream to be used for parsing to specific type. The Accept header is
* set to application/xml
*
* @param url - url of the web resource
* @return - the Stream for accessing the content of the body
* @throws IOException - if the access to remote resource fails
Expand All @@ -92,20 +99,25 @@ public InputStream getURLContentAsStream(String url) throws IOException {
GetMethod get = new GetMethod(url);
get.setRequestHeader("Accept", "application/xml");

client.executeMethod(get);
if (get.getStatusCode() >= STATUS_OK_START && get.getStatusCode() <= STATUS_OK_END) {
return get.getResponseBodyAsStream();
} else {
if(logger.isWarnEnabled()) {
logger.warn("Received Status Code: {}, and Response Body: {}, for the url: {}",
get.getStatusCode(), get.getResponseBodyAsString(), url);
try {
client.executeMethod(get);
if (get.getStatusCode() >= STATUS_OK_START && get.getStatusCode() <= STATUS_OK_END) {
return get.getResponseBodyAsStream();
} else {
if (logger.isWarnEnabled()) {
logger.warn("Received Status Code: {}, and Response Body: {}, for the url: {}",
get.getStatusCode(), get.getResponseBodyAsString(), url);
}
return null;
}
} finally {
get.releaseConnection();
}
}

/**
* retrieve the response as stream to be used for parsing to specific type
*
* @param url - url of the web resource
* @param body - the request body
* @return - the Stream for accessing the content of the body
Expand All @@ -119,15 +131,19 @@ public InputStream postRequest(String url, String body) throws IOException {
new StringRequestEntity(body, "application/json;charset=UTF-8", null);
post.setRequestEntity(requestBody);

client.executeMethod(post);
if (post.getStatusCode() >= STATUS_OK_START && post.getStatusCode() <= STATUS_OK_END) {
return post.getResponseBodyAsStream();
} else {
if(logger.isWarnEnabled()) {
logger.warn("Received Status Code: {}, and Response Body: {}, for the url: {}",
post.getStatusCode(), post.getResponseBodyAsString(), url);
try {
client.executeMethod(post);
if (post.getStatusCode() >= STATUS_OK_START && post.getStatusCode() <= STATUS_OK_END) {
return post.getResponseBodyAsStream();
} else {
if (logger.isWarnEnabled()) {
logger.warn("Received Status Code: {}, and Response Body: {}, for the url: {}",
post.getStatusCode(), post.getResponseBodyAsString(), url);
}
return null;
}
} finally {
post.releaseConnection();
}
}

Expand All @@ -140,7 +156,7 @@ public InputStream postRequest(String url, String body) throws IOException {
*/
private HttpClient getHttpClient(int connectionRetry, int conectionTimeout) {
if (this.httpClient == null) {
HttpClient client = new HttpClient();
HttpClient client = new HttpClient(new MultiThreadedHttpConnectionManager());

// configure retry handler
client.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
Expand Down
Loading