Skip to content

Commit

Permalink
replace with fess-crawler-opensearch
Browse files Browse the repository at this point in the history
  • Loading branch information
@marevol
marevol committed Nov 7, 2024
1 parent 8013500 commit 405751d
Show file tree
Hide file tree
Showing 29 changed files with 137 additions and 136 deletions.
0 fess-crawler-es/.gitignore → fess-crawler-opensearch/.gitignore
View file
File renamed without changes.
4 changes: 2 additions & 2 deletions fess-crawler-es/pom.xml → fess-crawler-opensearch/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<?xml version="1.0"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>fess-crawler-es</artifactId>
<artifactId>fess-crawler-opensearch</artifactId>
<packaging>jar</packaging>
<name>Fess Crawler Elasticsearch</name>
<parent>
Expand Down Expand Up @@ -36,7 +36,7 @@
<configuration>
<archive>
<manifestEntries>
<Automatic-Module-Name>org.codelibs.fess.crawler.es</Automatic-Module-Name>
<Automatic-Module-Name>org.codelibs.fess.crawler.opensearch</Automatic-Module-Name>
</manifestEntries>
</archive>
</configuration>
Expand Down
8 changes: 4 additions & 4 deletions ...libs/fess/crawler/client/FesenClient.java → ...libs/fess/crawler/client/FesenClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.lang.ThreadUtil;
import org.codelibs.fesen.client.HttpClient;
import org.codelibs.fess.crawler.exception.EsAccessException;
import org.codelibs.fess.crawler.exception.OpenSearchAccessException;
import org.opensearch.OpenSearchException;
import org.opensearch.action.ActionRequest;
import org.opensearch.action.ActionType;
Expand Down Expand Up @@ -101,9 +101,9 @@
import jakarta.annotation.PreDestroy;

public class FesenClient implements Client {
public static final String HTTP_ADDRESS = "crawler.es.http_address";
public static final String HTTP_ADDRESS = "crawler.opensearch.http_address";

public static final String TARGET_INDICES = "crawler.es.target_indices";
public static final String TARGET_INDICES = "crawler.opensearch.target_indices";

private static final Logger logger = LoggerFactory.getLogger(FesenClient.class);

Expand Down Expand Up @@ -495,7 +495,7 @@ public int deleteByQuery(final String index, final String type, final QueryBuild
return bulkRequest.execute();
});
if (bulkResponse.hasFailures()) {
throw new EsAccessException(bulkResponse.buildFailureMessage());
throw new OpenSearchAccessException(bulkResponse.buildFailureMessage());
}

final String sid = scrollId;
Expand Down
10 changes: 5 additions & 5 deletions ...s/fess/crawler/entity/EsAccessResult.java → ...rawler/entity/OpenSearchAccessResult.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@
import java.io.IOException;

import org.codelibs.core.beans.util.BeanUtil;
import org.codelibs.fess.crawler.service.impl.EsDataService;
import org.codelibs.fess.crawler.service.impl.OpenSearchDataService;
import org.lastaflute.di.core.SingletonLaContainer;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;

public class EsAccessResult extends AccessResultImpl<String> implements ToXContent {
public class OpenSearchAccessResult extends AccessResultImpl<String> implements ToXContent {

public static final String ID = "id";

Expand Down Expand Up @@ -63,7 +63,7 @@ public void init(final ResponseData responseData, final ResultData resultData) {
BeanUtil.copyBeanToBean(responseData, this);
}

final EsAccessResultData accessResultData = new EsAccessResultData();
final OpenSearchAccessResultData accessResultData = new OpenSearchAccessResultData();
if (resultData != null) {
BeanUtil.copyBeanToBean(resultData, accessResultData);
}
Expand All @@ -73,8 +73,8 @@ public void init(final ResponseData responseData, final ResultData resultData) {
@Override
public AccessResultData<String> getAccessResultData() {
if (!initializedData) {
final EsDataService dataService = SingletonLaContainer.getComponent(EsDataService.class);
final EsAccessResult accessResult = dataService.getAccessResult(getSessionId(), getUrl());
final OpenSearchDataService dataService = SingletonLaContainer.getComponent(OpenSearchDataService.class);
final OpenSearchAccessResult accessResult = dataService.getAccessResult(getSessionId(), getUrl());
if (accessResult != null && accessResult.accessResultData != null) {
setAccessResultData(accessResult.accessResultData);
} else {
Expand Down
10 changes: 5 additions & 5 deletions ...ss/crawler/entity/EsAccessResultData.java → ...er/entity/OpenSearchAccessResultData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,11 @@
import java.util.Map;

import org.codelibs.core.misc.Base64Util;
import org.codelibs.fess.crawler.exception.EsAccessException;
import org.codelibs.fess.crawler.exception.OpenSearchAccessException;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;

public class EsAccessResultData extends AccessResultDataImpl<String> implements ToXContent {
public class OpenSearchAccessResultData extends AccessResultDataImpl<String> implements ToXContent {

public static final String ID = "id";

Expand All @@ -33,18 +33,18 @@ public class EsAccessResultData extends AccessResultDataImpl<String> implements

public static final String ENCODING = "encoding";

public EsAccessResultData() {
public OpenSearchAccessResultData() {
}

public EsAccessResultData(final Map<String, Object> src) {
public OpenSearchAccessResultData(final Map<String, Object> src) {
setTransformerName((String) src.get(TRANSFORMER_NAME));
setEncoding((String) src.get(ENCODING));
final String dataStr = (String) src.get(DATA);
if (dataStr != null) {
try {
setData(Base64Util.decode(dataStr));
} catch (final Exception e) {
throw new EsAccessException("data: " + dataStr, e);
throw new OpenSearchAccessException("data: " + dataStr, e);
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion ...libs/fess/crawler/entity/EsUrlFilter.java → ...s/crawler/entity/OpenSearchUrlFilter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;

public class EsUrlFilter implements ToXContent {
public class OpenSearchUrlFilter implements ToXContent {

public static final String SESSION_ID = "sessionId";

Expand Down
2 changes: 1 addition & 1 deletion ...elibs/fess/crawler/entity/EsUrlQueue.java → ...ss/crawler/entity/OpenSearchUrlQueue.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;

public class EsUrlQueue extends UrlQueueImpl<String> implements ToXContent {
public class OpenSearchUrlQueue extends UrlQueueImpl<String> implements ToXContent {

public static final String ID = "id";

Expand Down
6 changes: 3 additions & 3 deletions .../crawler/exception/EsAccessException.java → .../exception/OpenSearchAccessException.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,15 @@
*/
package org.codelibs.fess.crawler.exception;

public class EsAccessException extends CrawlerSystemException {
public class OpenSearchAccessException extends CrawlerSystemException {

private static final long serialVersionUID = 1L;

public EsAccessException(final String message) {
public OpenSearchAccessException(final String message) {
super(message);
}

public EsAccessException(final String message, final Throwable cause) {
public OpenSearchAccessException(final String message, final Throwable cause) {
super(message, cause);
}

Expand Down
44 changes: 22 additions & 22 deletions .../service/impl/AbstractCrawlerService.java → .../service/impl/AbstractCrawlerService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,11 @@
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.security.MessageDigestUtil;
import org.codelibs.fess.crawler.client.FesenClient;
import org.codelibs.fess.crawler.entity.EsAccessResult;
import org.codelibs.fess.crawler.entity.EsAccessResultData;
import org.codelibs.fess.crawler.entity.OpenSearchAccessResult;
import org.codelibs.fess.crawler.entity.OpenSearchAccessResultData;
import org.codelibs.fess.crawler.exception.CrawlingAccessException;
import org.codelibs.fess.crawler.exception.EsAccessException;
import org.codelibs.fess.crawler.util.EsResultList;
import org.codelibs.fess.crawler.exception.OpenSearchAccessException;
import org.codelibs.fess.crawler.util.OpenSearchResultList;
import org.joda.time.DateTimeZone;
import org.joda.time.format.DateTimeFormatter;
import org.joda.time.format.ISODateTimeFormat;
Expand Down Expand Up @@ -208,22 +208,22 @@ protected XContentBuilder getXContentBuilder(final Object target) {
builder.flush();
return builder;
} catch (final IOException e) {
throw new EsAccessException("Failed to convert " + target + " to JSON.", e);
throw new OpenSearchAccessException("Failed to convert " + target + " to JSON.", e);
}
}

protected RefreshResponse refresh() {
try {
return getClient().get(c -> c.admin().indices().prepareRefresh(index).execute());
} catch (final Exception e) {
throw new EsAccessException("Failed to refresh.", e);
throw new OpenSearchAccessException("Failed to refresh.", e);
}
}

protected IndexResponse insert(final Object target, final OpType opType) {
final String url = getUrl(target);
if (url == null) {
throw new EsAccessException("url is null.");
throw new OpenSearchAccessException("url is null.");
}
final String id = getId(getSessionId(target), url);
try (final XContentBuilder source = getXContentBuilder(target)) {
Expand All @@ -235,9 +235,9 @@ protected IndexResponse insert(final Object target, final OpType opType) {
if (e.status() == RestStatus.CONFLICT) {
throw new CrawlingAccessException("[" + e.status() + "] Failed to insert " + id, e);
}
throw new EsAccessException("[" + e.status() + "] Failed to insert " + id, e);
throw new OpenSearchAccessException("[" + e.status() + "] Failed to insert " + id, e);
} catch (final Exception e) {
throw new EsAccessException("Failed to insert " + id, e);
throw new OpenSearchAccessException("Failed to insert " + id, e);
}
}

Expand Down Expand Up @@ -271,7 +271,7 @@ protected <T> void insertAll(final List<T> list, final OpType opType, final bool
}
}
if (failureBuf.length() > 0) {
throw new EsAccessException(failureBuf.toString());
throw new OpenSearchAccessException(failureBuf.toString());
}
}

Expand Down Expand Up @@ -309,7 +309,7 @@ protected <T> BulkResponse doInsertAll(final List<T> list, final OpType opType)
return bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE).execute();
});
} catch (final Exception e) {
throw new EsAccessException("Failed to insert " + list, e);
throw new OpenSearchAccessException("Failed to insert " + list, e);
}
}

Expand All @@ -319,7 +319,7 @@ protected boolean exists(final String sessionId, final String url) {
final GetResponse response = getClient().get(c -> c.prepareGet(index, id).execute());
return response.isExists();
} catch (final Exception e) {
throw new EsAccessException("Failed to check if " + sessionId + ":" + url + " exists.", e);
throw new OpenSearchAccessException("Failed to check if " + sessionId + ":" + url + " exists.", e);
}
}

Expand All @@ -339,12 +339,12 @@ protected <T> T get(final Class<T> clazz, final String sessionId, final String u
final Map<String, Object> source = response.getSource();
final T bean = BeanUtil.copyMapToNewBean(source, clazz, option -> {
option.converter(new EsTimestampConverter(), timestampFields).excludeWhitespace();
option.exclude(EsAccessResult.ACCESS_RESULT_DATA);
option.exclude(OpenSearchAccessResult.ACCESS_RESULT_DATA);
});
@SuppressWarnings("unchecked")
final Map<String, Object> data = (Map<String, Object>) source.get(EsAccessResult.ACCESS_RESULT_DATA);
final Map<String, Object> data = (Map<String, Object>) source.get(OpenSearchAccessResult.ACCESS_RESULT_DATA);
if (data != null) {
((EsAccessResult) bean).setAccessResultData(new EsAccessResultData(data));
((OpenSearchAccessResult) bean).setAccessResultData(new OpenSearchAccessResultData(data));
}
setId(bean, id);
return bean;
Expand Down Expand Up @@ -390,7 +390,7 @@ protected <T> List<T> getList(final Class<T> clazz, final Consumer<SearchRequest
callback.accept(builder);
return builder.execute();
});
final EsResultList<T> targetList = new EsResultList<>();
final OpenSearchResultList<T> targetList = new OpenSearchResultList<>();
final SearchHits hits = response.getHits();
final TotalHits totalHits = hits.getTotalHits();
final long totalHitsValue = totalHits != null ? totalHits.value : 0;
Expand All @@ -402,18 +402,18 @@ protected <T> List<T> getList(final Class<T> clazz, final Consumer<SearchRequest
final Map<String, Object> source = searchHit.getSourceAsMap();
final T target = BeanUtil.copyMapToNewBean(source, clazz, option -> {
option.converter(new EsTimestampConverter(), timestampFields).excludeWhitespace();
option.exclude(EsAccessResult.ACCESS_RESULT_DATA);
option.exclude(OpenSearchAccessResult.ACCESS_RESULT_DATA);
});
@SuppressWarnings("unchecked")
final Map<String, Object> data = (Map<String, Object>) source.get(EsAccessResult.ACCESS_RESULT_DATA);
final Map<String, Object> data = (Map<String, Object>) source.get(OpenSearchAccessResult.ACCESS_RESULT_DATA);
if (data != null) {
((EsAccessResult) target).setAccessResultData(new EsAccessResultData(data));
((OpenSearchAccessResult) target).setAccessResultData(new OpenSearchAccessResultData(data));
}
setId(target, searchHit.getId());
targetList.add(target);
}
} catch (final Exception e) {
throw new EsAccessException("response: " + response, e);
throw new OpenSearchAccessException("response: " + response, e);
}
}
return targetList;
Expand All @@ -426,7 +426,7 @@ protected boolean delete(final String sessionId, final String url) {
getClient().get(c -> c.prepareDelete().setIndex(index).setId(id).setRefreshPolicy(RefreshPolicy.IMMEDIATE).execute());
return response.getResult() == Result.DELETED;
} catch (final Exception e) {
throw new EsAccessException("Failed to delete " + sessionId + ":" + url, e);
throw new OpenSearchAccessException("Failed to delete " + sessionId + ":" + url, e);
}
}

Expand Down Expand Up @@ -461,7 +461,7 @@ public void delete(final Consumer<SearchRequestBuilder> callback) {
return bulkBuilder.execute();
});
if (bulkResponse.hasFailures()) {
throw new EsAccessException(bulkResponse.buildFailureMessage());
throw new OpenSearchAccessException(bulkResponse.buildFailureMessage());
}

final String sid = scrollId;
Expand Down
Loading

0 comments on commit 405751d

Please sign in to comment.