Skip to content

Commit

Permalink
fix #2834 Added language detection and default value settings to bulk…
Browse files Browse the repository at this point in the history 
… document indexing in Admin API.
  • Loading branch information
@marevol
marevol committed Jul 25, 2024
1 parent 6885cbd commit 09fb29e
Show file tree
Hide file tree
Showing 3 changed files with 71 additions and 12 deletions.
56 changes: 53 additions & 3 deletions src/main/java/org/codelibs/fess/app/web/api/admin/documents/ApiAdminDocumentsAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package org.codelibs.fess.app.web.api.admin.documents;

import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand All @@ -31,6 +32,7 @@
import org.codelibs.fess.es.client.SearchEngineClient;
import org.codelibs.fess.helper.CrawlingConfigHelper;
import org.codelibs.fess.helper.CrawlingInfoHelper;
import org.codelibs.fess.helper.LanguageHelper;
import org.codelibs.fess.thumbnail.ThumbnailManager;
import org.codelibs.fess.util.ComponentUtil;
import org.lastaflute.web.Execute;
Expand Down Expand Up @@ -66,12 +68,60 @@ public class ApiAdminDocumentsAction extends FessApiAdminAction {
throwValidationErrorApi(messages -> messages.addErrorsCrudFailedToCreateCrudTable(GLOBAL, "documents is empty."));
}
final String indexFieldId = fessConfig.getIndexFieldId();
final String indexFieldDocId = fessConfig.getIndexFieldDocId();
final String indexFieldContentLength = fessConfig.getIndexFieldContentLength();
final String indexFieldTitle = fessConfig.getIndexFieldTitle();
final String indexFieldContent = fessConfig.getIndexFieldContent();
final String indexFieldFavoriteCount = fessConfig.getIndexFieldFavoriteCount();
final String indexFieldClickCount = fessConfig.getIndexFieldClickCount();
final String indexFieldBoost = fessConfig.getIndexFieldBoost();
final String indexFieldRole = fessConfig.getIndexFieldRole();
final String indexFieldLastModified = fessConfig.getIndexFieldLastModified();
final String indexFieldTimestamp = fessConfig.getIndexFieldTimestamp();
final String indexFieldLang = fessConfig.getIndexFieldLang();
final List<String> guestRoleList = fessConfig.getSearchGuestRoleList();
final Date now = systemHelper.getCurrentTime();
final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
final LanguageHelper languageHelper = ComponentUtil.getLanguageHelper();
final List<Map<String, Object>> docList = body.documents.stream().map(doc -> {
if (!doc.containsKey(indexFieldContentLength)) {
long contentLength = 0;
if (doc.get(indexFieldTitle) instanceof final String title) {
contentLength += title.length();
}
if (doc.get(indexFieldContent) instanceof final String content) {
contentLength += content.length();
}
doc.put(indexFieldContentLength, contentLength);
}
if (!doc.containsKey(indexFieldFavoriteCount)) {
doc.put(indexFieldFavoriteCount, 0L);
}
if (!doc.containsKey(indexFieldClickCount)) {
doc.put(indexFieldClickCount, 0L);
}
if (!doc.containsKey(indexFieldBoost)) {
doc.put(indexFieldBoost, 1.0f);
}
if (!doc.containsKey(indexFieldRole)) {
doc.put(indexFieldRole, guestRoleList);
}
if (!doc.containsKey(indexFieldLastModified)) {
doc.put(indexFieldLastModified, now);
}
if (!doc.containsKey(indexFieldTimestamp)) {
doc.put(indexFieldTimestamp, now);
}
AdminSearchlistAction.validateFields(doc, this::throwValidationErrorApi);
final Map<String, Object> newDoc = fessConfig.convertToStorableDoc(doc);
final String newId = crawlingInfoHelper.generateId(newDoc);
newDoc.put(indexFieldId, newId);
newDoc.put(indexFieldId, crawlingInfoHelper.generateId(newDoc));
newDoc.put(indexFieldDocId, systemHelper.generateDocId(newDoc));
if (newDoc.get(indexFieldLang) instanceof final List<?> langList) {
if (langList.contains("auto")) {
newDoc.remove(indexFieldLang);
}
languageHelper.updateDocument(newDoc);
}
return newDoc;
}).toList();
if (fessConfig.isThumbnailCrawlerEnabled()) {
Expand All @@ -96,7 +146,7 @@ public class ApiAdminDocumentsAction extends FessApiAdminAction {
});
return asJson(new ApiBulkResponse().items(Arrays.stream(response.getItems()).map(item -> {
final Map<String, Object> itemMap = new HashMap<>();
itemMap.put("status", item.status().name());
itemMap.put("result", item.status().name());
if (item.isFailed()) {
itemMap.put("message", item.getFailureMessage());
} else {
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/codelibs/fess/helper/RoleQueryHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ public Set<String> build(final SearchRequestType searchRequestType) {
throw new InvalidAccessTokenException("invalid_token", "Access token is requried.");
}
if (!hasAccessToken || roleSet.isEmpty()) {
roleSet.addAll(fessConfig.getSearchGuestPermissionList());
roleSet.addAll(fessConfig.getSearchGuestRoleList());
}
});
} catch (final RuntimeException e) {
Expand Down
25 changes: 17 additions & 8 deletions src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ public interface FessProp {

String AUTHENTICATION_ADMIN_ROLES = "authenticationAdminRoles";

String SEARCH_GUEST_PERMISSION_LIST = "searchGuestPermissionList";
String SEARCH_GUEST_ROLE_LIST = "searchGuestPermissionList";

String SUGGEST_SEARCH_LOG_PERMISSIONS = "suggestSearchLogPermissions";

Expand Down Expand Up @@ -1276,15 +1276,15 @@ default boolean isValidSearchLogPermissions(final String[] permissions) {

String getRoleSearchGuestPermissions();

default List<String> getSearchGuestPermissionList() {
default List<String> getSearchGuestRoleList() {
@SuppressWarnings("unchecked")
List<String> list = (List<String>) propMap.get(SEARCH_GUEST_PERMISSION_LIST);
List<String> list = (List<String>) propMap.get(SEARCH_GUEST_ROLE_LIST);
if (list == null) {
final PermissionHelper permissionHelper = ComponentUtil.getPermissionHelper();
list = split(getRoleSearchGuestPermissions(), ",")
.get(stream -> stream.map(s -> permissionHelper.encode(s)).filter(StringUtil::isNotBlank).collect(Collectors.toList()));
list.add(getRoleSearchUserPrefix() + Constants.GUEST_USER);
propMap.put(SEARCH_GUEST_PERMISSION_LIST, list);
propMap.put(SEARCH_GUEST_ROLE_LIST, list);
}
return list;
}
Expand Down Expand Up @@ -1331,9 +1331,14 @@ default boolean validateIndexDateFields(final Map<String, Object> source) {
}

default List<String> invalidIndexDateFields(final Map<String, Object> source) {
return split(getIndexAdminDateFields(), ",")
.get(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).filter(s -> isNonEmptyValue(source.get(s)))
.filter(s -> !validateDateTimeString(source.get(s))).collect(Collectors.toList()));
return split(getIndexAdminDateFields(), ",").get(
stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).filter(s -> isNonEmptyValue(source.get(s))).filter(s -> {
final Object obj = source.get(s);
if (obj instanceof Date) {
return false;
}
return !validateDateTimeString(source.get(s));
}).collect(Collectors.toList()));
}

default boolean validateDateTimeString(final Object obj) {
Expand Down Expand Up @@ -1516,7 +1521,11 @@ default Map<String, Object> convertToStorableDoc(final Map<String, Object> sourc
}
} else if (dateFieldSet.contains(key)) {
// TODO time zone
value = FessFunctions.parseDate(value.toString());
if (value instanceof Date) {
// nothing
} else {
value = FessFunctions.parseDate(value.toString());
}
} else if (integerFieldSet.contains(key)) {
if (value instanceof Number num) {
value = num.intValue();
Expand Down

0 comments on commit 09fb29e

Please sign in to comment.