From 1cf1427ff64a3f943cfe4d91729bfb301d3b2efc Mon Sep 17 00:00:00 2001 From: Matt Wheeler Date: Tue, 28 Jan 2014 11:21:30 -0800 Subject: [PATCH] SENSEI-302 Modify request API to allow clients to specify which stored fields to fetch --- .../search/client/SenseiServiceProxy.java | 29 +++++++++++++-- .../client/req/SenseiClientRequest.java | 28 +++++++++++---- .../com/senseidb/test/client/Examples.java | 2 +- .../senseidb/test/client/SendRawQuery.java | 2 +- .../cluster/client/BrowseRequestBuilder.java | 2 +- .../senseidb/search/node/ResultMerger.java | 16 +++++++-- .../senseidb/search/node/SenseiBroker.java | 22 ++++++++++-- .../senseidb/search/req/SenseiRequest.java | 35 +++++++++++++------ .../servlet/AbstractSenseiClientServlet.java | 8 ++++- .../servlet/DefaultSenseiJSONServlet.java | 12 ++++--- .../servlet/SenseiSearchServletParams.java | 3 +- .../svc/impl/CoreSenseiServiceImpl.java | 2 +- .../svc/impl/HttpRestSenseiServiceImpl.java | 6 ++-- .../com/senseidb/util/RequestConverter.java | 23 +++++++++++- .../com/senseidb/util/RequestConverter2.java | 9 +++-- .../test/TestHttpRestSenseiServiceImpl.java | 6 ++-- .../senseidb/test/TestRequestConverter2.java | 3 +- .../java/com/senseidb/test/TestSensei.java | 2 +- 18 files changed, 164 insertions(+), 46 deletions(-) diff --git a/clients/java/src/main/java/com/senseidb/search/client/SenseiServiceProxy.java b/clients/java/src/main/java/com/senseidb/search/client/SenseiServiceProxy.java index 91c1ab014..769a05833 100644 --- a/clients/java/src/main/java/com/senseidb/search/client/SenseiServiceProxy.java +++ b/clients/java/src/main/java/com/senseidb/search/client/SenseiServiceProxy.java @@ -72,8 +72,14 @@ public String sendBQLRaw(String bql) { } public Map sendGetRequest(long... uids) throws IOException, JSONException { + return sendGetRequest(true, true, uids); + } + + public Map sendGetRequest(boolean includeValue, boolean includeFields, + long... uids) throws IOException, JSONException { Map ret = new LinkedHashMap(uids.length); - String response = sendPostRaw(getStoreGetUrl(), new JSONArray(uids).toString()); + String response = sendPostRaw(getStoreGetUrl(includeValue, includeFields), + new JSONArray(uids).toString()); if (response == null || response.length() == 0) { return ret; } @@ -95,8 +101,25 @@ public String getSearchUrl() { } public String getStoreGetUrl() { - if (url != null) return url + "/get"; - return "http://" + host + ":" + port + "/sensei/get"; + return getStoreGetUrl(false, false); + } + + public String getStoreGetUrl(boolean includeValue, boolean includeFields) { + String baseUrl; + if (url != null) { + baseUrl = url + "/get"; + } else { + baseUrl = "http://" + host + ":" + port + "/sensei/get"; + } + if (includeValue && includeFields) { + return baseUrl + "?value=true&fields=true"; + } else if(includeValue) { + return baseUrl + "?value=true"; + } else if (includeFields) { + return baseUrl + "?fields=true"; + } else { + return baseUrl; + } } private JSONObject jsonResponse(String output) throws JSONException { diff --git a/clients/java/src/main/java/com/senseidb/search/client/req/SenseiClientRequest.java b/clients/java/src/main/java/com/senseidb/search/client/req/SenseiClientRequest.java index b7bcad6ae..a2ebe9b8a 100644 --- a/clients/java/src/main/java/com/senseidb/search/client/req/SenseiClientRequest.java +++ b/clients/java/src/main/java/com/senseidb/search/client/req/SenseiClientRequest.java @@ -47,9 +47,16 @@ public class SenseiClientRequest { private final List sort = new ArrayList(); private final Map facets = new HashMap(); /** - * Flag indicating whether stored fields are to be fetched + * Flag indicating whether all stored fields are to be fetched + */ + private boolean fetchAllStoredFields; + /** + * Flag indicating whether the stored value is to be fetched + */ + private boolean fetchStoredValue; + /** + * Stored fields to fetch */ - private boolean fetchStored; private List fieldsToFetch; private final List termVectors = new ArrayList(); /** @@ -95,8 +102,13 @@ public Builder paging(int size, int offset) { return this; } - public Builder fetchStored(boolean fetchStored) { - request.fetchStored = fetchStored; + public Builder fetchAllStoredFields(boolean fetchAllStoredFields) { + request.fetchAllStoredFields = fetchAllStoredFields; + return this; + } + + public Builder fetchStoredValue(boolean fetchStoredValue) { + request.fetchStoredValue = fetchStoredValue; return this; } @@ -244,8 +256,12 @@ public Map getFacets() { return facets; } - public boolean isFetchStored() { - return fetchStored; + public boolean isFetchAllStoredFields() { + return fetchAllStoredFields; + } + + public boolean isFetchStoredValue() { + return fetchStoredValue; } public List getTermVectors() { diff --git a/clients/java/src/test/java/com/senseidb/test/client/Examples.java b/clients/java/src/test/java/com/senseidb/test/client/Examples.java index 8f3870c4f..682c61bd1 100644 --- a/clients/java/src/test/java/com/senseidb/test/client/Examples.java +++ b/clients/java/src/test/java/com/senseidb/test/client/Examples.java @@ -45,7 +45,7 @@ public static void main(String[] args) throws Exception { Operator.and)) .addSelection( Selection.terms("color", Arrays.asList("red"), new ArrayList(), Operator.or)) - .paging(10, 0).fetchStored(true).addSort(Sort.desc("price")).build(); + .paging(10, 0).fetchAllStoredFields(true).addSort(Sort.desc("price")).build(); JSONObject serialized = (JSONObject) JsonSerializer.serialize(senseiRequest); System.out.println(serialized.toString(2)); SenseiResult senseiResult = new SenseiServiceProxy("localhost", 8080) diff --git a/clients/java/src/test/java/com/senseidb/test/client/SendRawQuery.java b/clients/java/src/test/java/com/senseidb/test/client/SendRawQuery.java index 3131f4b0b..a1327322e 100644 --- a/clients/java/src/test/java/com/senseidb/test/client/SendRawQuery.java +++ b/clients/java/src/test/java/com/senseidb/test/client/SendRawQuery.java @@ -15,7 +15,7 @@ public static void main(String[] args) throws Exception { SenseiClientRequest senseiRequest = SenseiClientRequest .builder() .paging(10, 0) - .fetchStored(true) + .fetchAllStoredFields(true) .addSelection( Selection.terms("color", Arrays.asList("red", "blue"), Collections. emptyList(), Operator.or)).build(); diff --git a/sensei-core/src/main/java/com/senseidb/cluster/client/BrowseRequestBuilder.java b/sensei-core/src/main/java/com/senseidb/cluster/client/BrowseRequestBuilder.java index ee85dba00..455340e9a 100644 --- a/sensei-core/src/main/java/com/senseidb/cluster/client/BrowseRequestBuilder.java +++ b/sensei-core/src/main/java/com/senseidb/cluster/client/BrowseRequestBuilder.java @@ -87,7 +87,7 @@ public void clear() { _req = new SenseiRequest(); _req.setOffset(0); _req.setCount(5); - _req.setFetchStoredFields(true); + _req.setFetchAllStoredFields(true); } public void clearSelections() { diff --git a/sensei-core/src/main/java/com/senseidb/search/node/ResultMerger.java b/sensei-core/src/main/java/com/senseidb/search/node/ResultMerger.java index 78ac24861..751040c56 100644 --- a/sensei-core/src/main/java/com/senseidb/search/node/ResultMerger.java +++ b/sensei-core/src/main/java/com/senseidb/search/node/ResultMerger.java @@ -16,6 +16,7 @@ import java.util.Set; import org.apache.log4j.Logger; +import org.apache.lucene.document.DocumentStoredFieldVisitor; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.SortField; @@ -72,22 +73,31 @@ SenseiHit getSenseiHit(SenseiRequest req) { } ZoieSegmentReader zoieSegmentReader = (ZoieSegmentReader) innerReader; SenseiHit hit = new SenseiHit(); - if (req.isFetchStoredFields()) { + + if (req.isFetchAllStoredFields()) { try { hit.setStoredFields(reader.document(doc)); } catch (Exception e) { logger.error(e.getMessage(), e); } + } else if (req.getStoredFieldsToFetch() != null && !req.getStoredFieldsToFetch().isEmpty()) { + try { + DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor(req.getStoredFieldsToFetch()); + reader.document(doc, visitor); + hit.setStoredFields(visitor.getDocument()); + } catch(Exception e) { + logger.error(e.getMessage(),e); + } } - if (req.isFetchStoredFields()) { + if (req.isFetchStoredValue()) { try { BytesRef bytesRef = zoieSegmentReader.getStoredValue(doc); if (bytesRef != null) { hit.setStoredValue(bytesRef.bytes); } } catch (Exception e) { - logger.error("Exception in getSenseiHit", e); + logger.error("Exception fetching stored value in getSenseiHit", e); } } diff --git a/sensei-core/src/main/java/com/senseidb/search/node/SenseiBroker.java b/sensei-core/src/main/java/com/senseidb/search/node/SenseiBroker.java index 6d3a100a9..a6b93d4a4 100644 --- a/sensei-core/src/main/java/com/senseidb/search/node/SenseiBroker.java +++ b/sensei-core/src/main/java/com/senseidb/search/node/SenseiBroker.java @@ -1,7 +1,10 @@ package com.senseidb.search.node; import java.net.InetSocketAddress; + +import java.util.ArrayList; import java.util.List; +import java.util.ListIterator; import java.util.Map; import java.util.Set; @@ -52,6 +55,19 @@ public static void recoverSrcData(SenseiResult res, SenseiHit[] hits, boolean is byte[] dataBytes = hit.getStoredValue(); if (dataBytes == null || dataBytes.length == 0) { dataBytes = hit.getFieldBinaryValue(AbstractZoieIndexable.DOCUMENT_STORE_FIELD); + if (dataBytes != null) { + List fields = new ArrayList(hit.getStoredFields()); + ListIterator lIter = fields.listIterator(); + while (lIter.hasNext()) { + SerializableField field = lIter.next(); + if (AbstractZoieIndexable.DOCUMENT_STORE_FIELD.equals(field.name())) { + lIter.remove(); + } + } + hit.setStoredFields(fields); + } + } else { + hit.setStoredValue(null); } if (dataBytes != null && dataBytes.length > 0) { @@ -83,9 +99,11 @@ public static void recoverSrcData(SenseiResult res, SenseiHit[] hits, boolean is public SenseiResult mergeResults(SenseiRequest request, List resultList) { SenseiResult res = ResultMerger.merge(request, resultList, false); - if (request.isFetchStoredFields()) { + boolean fetchStoredFields = request.isFetchAllStoredFields() || + request.getStoredFieldsToFetch() != null && !request.getStoredFieldsToFetch().isEmpty(); + if (fetchStoredFields || request.isFetchStoredValue()) { long start = System.currentTimeMillis(); - recoverSrcData(res, res.getSenseiHits(), request.isFetchStoredFields()); + recoverSrcData(res, res.getSenseiHits(), fetchStoredFields); res.setTime(res.getTime() + (System.currentTimeMillis() - start)); } diff --git a/sensei-core/src/main/java/com/senseidb/search/req/SenseiRequest.java b/sensei-core/src/main/java/com/senseidb/search/req/SenseiRequest.java index 8ee9c4b9d..5cb120f9b 100644 --- a/sensei-core/src/main/java/com/senseidb/search/req/SenseiRequest.java +++ b/sensei-core/src/main/java/com/senseidb/search/req/SenseiRequest.java @@ -40,8 +40,9 @@ public class SenseiRequest implements AbstractSenseiRequest, Cloneable { private int _count; private int _origOffset; private int _origCount; - private boolean _fetchStoredFields; - private boolean _origFetchStoredFields; + private boolean _fetchAllStoredFields; + private boolean _origFetchAllStoredFields; + private boolean _fetchStoredValue; private Map _facetInitParamMap; private Set _partitions; private boolean _showExplanation; @@ -63,7 +64,8 @@ public SenseiRequest() { _selections = new HashMap(); _sortSpecs = new ArrayList(); _facetSpecMap = new HashMap(); - _fetchStoredFields = false; + _fetchAllStoredFields = false; + _fetchStoredValue = false; _partitions = null; _showExplanation = false; _routeParam = null; @@ -204,7 +206,7 @@ public Map getFacetSpecs() { public void saveState() { _origOffset = _offset; _origCount = _count; - _origFetchStoredFields = _fetchStoredFields; + _origFetchAllStoredFields = _fetchAllStoredFields; if (_origFacetSpecMaxCounts == null && _facetSpecMap != null) { _origFacetSpecMaxCounts = new HashMap(); for (Map.Entry entry : _facetSpecMap.entrySet()) { @@ -219,7 +221,7 @@ public void saveState() { public void restoreState() { _offset = _origOffset; _count = _origCount; - _fetchStoredFields = _origFetchStoredFields; + _fetchAllStoredFields = _origFetchAllStoredFields; if (_facetSpecMap != null) { for (Map.Entry entry : _facetSpecMap.entrySet()) { FacetSpec spec = entry.getValue(); @@ -252,12 +254,20 @@ public void clearSort() { _sortSpecs.clear(); } - public boolean isFetchStoredFields() { - return _fetchStoredFields; + public boolean isFetchAllStoredFields() { + return _fetchAllStoredFields; } - public void setFetchStoredFields(boolean fetchStoredFields) { - _fetchStoredFields = fetchStoredFields; + public void setFetchAllStoredFields(boolean fetchAllStoredFields) { + _fetchAllStoredFields = fetchAllStoredFields; + } + + public boolean isFetchStoredValue(){ + return _fetchStoredValue; + } + + public void setFetchStoredValue(boolean fetchStoredValue) { + _fetchStoredValue = fetchStoredValue; } /** @@ -467,7 +477,9 @@ public String toString() { if (_routeParam != null) buf.append("route param: ").append(_routeParam).append('\n'); if (_groupBy != null) buf.append("group by: ").append(_groupBy).append('\n'); buf.append("max per group: ").append(_maxPerGroup).append('\n'); - buf.append("fetch stored fields: ").append(_fetchStoredFields).append('\n'); + buf.append("fetch all stored fields: ").append(_fetchAllStoredFields).append('\n'); + buf.append("stored fields to fetch: ").append(_storedFieldsToFetch).append('\n'); + buf.append("fetch stored value: ").append(_fetchStoredValue).append('\n'); return buf.toString(); } @@ -492,7 +504,8 @@ public SenseiRequest clone() { clone.setQuery(this.getQuery()); clone.setOffset(this.getOffset()); clone.setCount(this.getCount()); - clone.setFetchStoredFields(this.isFetchStoredFields()); + clone.setFetchAllStoredFields(this.isFetchAllStoredFields()); + clone.setFetchStoredValue(this.isFetchStoredValue()); clone.setFacetHandlerInitParamMap(this.getFacetHandlerInitParamMap()); clone.setPartitions(this.getPartitions()); clone.setShowExplanation(this.isShowExplanation()); diff --git a/sensei-core/src/main/java/com/senseidb/servlet/AbstractSenseiClientServlet.java b/sensei-core/src/main/java/com/senseidb/servlet/AbstractSenseiClientServlet.java index ae062879d..f14fb687f 100644 --- a/sensei-core/src/main/java/com/senseidb/servlet/AbstractSenseiClientServlet.java +++ b/sensei-core/src/main/java/com/senseidb/servlet/AbstractSenseiClientServlet.java @@ -453,12 +453,18 @@ private void handleStoreGetRequest(HttpServletRequest req, HttpServletResponse r if (jsonString != null) ids = new FastJSONArray(jsonString); } + boolean includeFields = req.getParameter("fields") == null || + Boolean.TRUE.toString().equalsIgnoreCase(req.getParameter("fields")); + boolean includeValue = req.getParameter("value") == null || + Boolean.TRUE.toString().equalsIgnoreCase(req.getParameter("value")); + query = "get=" + String.valueOf(ids); String[] vals = RequestConverter2.getStrings(ids); if (vals != null && vals.length != 0) { senseiReq = new SenseiRequest(); - senseiReq.setFetchStoredFields(true); + senseiReq.setFetchAllStoredFields(includeFields); + senseiReq.setFetchStoredValue(includeValue); senseiReq.setCount(vals.length); BrowseSelection sel = new BrowseSelection(SenseiFacetHandlerBuilder.UID_FACET_NAME); sel.setValues(vals); diff --git a/sensei-core/src/main/java/com/senseidb/servlet/DefaultSenseiJSONServlet.java b/sensei-core/src/main/java/com/senseidb/servlet/DefaultSenseiJSONServlet.java index 64b215bc6..ae5352812 100644 --- a/sensei-core/src/main/java/com/senseidb/servlet/DefaultSenseiJSONServlet.java +++ b/sensei-core/src/main/java/com/senseidb/servlet/DefaultSenseiJSONServlet.java @@ -17,7 +17,8 @@ import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_FACET_ORDER; import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_FACET_ORDER_HITS; import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_FACET_ORDER_VAL; -import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_FETCH_STORED; +import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_FETCH_ALL_STORED_FIELDS; +import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_FETCH_STORED_VALUE; import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_FETCH_TERMVECTOR; import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_FIELDS_TO_FETCH; import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_GROUP_BY; @@ -322,8 +323,10 @@ public static JSONArray buildJSONHits(SenseiRequest req, SenseiHit[] hits) throw // get fetchStored even if request does not have it because it could be set at the // federated broker level - if (selectSet == null || selectSet.contains(PARAM_RESULT_HIT_SRC_DATA) - || req.isFetchStoredFields() || hit.getSrcData() != null) { + if (selectSet == null || selectSet.contains(PARAM_RESULT_HIT_SRC_DATA) || + req.isFetchAllStoredFields() || + (req.getStoredFieldsToFetch() != null && !req.getStoredFieldsToFetch().isEmpty()) || hit.getSrcData() != null) + { hitObj.put(PARAM_RESULT_HIT_SRC_DATA, hit.getSrcData()); } if (fieldMap != null) { @@ -518,7 +521,8 @@ public static void convertScalarParams(SenseiRequest senseiReq, DataConfiguratio senseiReq.setOffset(params.getInt(PARAM_OFFSET, 0)); senseiReq.setCount(params.getInt(PARAM_COUNT, 10)); senseiReq.setShowExplanation(params.getBoolean(PARAM_SHOW_EXPLAIN, false)); - senseiReq.setFetchStoredFields(params.getBoolean(PARAM_FETCH_STORED, false)); + senseiReq.setFetchAllStoredFields(params.getBoolean(PARAM_FETCH_ALL_STORED_FIELDS, false)); + senseiReq.setFetchStoredValue(params.getBoolean(PARAM_FETCH_STORED_VALUE, false)); String[] fetchTVs = params.getStringArray(PARAM_FETCH_TERMVECTOR); if (fetchTVs != null && fetchTVs.length > 0) { diff --git a/sensei-core/src/main/java/com/senseidb/servlet/SenseiSearchServletParams.java b/sensei-core/src/main/java/com/senseidb/servlet/SenseiSearchServletParams.java index 2fa9cccc8..5b66210a4 100644 --- a/sensei-core/src/main/java/com/senseidb/servlet/SenseiSearchServletParams.java +++ b/sensei-core/src/main/java/com/senseidb/servlet/SenseiSearchServletParams.java @@ -12,7 +12,8 @@ public interface SenseiSearchServletParams { public static final String PARAM_SORT_SCORE_REVERSE = "relrev"; public static final String PARAM_SORT_DOC = "doc"; public static final String PARAM_SORT_DOC_REVERSE = "docrev"; - public static final String PARAM_FETCH_STORED = "fetchstored"; + public static final String PARAM_FETCH_ALL_STORED_FIELDS = "fetchallstoredfields"; + public static final String PARAM_FETCH_STORED_VALUE = "fetchstoredvalue"; public static final String PARAM_FIELDS_TO_FETCH = "fieldstofetch"; public static final String PARAM_FETCH_TERMVECTOR = "fetchtermvector"; public static final String PARAM_SHOW_EXPLAIN = "showexplain"; diff --git a/sensei-core/src/main/java/com/senseidb/svc/impl/CoreSenseiServiceImpl.java b/sensei-core/src/main/java/com/senseidb/svc/impl/CoreSenseiServiceImpl.java index fd98a2f6d..df0440531 100644 --- a/sensei-core/src/main/java/com/senseidb/svc/impl/CoreSenseiServiceImpl.java +++ b/sensei-core/src/main/java/com/senseidb/svc/impl/CoreSenseiServiceImpl.java @@ -78,7 +78,7 @@ private SenseiResult browse(SenseiRequest senseiRequest, MultiBoboBrowser browse } // SortCollector collector = // browser.getSortCollector(req.getSort(),req.getQuery(), offset, count, - // req.isFetchStoredFields(),false); + // req.isFetchAllStoredFields(),false); // Map facetCollectors = new HashMap(); diff --git a/sensei-core/src/main/java/com/senseidb/svc/impl/HttpRestSenseiServiceImpl.java b/sensei-core/src/main/java/com/senseidb/svc/impl/HttpRestSenseiServiceImpl.java index 913c1c049..e5475a325 100644 --- a/sensei-core/src/main/java/com/senseidb/svc/impl/HttpRestSenseiServiceImpl.java +++ b/sensei-core/src/main/java/com/senseidb/svc/impl/HttpRestSenseiServiceImpl.java @@ -322,8 +322,10 @@ public static void convertPartitionParams(List qparams, Set qparams, SenseiRequest req) { - qparams.add(new BasicNameValuePair(SenseiSearchServletParams.PARAM_FETCH_STORED, Boolean - .toString(req.isFetchStoredFields()))); + qparams.add(new BasicNameValuePair(SenseiSearchServletParams.PARAM_FETCH_ALL_STORED_FIELDS, Boolean + .toString(req.isFetchAllStoredFields()))); + qparams.add(new BasicNameValuePair(SenseiSearchServletParams.PARAM_FETCH_STORED_VALUE, Boolean + .toString(req.isFetchStoredValue()))); qparams.add(new BasicNameValuePair(SenseiSearchServletParams.PARAM_SHOW_EXPLAIN, Boolean .toString(req.isShowExplanation()))); qparams.add(new BasicNameValuePair(SenseiSearchServletParams.PARAM_OFFSET, Integer.toString(req diff --git a/sensei-core/src/main/java/com/senseidb/util/RequestConverter.java b/sensei-core/src/main/java/com/senseidb/util/RequestConverter.java index c95975401..3054748a3 100644 --- a/sensei-core/src/main/java/com/senseidb/util/RequestConverter.java +++ b/sensei-core/src/main/java/com/senseidb/util/RequestConverter.java @@ -1,9 +1,11 @@ package com.senseidb.util; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.Map; +import java.util.Set; import org.apache.commons.configuration.BaseConfiguration; import org.apache.commons.configuration.Configuration; import org.apache.log4j.Logger; @@ -15,6 +17,7 @@ import com.senseidb.search.node.SenseiQueryBuilder; import com.senseidb.search.node.SenseiQueryBuilderFactory; import com.senseidb.search.req.SenseiRequest; +import proj.zoie.api.indexing.AbstractZoieIndexable; public class RequestConverter { private static Logger logger = Logger.getLogger(RequestConverter.class); @@ -26,7 +29,20 @@ public static BrowseRequest convert(SenseiRequest req, breq.setOffset(req.getOffset()); breq.setCount(req.getCount()); breq.setSort(req.getSort()); - breq.setFetchStoredFields(req.isFetchStoredFields()); + if (!deferFetchFields(req)) { + breq.setFetchAllFields(req.isFetchAllStoredFields()); + + Set fetchFields = req.getStoredFieldsToFetch(); + // Rewrite fetchStoredFields for zoie store. + if (req.isFetchStoredValue()) { + if (fetchFields == null) { + fetchFields = new HashSet(); + req.setStoredFieldsToFetch(fetchFields); + } + fetchFields.add(AbstractZoieIndexable.DOCUMENT_STORE_FIELD); + } + breq.setFieldsToFetch(fetchFields); + } breq.setShowExplanation(req.isShowExplanation()); breq.setTermVectorsToFetch(req.getTermVectorsToFetch()); breq.setGroupBy(req.getGroupBy()); @@ -65,6 +81,11 @@ public static BrowseRequest convert(SenseiRequest req, return breq; } + public static boolean deferFetchFields(SenseiRequest req) { + // when group by is used, data is loaded after the fact + return req.getGroupBy() != null && req.getGroupBy().length > 0; + } + @SuppressWarnings("unchecked") public static Map parseParamConf(Configuration params, String prefix) { Iterator keys = params.getKeys(prefix); diff --git a/sensei-core/src/main/java/com/senseidb/util/RequestConverter2.java b/sensei-core/src/main/java/com/senseidb/util/RequestConverter2.java index a8bedd78c..40e3c07c3 100644 --- a/sensei-core/src/main/java/com/senseidb/util/RequestConverter2.java +++ b/sensei-core/src/main/java/com/senseidb/util/RequestConverter2.java @@ -87,7 +87,7 @@ public class RequestConverter2 { public static final String SORT_SCORE = "_score"; public static final String SORT_RELEVANCE = "relevance"; - public static final String FETCH_STORED = "fetchStored"; + public static final String FETCH_ALL_STORED_FIELDS = "fetchAllStoredFields"; public static final String FIELDS_TO_FETCH = "fieldsToFetch"; public static final String FETCH_STORED_VALUE = "fetchStoredValue"; @@ -380,8 +380,11 @@ else if (type.equals(RequestConverter2.FACETINIT_TYPE_DOUBLE)) param.putDoublePa // other - boolean fetchStored = json.optBoolean(RequestConverter2.FETCH_STORED); - req.setFetchStoredFields(fetchStored); + boolean fetchStoredValue = json.optBoolean(RequestConverter2.FETCH_STORED_VALUE); + req.setFetchStoredValue(fetchStoredValue); + + boolean fetchStored = json.optBoolean(RequestConverter2.FETCH_ALL_STORED_FIELDS); + req.setFetchAllStoredFields(fetchStored); String[] fieldsToFetch = getStrings(json, RequestConverter2.FIELDS_TO_FETCH); if (fieldsToFetch != null && fieldsToFetch.length > 0) { diff --git a/sensei-core/src/test/java/com/senseidb/test/TestHttpRestSenseiServiceImpl.java b/sensei-core/src/test/java/com/senseidb/test/TestHttpRestSenseiServiceImpl.java index 212d51ca1..c54746afe 100644 --- a/sensei-core/src/test/java/com/senseidb/test/TestHttpRestSenseiServiceImpl.java +++ b/sensei-core/src/test/java/com/senseidb/test/TestHttpRestSenseiServiceImpl.java @@ -44,7 +44,7 @@ public class TestHttpRestSenseiServiceImpl extends TestCase { private static final int EXPECTED_COUNT = 72; private static final int EXPECTED_OFFSET = 227; - private static final boolean EXPECTED_FETCH_STORED_FIELDS = true; + private static final boolean EXPECTED_FETCH_ALL_STORED_FIELDS = true; private static final boolean EXPECTED_SHOW_EXPLANATION = true; public TestHttpRestSenseiServiceImpl(String name) { @@ -195,7 +195,7 @@ public void testConvertScalarValues() throws SenseiException, UnsupportedEncodin aRequest.setCount(EXPECTED_COUNT); aRequest.setOffset(EXPECTED_OFFSET); - aRequest.setFetchStoredFields(EXPECTED_FETCH_STORED_FIELDS); + aRequest.setFetchAllStoredFields(EXPECTED_FETCH_ALL_STORED_FIELDS); aRequest.setShowExplanation(EXPECTED_SHOW_EXPLANATION); SenseiRequest bRequest = new SenseiRequest(); @@ -332,7 +332,7 @@ private SenseiRequest createNonRandomSenseiRequest() throws JSONException { void createScalarValues(SenseiRequest req) { req.setCount(EXPECTED_COUNT); req.setOffset(EXPECTED_OFFSET); - req.setFetchStoredFields(EXPECTED_FETCH_STORED_FIELDS); + req.setFetchAllStoredFields(EXPECTED_FETCH_ALL_STORED_FIELDS); req.setShowExplanation(EXPECTED_SHOW_EXPLANATION); } diff --git a/sensei-core/src/test/java/com/senseidb/test/TestRequestConverter2.java b/sensei-core/src/test/java/com/senseidb/test/TestRequestConverter2.java index 76fe6ff90..a4f05d026 100644 --- a/sensei-core/src/test/java/com/senseidb/test/TestRequestConverter2.java +++ b/sensei-core/src/test/java/com/senseidb/test/TestRequestConverter2.java @@ -133,7 +133,8 @@ public void test() throws Exception { assertTrue("secondary sort by is not correct", req.getSort()[1].equals(SortField.FIELD_SCORE)); // test fetchStored; - assertTrue("fetchStored is not correct", req.isFetchStoredFields() == false); + assertTrue("fetchAllStoredFields is not correct", req.isFetchAllStoredFields() == false); + assertTrue("fetchStoredValue is not correct", req.isFetchStoredValue() == false); // test fetchTermVectors; // assertTrue("fetchTermVectors is not correct", req.getTermVectorsToFetch().size() ==0); diff --git a/sensei-core/src/test/java/com/senseidb/test/TestSensei.java b/sensei-core/src/test/java/com/senseidb/test/TestSensei.java index 526a30bf3..2a9bea12f 100644 --- a/sensei-core/src/test/java/com/senseidb/test/TestSensei.java +++ b/sensei-core/src/test/java/com/senseidb/test/TestSensei.java @@ -1195,7 +1195,7 @@ public static JSONObject search(JSONObject req) throws Exception { } public static JSONObject searchGet(JSONArray req) throws Exception { - return search(new URL(SenseiStarter.SenseiUrl.toString() + "/get"), req.toString()); + return search(new URL(SenseiStarter.SenseiUrl.toString() + "/get?fields=false&value=true"), req.toString()); } public static JSONObject search(URL url, String req) throws Exception {