SENSEI-302 Modify request API to allow clients to specify which stored f...

clients/java/src/main/java/com/senseidb/search/client/SenseiServiceProxy.java

@@ -72,8 +72,14 @@ public String sendBQLRaw(String bql) {
   }
 
   public Map<Long, JSONObject> sendGetRequest(long... uids) throws IOException, JSONException {
+    return sendGetRequest(true, true, uids);
+  }
+
+  public Map<Long, JSONObject> sendGetRequest(boolean includeValue, boolean includeFields,
+                                              long... uids) throws IOException, JSONException {
     Map<Long, JSONObject> ret = new LinkedHashMap<Long, JSONObject>(uids.length);
-    String response = sendPostRaw(getStoreGetUrl(), new JSONArray(uids).toString());
+    String response = sendPostRaw(getStoreGetUrl(includeValue, includeFields),
+      new JSONArray(uids).toString());
     if (response == null || response.length() == 0) {
       return ret;
     }
@@ -95,8 +101,25 @@ public String getSearchUrl() {
   }
 
   public String getStoreGetUrl() {
-    if (url != null) return url + "/get";
-    return "http://" + host + ":" + port + "/sensei/get";
+    return getStoreGetUrl(false, false);
+  }
+
+  public String getStoreGetUrl(boolean includeValue, boolean includeFields) {
+    String baseUrl;
+    if (url != null) {
+      baseUrl = url + "/get";
+    } else {
+      baseUrl = "http://" + host + ":" + port + "/sensei/get";
+    }
+    if (includeValue && includeFields) {
+      return baseUrl + "?value=true&fields=true";
+    } else if(includeValue) {
+      return baseUrl + "?value=true";
+    } else if (includeFields) {
+      return baseUrl + "?fields=true";
+    } else {
+      return baseUrl;
+    }
   }
 
   private JSONObject jsonResponse(String output) throws JSONException {

clients/java/src/main/java/com/senseidb/search/client/req/SenseiClientRequest.java

@@ -47,9 +47,16 @@ public class SenseiClientRequest {
   private final List<Object> sort = new ArrayList<Object>();
   private final Map<String, Facet> facets = new HashMap<String, Facet>();
   /**
-   * Flag indicating whether stored fields are to be fetched
+   * Flag indicating whether all stored fields are to be fetched
+   */
+  private boolean fetchAllStoredFields;
+  /**
+   * Flag indicating whether the stored value is to be fetched
+   */
+  private boolean fetchStoredValue;
+  /**
+   * Stored fields to fetch
    */
-  private boolean fetchStored;
   private List<String> fieldsToFetch;
   private final List<String> termVectors = new ArrayList<String>();
   /**
@@ -95,8 +102,13 @@ public Builder paging(int size, int offset) {
       return this;
     }
 
-    public Builder fetchStored(boolean fetchStored) {
-      request.fetchStored = fetchStored;
+    public Builder fetchAllStoredFields(boolean fetchAllStoredFields) {
+      request.fetchAllStoredFields = fetchAllStoredFields;
+      return this;
+    }
+
+    public Builder fetchStoredValue(boolean fetchStoredValue) {
+      request.fetchStoredValue = fetchStoredValue;
       return this;
     }
 
@@ -244,8 +256,12 @@ public Map<String, Facet> getFacets() {
     return facets;
   }
 
-  public boolean isFetchStored() {
-    return fetchStored;
+  public boolean isFetchAllStoredFields() {
+    return fetchAllStoredFields;
+  }
+
+  public boolean isFetchStoredValue() {
+    return fetchStoredValue;
   }
 
   public List<String> getTermVectors() {

clients/java/src/test/java/com/senseidb/test/client/Examples.java

@@ -45,7 +45,7 @@ public static void main(String[] args) throws Exception {
             Operator.and))
         .addSelection(
           Selection.terms("color", Arrays.asList("red"), new ArrayList<String>(), Operator.or))
-        .paging(10, 0).fetchStored(true).addSort(Sort.desc("price")).build();
+        .paging(10, 0).fetchAllStoredFields(true).addSort(Sort.desc("price")).build();
     JSONObject serialized = (JSONObject) JsonSerializer.serialize(senseiRequest);
     System.out.println(serialized.toString(2));
     SenseiResult senseiResult = new SenseiServiceProxy("localhost", 8080)

clients/java/src/test/java/com/senseidb/test/client/SendRawQuery.java

@@ -15,7 +15,7 @@ public static void main(String[] args) throws Exception {
     SenseiClientRequest senseiRequest = SenseiClientRequest
         .builder()
         .paging(10, 0)
-        .fetchStored(true)
+        .fetchAllStoredFields(true)
         .addSelection(
           Selection.terms("color", Arrays.asList("red", "blue"), Collections.<String> emptyList(),
             Operator.or)).build();

sensei-core/src/main/java/com/senseidb/cluster/client/BrowseRequestBuilder.java

@@ -87,7 +87,7 @@ public void clear() {
     _req = new SenseiRequest();
     _req.setOffset(0);
     _req.setCount(5);
-    _req.setFetchStoredFields(true);
+    _req.setFetchAllStoredFields(true);
   }
 
   public void clearSelections() {

sensei-core/src/main/java/com/senseidb/search/node/ResultMerger.java

@@ -16,6 +16,7 @@
 import java.util.Set;
 
 import org.apache.log4j.Logger;
+import org.apache.lucene.document.DocumentStoredFieldVisitor;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.SortField;
@@ -72,22 +73,31 @@ SenseiHit getSenseiHit(SenseiRequest req) {
       }
       ZoieSegmentReader<?> zoieSegmentReader = (ZoieSegmentReader<?>) innerReader;
       SenseiHit hit = new SenseiHit();
-      if (req.isFetchStoredFields()) {
+
+      if (req.isFetchAllStoredFields()) {
         try {
           hit.setStoredFields(reader.document(doc));
         } catch (Exception e) {
           logger.error(e.getMessage(), e);
         }
+      } else if (req.getStoredFieldsToFetch() != null && !req.getStoredFieldsToFetch().isEmpty()) {
+        try {
+          DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor(req.getStoredFieldsToFetch());
+          reader.document(doc, visitor);
+          hit.setStoredFields(visitor.getDocument());
+        } catch(Exception e) {
+          logger.error(e.getMessage(),e);
+        }
       }
 
-      if (req.isFetchStoredFields()) {
+      if (req.isFetchStoredValue()) {
         try {
           BytesRef bytesRef = zoieSegmentReader.getStoredValue(doc);
           if (bytesRef != null) {
             hit.setStoredValue(bytesRef.bytes);
           }
         } catch (Exception e) {
-          logger.error("Exception in getSenseiHit", e);
+          logger.error("Exception fetching stored value in getSenseiHit", e);
         }
       }
 

sensei-core/src/main/java/com/senseidb/search/node/SenseiBroker.java

@@ -1,7 +1,10 @@
 package com.senseidb.search.node;
 
 import java.net.InetSocketAddress;
+
+import java.util.ArrayList;
 import java.util.List;
+import java.util.ListIterator;
 import java.util.Map;
 import java.util.Set;
 
@@ -52,6 +55,19 @@ public static void recoverSrcData(SenseiResult res, SenseiHit[] hits, boolean is
           byte[] dataBytes = hit.getStoredValue();
           if (dataBytes == null || dataBytes.length == 0) {
             dataBytes = hit.getFieldBinaryValue(AbstractZoieIndexable.DOCUMENT_STORE_FIELD);
+            if (dataBytes != null) {
+              List<SerializableField> fields = new ArrayList<SerializableField>(hit.getStoredFields());
+              ListIterator<SerializableField> lIter = fields.listIterator();
+              while (lIter.hasNext()) {
+                SerializableField field = lIter.next();
+                if (AbstractZoieIndexable.DOCUMENT_STORE_FIELD.equals(field.name())) {
+                  lIter.remove();
+                }
+              }
+              hit.setStoredFields(fields);
+            }
+          } else {
+            hit.setStoredValue(null);
           }
 
           if (dataBytes != null && dataBytes.length > 0) {
@@ -83,9 +99,11 @@ public static void recoverSrcData(SenseiResult res, SenseiHit[] hits, boolean is
   public SenseiResult mergeResults(SenseiRequest request, List<SenseiResult> resultList) {
     SenseiResult res = ResultMerger.merge(request, resultList, false);
 
-    if (request.isFetchStoredFields()) {
+    boolean fetchStoredFields = request.isFetchAllStoredFields() ||
+      request.getStoredFieldsToFetch() != null && !request.getStoredFieldsToFetch().isEmpty();
+    if (fetchStoredFields || request.isFetchStoredValue()) {
       long start = System.currentTimeMillis();
-      recoverSrcData(res, res.getSenseiHits(), request.isFetchStoredFields());
+      recoverSrcData(res, res.getSenseiHits(), fetchStoredFields);
       res.setTime(res.getTime() + (System.currentTimeMillis() - start));
     }
 

sensei-core/src/main/java/com/senseidb/search/req/SenseiRequest.java

@@ -40,8 +40,9 @@ public class SenseiRequest implements AbstractSenseiRequest, Cloneable {
   private int _count;
   private int _origOffset;
   private int _origCount;
-  private boolean _fetchStoredFields;
-  private boolean _origFetchStoredFields;
+  private boolean _fetchAllStoredFields;
+  private boolean _origFetchAllStoredFields;
+  private boolean _fetchStoredValue;
   private Map<String, FacetHandlerInitializerParam> _facetInitParamMap;
   private Set<Integer> _partitions;
   private boolean _showExplanation;
@@ -63,7 +64,8 @@ public SenseiRequest() {
     _selections = new HashMap<String, BrowseSelection>();
     _sortSpecs = new ArrayList<SerializableSortField>();
     _facetSpecMap = new HashMap<String, FacetSpec>();
-    _fetchStoredFields = false;
+    _fetchAllStoredFields = false;
+    _fetchStoredValue = false;
     _partitions = null;
     _showExplanation = false;
     _routeParam = null;
@@ -204,7 +206,7 @@ public Map<String, FacetSpec> getFacetSpecs() {
   public void saveState() {
     _origOffset = _offset;
     _origCount = _count;
-    _origFetchStoredFields = _fetchStoredFields;
+    _origFetchAllStoredFields = _fetchAllStoredFields;
     if (_origFacetSpecMaxCounts == null && _facetSpecMap != null) {
       _origFacetSpecMaxCounts = new HashMap<String, Integer>();
       for (Map.Entry<String, FacetSpec> entry : _facetSpecMap.entrySet()) {
@@ -219,7 +221,7 @@ public void saveState() {
   public void restoreState() {
     _offset = _origOffset;
     _count = _origCount;
-    _fetchStoredFields = _origFetchStoredFields;
+    _fetchAllStoredFields = _origFetchAllStoredFields;
     if (_facetSpecMap != null) {
       for (Map.Entry<String, FacetSpec> entry : _facetSpecMap.entrySet()) {
         FacetSpec spec = entry.getValue();
@@ -252,12 +254,20 @@ public void clearSort() {
     _sortSpecs.clear();
   }
 
-  public boolean isFetchStoredFields() {
-    return _fetchStoredFields;
+  public boolean isFetchAllStoredFields() {
+    return _fetchAllStoredFields;
   }
 
-  public void setFetchStoredFields(boolean fetchStoredFields) {
-    _fetchStoredFields = fetchStoredFields;
+  public void setFetchAllStoredFields(boolean fetchAllStoredFields) {
+    _fetchAllStoredFields = fetchAllStoredFields;
+  }
+
+  public boolean isFetchStoredValue(){
+    return _fetchStoredValue;
+  }
+
+  public void setFetchStoredValue(boolean fetchStoredValue) {
+    _fetchStoredValue = fetchStoredValue;
   }
 
   /**
@@ -467,7 +477,9 @@ public String toString() {
     if (_routeParam != null) buf.append("route param: ").append(_routeParam).append('\n');
     if (_groupBy != null) buf.append("group by: ").append(_groupBy).append('\n');
     buf.append("max per group: ").append(_maxPerGroup).append('\n');
-    buf.append("fetch stored fields: ").append(_fetchStoredFields).append('\n');
+    buf.append("fetch all stored fields: ").append(_fetchAllStoredFields).append('\n');
+    buf.append("stored fields to fetch: ").append(_storedFieldsToFetch).append('\n');
+    buf.append("fetch stored value: ").append(_fetchStoredValue).append('\n');
     return buf.toString();
   }
 
@@ -492,7 +504,8 @@ public SenseiRequest clone() {
     clone.setQuery(this.getQuery());
     clone.setOffset(this.getOffset());
     clone.setCount(this.getCount());
-    clone.setFetchStoredFields(this.isFetchStoredFields());
+    clone.setFetchAllStoredFields(this.isFetchAllStoredFields());
+    clone.setFetchStoredValue(this.isFetchStoredValue());
     clone.setFacetHandlerInitParamMap(this.getFacetHandlerInitParamMap());
     clone.setPartitions(this.getPartitions());
     clone.setShowExplanation(this.isShowExplanation());

sensei-core/src/main/java/com/senseidb/servlet/AbstractSenseiClientServlet.java

@@ -453,12 +453,18 @@ private void handleStoreGetRequest(HttpServletRequest req, HttpServletResponse r
         if (jsonString != null) ids = new FastJSONArray(jsonString);
       }
 
+      boolean includeFields = req.getParameter("fields") == null ||
+        Boolean.TRUE.toString().equalsIgnoreCase(req.getParameter("fields"));
+      boolean includeValue = req.getParameter("value") == null ||
+        Boolean.TRUE.toString().equalsIgnoreCase(req.getParameter("value"));
+
       query = "get=" + String.valueOf(ids);
 
       String[] vals = RequestConverter2.getStrings(ids);
       if (vals != null && vals.length != 0) {
         senseiReq = new SenseiRequest();
-        senseiReq.setFetchStoredFields(true);
+        senseiReq.setFetchAllStoredFields(includeFields);
+        senseiReq.setFetchStoredValue(includeValue);
         senseiReq.setCount(vals.length);
         BrowseSelection sel = new BrowseSelection(SenseiFacetHandlerBuilder.UID_FACET_NAME);
         sel.setValues(vals);

sensei-core/src/main/java/com/senseidb/servlet/DefaultSenseiJSONServlet.java

@@ -17,7 +17,8 @@
 import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_FACET_ORDER;
 import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_FACET_ORDER_HITS;
 import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_FACET_ORDER_VAL;
-import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_FETCH_STORED;
+import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_FETCH_ALL_STORED_FIELDS;
+import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_FETCH_STORED_VALUE;
 import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_FETCH_TERMVECTOR;
 import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_FIELDS_TO_FETCH;
 import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_GROUP_BY;
@@ -322,8 +323,10 @@ public static JSONArray buildJSONHits(SenseiRequest req, SenseiHit[] hits) throw
 
       // get fetchStored even if request does not have it because it could be set at the
       // federated broker level
-      if (selectSet == null || selectSet.contains(PARAM_RESULT_HIT_SRC_DATA)
-          || req.isFetchStoredFields() || hit.getSrcData() != null) {
+      if (selectSet == null || selectSet.contains(PARAM_RESULT_HIT_SRC_DATA) ||
+        req.isFetchAllStoredFields() ||
+        (req.getStoredFieldsToFetch() != null && !req.getStoredFieldsToFetch().isEmpty()) || hit.getSrcData() != null)
+      {
         hitObj.put(PARAM_RESULT_HIT_SRC_DATA, hit.getSrcData());
       }
       if (fieldMap != null) {
@@ -518,7 +521,8 @@ public static void convertScalarParams(SenseiRequest senseiReq, DataConfiguratio
     senseiReq.setOffset(params.getInt(PARAM_OFFSET, 0));
     senseiReq.setCount(params.getInt(PARAM_COUNT, 10));
     senseiReq.setShowExplanation(params.getBoolean(PARAM_SHOW_EXPLAIN, false));
-    senseiReq.setFetchStoredFields(params.getBoolean(PARAM_FETCH_STORED, false));
+    senseiReq.setFetchAllStoredFields(params.getBoolean(PARAM_FETCH_ALL_STORED_FIELDS, false));
+    senseiReq.setFetchStoredValue(params.getBoolean(PARAM_FETCH_STORED_VALUE, false));
 
     String[] fetchTVs = params.getStringArray(PARAM_FETCH_TERMVECTOR);
     if (fetchTVs != null && fetchTVs.length > 0) {

sensei-core/src/main/java/com/senseidb/servlet/SenseiSearchServletParams.java

@@ -12,7 +12,8 @@ public interface SenseiSearchServletParams {
   public static final String PARAM_SORT_SCORE_REVERSE = "relrev";
   public static final String PARAM_SORT_DOC = "doc";
   public static final String PARAM_SORT_DOC_REVERSE = "docrev";
-  public static final String PARAM_FETCH_STORED = "fetchstored";
+  public static final String PARAM_FETCH_ALL_STORED_FIELDS = "fetchallstoredfields";
+  public static final String PARAM_FETCH_STORED_VALUE = "fetchstoredvalue";
   public static final String PARAM_FIELDS_TO_FETCH = "fieldstofetch";
   public static final String PARAM_FETCH_TERMVECTOR = "fetchtermvector";
   public static final String PARAM_SHOW_EXPLAIN = "showexplain";

sensei-core/src/main/java/com/senseidb/svc/impl/CoreSenseiServiceImpl.java

@@ -78,7 +78,7 @@ private SenseiResult browse(SenseiRequest senseiRequest, MultiBoboBrowser browse
     }
     // SortCollector collector =
     // browser.getSortCollector(req.getSort(),req.getQuery(), offset, count,
-    // req.isFetchStoredFields(),false);
+    // req.isFetchAllStoredFields(),false);
 
     // Map<String, FacetAccessible> facetCollectors = new HashMap<String,
     // FacetAccessible>();

sensei-core/src/main/java/com/senseidb/svc/impl/HttpRestSenseiServiceImpl.java

@@ -322,8 +322,10 @@ public static void convertPartitionParams(List<NameValuePair> qparams, Set<Integ
   }
 
   public static void convertScalarParams(List<NameValuePair> qparams, SenseiRequest req) {
-    qparams.add(new BasicNameValuePair(SenseiSearchServletParams.PARAM_FETCH_STORED, Boolean
-        .toString(req.isFetchStoredFields())));
+    qparams.add(new BasicNameValuePair(SenseiSearchServletParams.PARAM_FETCH_ALL_STORED_FIELDS, Boolean
+        .toString(req.isFetchAllStoredFields())));
+    qparams.add(new BasicNameValuePair(SenseiSearchServletParams.PARAM_FETCH_STORED_VALUE, Boolean
+        .toString(req.isFetchStoredValue())));
     qparams.add(new BasicNameValuePair(SenseiSearchServletParams.PARAM_SHOW_EXPLAIN, Boolean
         .toString(req.isShowExplanation())));
     qparams.add(new BasicNameValuePair(SenseiSearchServletParams.PARAM_OFFSET, Integer.toString(req