SENSEI-327 Per-partition execution stats in response

clients/java/src/main/java/com/senseidb/search/client/res/SenseiRequestExecStatus.java

@@ -0,0 +1,27 @@
+package com.senseidb.search.client.res;
+
+public class SenseiRequestExecStatus {
+  private long time;
+  private int resultCount;
+
+  public long getTime() {
+    return time;
+  }
+
+  public void setTime(long time) {
+    this.time = time;
+  }
+
+  public int getResultCount() {
+    return resultCount;
+  }
+
+  public void setResultCount(int resultCount) {
+    this.resultCount = resultCount;
+  }
+
+  @Override
+  public String toString() {
+    return resultCount + " results in " + getTime() + "ms";
+  }
+}

clients/java/src/main/java/com/senseidb/search/client/res/SenseiResult.java

@@ -19,6 +19,7 @@ public class SenseiResult {
   private String parsedQuery;
   private Long time;
   private Map<String, List<FacetResult>> facets;
+  private Map<String, SenseiRequestExecStatus> partitionExecStats;
 
   private JSONObject mapReduceResult;
   private Integer errorCode;
@@ -42,7 +43,13 @@ public void setTid(Long tid) {
   public Integer getTotaldocs() {
     return totaldocs;
   }
+    public Map<String, SenseiRequestExecStatus> getPartitionExecStats() {
+        return partitionExecStats;
+    }
 
+    public void setPartitionExecStats(Map<String, SenseiRequestExecStatus> partitionExecStats) {
+        this.partitionExecStats = partitionExecStats;
+    }
   public void setTotaldocs(Integer totaldocs) {
     this.totaldocs = totaldocs;
   }

sensei-core/src/main/java/com/senseidb/search/node/ResultMerger.java

@@ -1,5 +1,6 @@
 package com.senseidb.search.node;
 
+import com.senseidb.search.req.SenseiRequestExecStats;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -565,11 +566,15 @@ public static SenseiResult merge(final SenseiRequest req, Collection<SenseiResul
     List<FacetAccessible>[] groupAccessibles = extractFacetAccessible(results);
 
     // Merge your facets
+    Map<String, SenseiRequestExecStats> mergedExecTimes = new HashMap<String, SenseiRequestExecStats>();
     for (SenseiResult res : results) {
       Map<String, FacetAccessible> facetMap = res.getFacetMap();
       if (facetMap != null) {
         facetList.add(facetMap);
       }
+      if (res.getPartitionExecStats() != null) {
+        mergedExecTimes.putAll(res.getPartitionExecStats());
+      }
     }
 
     Map<String, FacetAccessible> mergedFacetMap = null;
@@ -696,6 +701,7 @@ public static SenseiResult merge(final SenseiRequest req, Collection<SenseiResul
     }
 
     SenseiResult merged = new SenseiResult();
+    merged.setPartitionExecStats(mergedExecTimes);
     merged.setHits(hits);
     merged.setNumHitsLong(numHits);
     merged.setNumGroupsLong(numGroups);

sensei-core/src/main/java/com/senseidb/search/req/AbstractSenseiResult.java

@@ -1,11 +1,14 @@
 package com.senseidb.search.req;
 
 import java.io.Serializable;
+import java.util.Map;
 
-public interface AbstractSenseiResult extends Serializable {
+public interface AbstractSenseiResult<T extends SenseiExecStats> extends Serializable {
   public abstract long getTime();
 
   public abstract void setTime(long searchTimeMillis);
-
+  public T getExecStats();
   public void addError(SenseiError error);
+  public Map<String, T> getPartitionExecStats();
+  public void setPartitionExecStats(Map<String, T> partitionExecStats);
 }

sensei-core/src/main/java/com/senseidb/search/req/SenseiExecStats.java

@@ -0,0 +1,24 @@
+package com.senseidb.search.req;
+
+import java.io.Serializable;
+
+/**
+ * Represents runtime execution status for a request or subset (e.g. a query executed against a single partition or
+ * node)
+ */
+public class SenseiExecStats implements Serializable {
+  private final long time;
+
+  public SenseiExecStats(long time) {
+    this.time = time;
+  }
+
+  public long getTime() {
+    return time;
+  }
+
+  @Override
+  public String toString() {
+    return time + "ms";
+  }
+}

sensei-core/src/main/java/com/senseidb/search/req/SenseiRequestExecStats.java

@@ -0,0 +1,23 @@
+package com.senseidb.search.req;
+
+/**
+ * Contains runtime and number of results returned for execution of a request or subset of a request.
+ */
+public class SenseiRequestExecStats extends SenseiExecStats {
+
+  private final int resultCount;
+
+  public SenseiRequestExecStats(long time, int resultCount) {
+    super(time);
+    this.resultCount = resultCount;
+  }
+
+  public int getResultCount() {
+    return resultCount;
+  }
+
+  @Override
+  public String toString() {
+    return resultCount + " results in " + getTime() + "ms";
+  }
+}

sensei-core/src/main/java/com/senseidb/search/req/SenseiResult.java

@@ -13,7 +13,7 @@
 import com.browseengine.bobo.api.BrowseResult;
 import com.browseengine.bobo.api.FacetAccessible;
 
-public class SenseiResult extends BrowseResult implements AbstractSenseiResult {
+public class SenseiResult extends BrowseResult implements AbstractSenseiResult<SenseiRequestExecStats> {
 
   private static final long serialVersionUID = 1L;
 
@@ -22,7 +22,8 @@ public class SenseiResult extends BrowseResult implements AbstractSenseiResult {
   long numberOfHitsLong = 0;
   long numberOfGroupsLong = 0;
   private List<SenseiError> errors;
-
+  private Map<String, SenseiRequestExecStats> partitionExecStats;
+
   public SenseiHit[] getSenseiHits() {
     BrowseHit[] hits = getHits();
     if (hits == null || hits.length == 0) {
@@ -39,6 +40,11 @@ public String getParsedQuery() {
     return _parsedQuery;
   }
 
+  @Override
+  public void setPartitionExecStats(Map<String, SenseiRequestExecStats> partitionExecStats) {
+    this.partitionExecStats = partitionExecStats;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (!(o instanceof SenseiResult)) return false;
@@ -178,11 +184,21 @@ public void setNumGroupsLong(long numberGroupsLong) {
     this.numberOfGroupsLong = numberGroupsLong;
   }
 
+  @Override
+  public SenseiRequestExecStats getExecStats() {
+    return new SenseiRequestExecStats(getTime(), getNumHits());
+  }
+
+
   @Override
   public void addError(SenseiError error) {
     if (errors == null) errors = new ArrayList<SenseiError>();
 
     errors.add(error);
   }
 
+  @Override
+  public Map<String, SenseiRequestExecStats> getPartitionExecStats() {
+    return partitionExecStats;
+  }
 }

sensei-core/src/main/java/com/senseidb/search/req/SenseiSystemInfo.java

@@ -12,7 +12,7 @@
 import java.util.Map;
 import java.util.Set;
 
-public class SenseiSystemInfo implements AbstractSenseiResult {
+public class SenseiSystemInfo implements AbstractSenseiResult<SenseiExecStats> {
 
   public static class SenseiFacetInfo implements Serializable {
     private static final long serialVersionUID = 1L;
@@ -113,6 +113,7 @@ public String toString() {
   private Set<SenseiFacetInfo> _facetInfos;
   private String _schema; /* JSONObject is not protobuf serializerable, we use string here. */
   private List<SenseiNodeInfo> _clusterInfo;
+  private Map<String, SenseiExecStats> partitionExecStats;
 
   private List<SenseiError> errors;
 
@@ -135,6 +136,11 @@ public void setTime(long searchTimeMillis) {
     _searchTimeMillis = searchTimeMillis;
   }
 
+  @Override
+  public SenseiExecStats getExecStats() {
+    return new SenseiExecStats(getTime());
+  }
+
   public int getNumDocs() {
     return _numDocs;
   }
@@ -196,9 +202,19 @@ public void addError(SenseiError error) {
     errors.add(error);
   }
 
+  @Override
+  public Map<String, SenseiExecStats> getPartitionExecStats() {
+    return partitionExecStats;
+  }
+
+  @Override
+  public void setPartitionExecStats(Map<String, SenseiExecStats> partitionExecStats) {
+    this.partitionExecStats = partitionExecStats;
+  }
+
   @Override
   public String toString() {
-    StringBuffer buf = new StringBuffer();
+    StringBuilder buf = new StringBuilder();
     buf.append("\t- Number of Documents: ").append(_numDocs);
     buf.append("\n\t- Last Modified: ").append(
       new SimpleDateFormat("EEE, MMM d, ''yy").format(new Date(_lastModified)));

sensei-core/src/main/java/com/senseidb/servlet/DefaultSenseiJSONServlet.java

@@ -54,6 +54,7 @@
 import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_RESULT_NUMGROUPS;
 import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_RESULT_NUMHITS;
 import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_RESULT_PARSEDQUERY;
+import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_RESULT_PARTITION_EXEC_STATS;
 import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_RESULT_SELECT_LIST;
 import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_RESULT_TID;
 import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_RESULT_TIME;
@@ -417,6 +418,7 @@ public static JSONObject buildJSONResult(SenseiRequest req, SenseiResult res) th
     jsonObj.put(PARAM_RESULT_NUMHITS, res.getNumHitsLong());
     jsonObj.put(PARAM_RESULT_NUMGROUPS, res.getNumGroupsLong());
     jsonObj.put(PARAM_RESULT_PARSEDQUERY, res.getParsedQuery());
+    jsonObj.put(PARAM_RESULT_PARTITION_EXEC_STATS, res.getPartitionExecStats());
     addErrors(jsonObj, res);
     SenseiHit[] hits = res.getSenseiHits();
     JSONArray hitArray = buildJSONHits(req, hits);

sensei-core/src/main/java/com/senseidb/servlet/SenseiSearchServletParams.java

@@ -96,4 +96,6 @@ public interface SenseiSearchServletParams {
   public static final String PARAM_RESULT_FACET_INFO_VALUE = "value";
   public static final String PARAM_RESULT_FACET_INFO_COUNT = "count";
   public static final String PARAM_RESULT_FACET_INFO_SELECTED = "selected";
+
+  public static final String PARAM_RESULT_PARTITION_EXEC_STATS = "partitionExecStats";
 }

sensei-core/src/main/java/com/senseidb/svc/impl/AbstractSenseiCoreService.java

@@ -150,6 +150,10 @@ public Res call() throws Exception {
 
                 long end = System.currentTimeMillis();
                 res.setTime(end - start);
+
+                res.setPartitionExecStats(
+                  Collections.singletonMap(_core.getNodeId() + "-" + partition, res.getExecStats()));
+
                 return res;
               }
             });
@@ -175,6 +179,8 @@ public Res call() throws Exception {
             }
             Res res = getEmptyResultInstance(e);
             res.setTime(-1);
+            res.setPartitionExecStats(
+              Collections.singletonMap(_core.getNodeId() + "-" + partitionArray[i], res.getExecStats()));
             resultList.add(res);
           }
         }

sensei-core/src/main/java/com/senseidb/svc/impl/HttpRestSenseiServiceImpl.java

@@ -1,5 +1,7 @@
 package com.senseidb.svc.impl;
 
+
+import com.senseidb.search.req.SenseiRequestExecStats;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -22,9 +24,7 @@
 import java.util.Properties;
 import java.util.Set;
 import java.util.zip.GZIPInputStream;
-
 import javax.net.ssl.SSLHandshakeException;
-
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.StringUtils;
@@ -82,6 +82,7 @@
 import com.senseidb.servlet.SenseiSearchServletParams;
 import com.senseidb.svc.api.SenseiException;
 import com.senseidb.svc.api.SenseiService;
+import com.senseidb.util.JSONUtil.FastJSONArray;
 import com.senseidb.util.JSONUtil.FastJSONObject;
 
 public class HttpRestSenseiServiceImpl implements SenseiService {
@@ -668,6 +669,8 @@ public static SenseiResult buildSenseiResult(JSONObject jsonObj) throws JSONExce
     result.setTid(Long.parseLong(jsonObj.getString(SenseiSearchServletParams.PARAM_RESULT_TID)));
     result.setTotalDocsLong(jsonObj.getInt(SenseiSearchServletParams.PARAM_RESULT_TOTALDOCS));
     result.setParsedQuery(jsonObj.getString(SenseiSearchServletParams.PARAM_RESULT_PARSEDQUERY));
+    result.setPartitionExecStats(
+      convertPartitionExecStats(jsonObj.getJSONObject(SenseiSearchServletParams.PARAM_RESULT_PARTITION_EXEC_STATS)));
     result.setNumHitsLong(jsonObj.getInt(SenseiSearchServletParams.PARAM_RESULT_NUMHITS));
     if (jsonObj.has(SenseiSearchServletParams.PARAM_RESULT_NUMGROUPS)) {
       result.setNumGroupsLong(jsonObj.getInt(SenseiSearchServletParams.PARAM_RESULT_NUMGROUPS));
@@ -764,6 +767,22 @@ private static List<SenseiSystemInfo.SenseiNodeInfo> convertClusterInfo(JSONArra
     return clusterInfo;
   }
 
+ private static Map<String, SenseiRequestExecStats> convertPartitionExecStats(JSONObject jsonObject)
+   throws JSONException {
+   Map<String, SenseiRequestExecStats> map = new HashMap<String, SenseiRequestExecStats>();
+
+   Iterator iter = jsonObject.sortedKeys();
+
+   while (iter.hasNext()) {
+     String key = (String) iter.next();
+     JSONObject stats = jsonObject.getJSONObject(key);
+     long time = stats.getLong("time");
+     int numResults = stats.getInt("resultCount");
+     map.put(key, new SenseiRequestExecStats(time, numResults));
+   }
+   return map;
+ }
+
   private static Map<String, FacetAccessible> convertFacetMap(JSONObject jsonObject)
       throws JSONException {
     Map<String, FacetAccessible> map = new HashMap<String, FacetAccessible>();

sensei-core/src/test/java/com/senseidb/test/TestHttpRestSenseiServiceImpl.java

@@ -1,10 +1,17 @@
 package com.senseidb.test;
 
+
+import com.senseidb.search.req.SenseiRequestExecStats;
+import java.io.ByteArrayInputStream;
+import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
+import java.io.StringWriter;
 import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -65,6 +72,7 @@ private SenseiResult createMockResultFromRequest(SenseiRequest request) {
     result.setParsedQuery("This is my parsed query value");
     result.setTime(Long.MAX_VALUE / 2);
     result.setNumHits(Integer.MAX_VALUE / 2);
+    result.setPartitionExecStats(Collections.singletonMap("node1", new SenseiRequestExecStats(1, 2)));
     result.setTid(1);
     result.setTotalDocs(512);
     result.setHits(createSenseiHits(10));
@@ -372,9 +380,7 @@ SenseiQuery createSenseiQuery() throws JSONException {
       obj.put("key" + i, "val" + i);
     }
 
-    SenseiQuery query = new SenseiJSONQuery(obj);
-
-    return query;
+    return new SenseiJSONQuery(obj);
   }
 
   SortField[] createSortFields() {