Adding the ability for sensei filters to estimate cardinality to optimize filter execution order

sensei-core/src/main/java/com/senseidb/search/node/broker/BrokerConfig.java

@@ -69,8 +69,8 @@ public BrokerConfig(Configuration senseiConf,
     zkurl = senseiConf.getString(SenseiConfigServletContextListener.SENSEI_CONF_ZKURL, zkurl);
     clusterName = senseiConf.getString(SenseiConfigServletContextListener.SENSEI_CONF_CLUSTER_NAME, clusterName);
     zkTimeout = senseiConf.getInt(SenseiConfigServletContextListener.SENSEI_CONF_ZKTIMEOUT, zkTimeout);
-    outlierMultiplier = senseiConf.getDouble(SenseiConfigServletContextListener.SENSEI_CONF_ZKTIMEOUT, 3.0);
-    outlierConstant = senseiConf.getDouble(SenseiConfigServletContextListener.SENSEI_CONF_ZKTIMEOUT, 150);
+    outlierMultiplier = senseiConf.getDouble(SenseiConfigServletContextListener.SENSEI_CONF_NC_OUTLIER_MULTIPLIER, 3.0);
+    outlierConstant = senseiConf.getDouble(SenseiConfigServletContextListener.SENSEI_CONF_NC_OUTLIER_CONSTANT, 150);
 
     connectTimeoutMillis = senseiConf.getInt(SenseiConfigServletContextListener.SENSEI_CONF_NC_CONN_TIMEOUT, 1000);
     writeTimeoutMillis = senseiConf.getInt(SenseiConfigServletContextListener.SENSEI_CONF_NC_WRITE_TIMEOUT, 150);

sensei-core/src/main/java/com/senseidb/search/query/filters/AndFilterConstructor.java

@@ -52,16 +52,16 @@ public AndFilterConstructor(QueryParser qparser)
   }
 
   @Override
-  protected Filter doConstructFilter(Object obj) throws Exception
+  protected SenseiFilter doConstructFilter(Object obj) throws Exception
   {
     JSONArray filterArray = (JSONArray)obj;
-    List<Filter> filters = new ArrayList<Filter>(filterArray.length());
+    List<SenseiFilter> filters = new ArrayList<SenseiFilter>(filterArray.length());
     for (int i=0; i<filterArray.length(); ++i)
     {
-      Filter filter = FilterConstructor.constructFilter(filterArray.getJSONObject(i), _qparser);
+      SenseiFilter filter = FilterConstructor.constructFilter(filterArray.getJSONObject(i), _qparser);
       if (filter != null)
         filters.add(filter);
     }
-    return new AndFilter(filters);
+    return new SenseiAndFilter(filters);
   }
 }

sensei-core/src/main/java/com/senseidb/search/query/filters/BooleanFilterConstructor.java

@@ -62,11 +62,11 @@ public BooleanFilterConstructor(QueryParser qparser)
   }
 
   @Override
-  protected Filter doConstructFilter(Object param) throws Exception
+  protected SenseiFilter doConstructFilter(Object param) throws Exception
   {
     JSONObject json = (JSONObject)param;
     Object obj = json.opt(MUST_PARAM);
-    List<Filter> andFilters = new ArrayList<Filter>();
+    List<SenseiFilter> andFilters = new ArrayList<SenseiFilter>();
     if (obj != null)
     {
       if (obj instanceof JSONArray)
@@ -90,29 +90,29 @@ else if (obj instanceof JSONObject)
         for (int i=0; i<((JSONArray)obj).length(); ++i)
         {
           andFilters.add(
-            new NotFilter(FilterConstructor.constructFilter(((JSONArray)obj).getJSONObject(i),
+            new SenseiNotFilter(FilterConstructor.constructFilter(((JSONArray)obj).getJSONObject(i),
                                                             _qparser)));
         }
       }
       else if (obj instanceof JSONObject)
       {
-        andFilters.add(new NotFilter(FilterConstructor.constructFilter((JSONObject)obj, _qparser)));
+        andFilters.add(new SenseiNotFilter(FilterConstructor.constructFilter((JSONObject)obj, _qparser)));
       }
     }
     JSONArray array = json.optJSONArray(SHOULD_PARAM);
     if (array != null)
     {
-      List<Filter> orFilters = new ArrayList<Filter>(array.length());
+      List<SenseiFilter> orFilters = new ArrayList<SenseiFilter>(array.length());
       for (int i=0; i<array.length(); ++i)
       {
         orFilters.add(FilterConstructor.constructFilter(array.getJSONObject(i), _qparser));
       }
       if (orFilters.size() > 0)
-        andFilters.add(new OrFilter(orFilters));
+        andFilters.add(new SenseiOrFilter(orFilters));
     }
 
     if (andFilters.size() > 0)
-      return new AndFilter(andFilters);
+      return new SenseiAndFilter(andFilters);
     else
       return null;
   }

sensei-core/src/main/java/com/senseidb/search/query/filters/ConstExpFilterConstructor.java

@@ -18,7 +18,10 @@
  */
 package com.senseidb.search.query.filters;
 
+import com.senseidb.search.query.MatchNoneDocsQuery;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryWrapperFilter;
 import org.json.JSONObject;
@@ -27,17 +30,38 @@
 import com.senseidb.util.JSONUtil.FastJSONArray;
 import com.senseidb.util.JSONUtil.FastJSONObject;
 
+import java.io.IOException;
+
 public class ConstExpFilterConstructor extends FilterConstructor
 {
   public static final String FILTER_TYPE = "const_exp";
 
   @Override
-  protected Filter doConstructFilter(Object json) throws Exception
+  protected SenseiFilter doConstructFilter(Object json) throws Exception
   {
     Query q = QueryConstructor.constructQuery(new FastJSONObject().put(FILTER_TYPE, (JSONObject)json), null);
     if (q == null)
       return null;
-    return new QueryWrapperFilter(q);
-  } 
+
+    final QueryWrapperFilter filter = new QueryWrapperFilter(q);
+
+    if(q instanceof MatchAllDocsQuery) {
+      return new SenseiFilter() {
+        @Override
+        public SenseiDocIdSet getSenseiDocIdSet(IndexReader reader) throws IOException {
+          return new SenseiDocIdSet(filter.getDocIdSet(reader), reader.maxDoc());
+        }
+      };
+    } else if(q instanceof MatchNoneDocsQuery) {
+      return new SenseiFilter() {
+        @Override
+        public SenseiDocIdSet getSenseiDocIdSet(IndexReader reader) throws IOException {
+          return new SenseiDocIdSet(filter.getDocIdSet(reader), 0);
+        }
+      };
+    } else {
+      return SenseiFilter.buildDefault(filter);
+    }
+  }
 
 }

sensei-core/src/main/java/com/senseidb/search/query/filters/CustomFilterConstructor.java

@@ -18,9 +18,12 @@
  */
 package com.senseidb.search.query.filters;
 
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.Filter;
 import org.json.JSONObject;
 
+import java.io.IOException;
+
 public class CustomFilterConstructor extends FilterConstructor
 {
 
@@ -33,15 +36,20 @@ public class CustomFilterConstructor extends FilterConstructor
 //  }
 
   @Override
-  protected Filter doConstructFilter(Object json) throws Exception
+  protected SenseiFilter doConstructFilter(Object json) throws Exception
   {
     try
     {
       String className = ((JSONObject)json).getString(CLASS_PARAM);
       Class filterClass = Class.forName(className);
 
-      Filter f = (Filter)filterClass.newInstance();
-      return f;
+      final Filter f = (Filter)filterClass.newInstance();
+
+      if(f instanceof SenseiFilter) {
+        return (SenseiFilter) f;
+      } else {
+        return SenseiFilter.buildDefault(f);
+      }
     }
     catch(Throwable t)
     {

sensei-core/src/main/java/com/senseidb/search/query/filters/FacetSelectionFilterConstructor.java

@@ -19,10 +19,9 @@
 package com.senseidb.search.query.filters;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.Map;
+import java.util.*;
 
+import com.browseengine.bobo.facets.filter.RandomAccessFilter;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.DocIdSet;
 import org.apache.lucene.search.Filter;
@@ -63,25 +62,24 @@ public static BrowseSelection buildFacetSelection(String name,JSONObject json) t
 
 
   @Override
-  protected Filter doConstructFilter(Object obj) throws Exception {
+  protected SenseiFilter doConstructFilter(Object obj) throws Exception {
     final JSONObject json = (JSONObject)obj;
-    return new Filter(){
+    return new SenseiFilter(){
 
       @Override
-      public DocIdSet getDocIdSet(IndexReader reader)
-          throws IOException {
+      public SenseiDocIdSet getSenseiDocIdSet(IndexReader reader) throws IOException {
         if (reader instanceof BoboIndexReader){
           BoboIndexReader boboReader = (BoboIndexReader)reader;
           Iterator<String> iter = json.keys();
-          ArrayList<DocIdSet> docSets = new ArrayList<DocIdSet>();
+          ArrayList<RandomAccessFilter> filters = new ArrayList<RandomAccessFilter>();
           while(iter.hasNext()){
             String key = iter.next();
             FacetHandler facetHandler = boboReader.getFacetHandler(key);
             if (facetHandler!=null){
               try{
                 JSONObject jsonObj = json.getJSONObject(key);
                 BrowseSelection sel = buildFacetSelection(key, jsonObj);
-                docSets.add(facetHandler.buildFilter(sel).getDocIdSet(boboReader));
+                filters.add(facetHandler.buildFilter(sel));
               }
               catch(Exception e){
                 throw new IOException(e.getMessage());
@@ -91,9 +89,15 @@ public DocIdSet getDocIdSet(IndexReader reader)
               throw new IOException(key+" is not defined as a facet handler");
             }
           }
-          if (docSets.size()==0) return null;
-          else if (docSets.size()==1) return docSets.get(0);
-          return new AndDocIdSet(docSets);
+          if(filters.isEmpty()) {
+            return null;
+          } else {
+            List<SenseiFilter> senseiFilters = new ArrayList<SenseiFilter>(filters.size());
+            for(RandomAccessFilter raf : filters) {
+              senseiFilters.add(SenseiFilter.build(raf));
+            }
+            return new SenseiAndFilter(senseiFilters).getSenseiDocIdSet(reader);
+          }
         }
         else{
           throw new IllegalStateException("reader not instance of "+BoboIndexReader.class);

sensei-core/src/main/java/com/senseidb/search/query/filters/FilterConstructor.java

@@ -100,7 +100,7 @@ public static Map<String,String> convertParams(JSONObject obj){
 		return paramMap;
 	}
 
-	public static Filter constructFilter(JSONObject json, QueryParser qparser) throws Exception
+	public static SenseiFilter constructFilter(JSONObject json, QueryParser qparser) throws Exception
   {
     if (json == null)
       return null;
@@ -118,6 +118,6 @@ public static Filter constructFilter(JSONObject json, QueryParser qparser) throw
     return filterConstructor.doConstructFilter(json.get(type));
   }
 
-	abstract protected Filter doConstructFilter(Object json/* JSONObject or JSONArray */) throws Exception;
+	abstract protected SenseiFilter doConstructFilter(Object json/* JSONObject or JSONArray */) throws Exception;
 
 }

sensei-core/src/main/java/com/senseidb/search/query/filters/NullFilterConstructor.java

@@ -20,6 +20,8 @@
 
 import java.io.IOException;
 
+import com.browseengine.bobo.facets.FacetHandler;
+import com.browseengine.bobo.facets.impl.MultiValueFacetHandler;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.DocIdSet;
 import org.apache.lucene.search.DocIdSetIterator;
@@ -35,28 +37,44 @@
 
 public class NullFilterConstructor extends FilterConstructor {
   public static final String FILTER_TYPE = "isNull";
+
   @Override
-  protected Filter doConstructFilter(Object json) throws Exception {
+  protected SenseiFilter doConstructFilter(Object json) throws Exception {
     final String fieldName =  json instanceof String ? (String) json : ((JSONObject) json).getString("field");
-    return new Filter() {
-
+    return new SenseiFilter() {
       @Override
-      public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
-        final Object data = ((BoboIndexReader)reader).getFacetData(fieldName);
-        if (data instanceof MultiValueFacetDataCache) {
-          return new DocIdSet() {            
+      public SenseiDocIdSet getSenseiDocIdSet(IndexReader reader) throws IOException {
+        BoboIndexReader boboReader = (BoboIndexReader) reader;
+        FacetHandler facetHandler = boboReader.getFacetHandler(fieldName);
+        Object facetData = facetHandler.getFacetData(boboReader);
+
+        if(facetData instanceof MultiValueFacetDataCache)
+        {
+          final MultiValueFacetDataCache facetDataCache = (MultiValueFacetDataCache) facetData;
+          int cardinality = facetDataCache.freqs[0];
+
+          DocIdSet docIdSet = new DocIdSet() {
             @Override
-            public DocIdSetIterator iterator() throws IOException {              
-              return new MultiValueFacetDocIdSetIterator((MultiValueFacetDataCache)data, 0);
+            public DocIdSetIterator iterator() throws IOException {
+              return new MultiValueFacetDocIdSetIterator(facetDataCache, 0);
             }
           };
-        } else if (data instanceof FacetDataCache) {
-          return new DocIdSet() {            
+
+          return new SenseiDocIdSet(docIdSet, cardinality);
+        }
+        else if (facetData instanceof FacetDataCache)
+        {
+          final FacetDataCache facetDataCache = (FacetDataCache) facetData;
+          int cardinality = facetDataCache.freqs[0];
+
+          DocIdSet docIdSet = new DocIdSet() {
             @Override
-            public DocIdSetIterator iterator() throws IOException {              
-              return new FacetFilter.FacetDocIdSetIterator((FacetDataCache) data, 0);
+            public DocIdSetIterator iterator() throws IOException {
+              return new FacetFilter.FacetDocIdSetIterator(facetDataCache, 0);
             }
           };
+
+          return new SenseiDocIdSet(docIdSet, cardinality);
         }
         throw new UnsupportedOperationException("The null filter is supported only for the bobo facetHandlers that use FacetDataCache");
       }

sensei-core/src/main/java/com/senseidb/search/query/filters/OrFilterConstructor.java

@@ -49,15 +49,15 @@ public OrFilterConstructor(QueryParser qparser)
   }
 
   @Override
-  protected Filter doConstructFilter(Object obj) throws Exception {
+  protected SenseiFilter doConstructFilter(Object obj) throws Exception {
     JSONArray filterArray = (JSONArray)obj;
-    List<Filter> filters = new ArrayList<Filter>(filterArray.length());
+    List<SenseiFilter> filters = new ArrayList<SenseiFilter>(filterArray.length());
     for (int i=0; i<filterArray.length(); ++i)
     {
-      Filter filter = FilterConstructor.constructFilter(filterArray.getJSONObject(i), _qparser);
+      SenseiFilter filter = FilterConstructor.constructFilter(filterArray.getJSONObject(i), _qparser);
       if (filter != null)
         filters.add(filter);
     }
-    return new OrFilter(filters);
+    return new SenseiOrFilter(filters);
   }
 }