From 5fda5e13c22b3064d180ead62dab4869b3692d1a Mon Sep 17 00:00:00 2001 From: Kanako Hayashi Date: Wed, 10 Apr 2013 16:22:09 -0700 Subject: [PATCH] =?UTF-8?q?Removing=20inMemorySenseiServiceClass,=20which?= =?UTF-8?q?=20improperly=20uses=20of=20JVM.=20Details=20in=20https://iwww.?= =?UTF-8?q?corp.linkedin.com/wiki/cf/display/ENGS/Sensei+JMX+NPE+Problem?= =?UTF-8?q?=20Seas=20doesn=E2=80=99t=20use=20this=20class.=20Removing=20th?= =?UTF-8?q?is=20class=20and=20associated=20classes=20to=20prevent=20future?= =?UTF-8?q?=20use.=20CNC=20version=20of=20Sensei=20causes=20a=20issue.=20J?= =?UTF-8?q?Ira=20-=20SI=20571.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ran Sensei test and Seas test. Passed both. --- .../java/com/senseidb/jmx/MockJMXServer.java | 280 ------------------ .../node/inmemory/InMemorySenseiService.java | 193 ------------ .../node/inmemory/MockIndexReaderFactory.java | 66 ----- .../search/node/inmemory/MockSenseiCore.java | 67 ----- .../node/inmemory/InMemoryIndexPerfEval.java | 96 ------ .../inmemory/InMemorySenseiServiceTest.java | 57 ---- .../federated/broker/FederatedBroker.java | 10 +- .../broker/proxy/GenericBrokerProxy.java | 73 ----- .../FederatedBrokerIntegrationTest.java | 94 ------ .../federated/broker/SingleNodeStarter.java | 4 - .../resources/federatedBroker-context.xml | 51 ---- 11 files changed, 2 insertions(+), 989 deletions(-) delete mode 100644 sensei-core/src/main/java/com/senseidb/jmx/MockJMXServer.java delete mode 100644 sensei-core/src/main/java/com/senseidb/search/node/inmemory/InMemorySenseiService.java delete mode 100644 sensei-core/src/main/java/com/senseidb/search/node/inmemory/MockIndexReaderFactory.java delete mode 100644 sensei-core/src/main/java/com/senseidb/search/node/inmemory/MockSenseiCore.java delete mode 100644 sensei-core/src/test/java/com/senseidb/search/node/inmemory/InMemoryIndexPerfEval.java delete mode 100644 sensei-core/src/test/java/com/senseidb/search/node/inmemory/InMemorySenseiServiceTest.java delete mode 100644 sensei-federated-broker/src/main/java/com/senseidb/federated/broker/proxy/GenericBrokerProxy.java delete mode 100644 sensei-federated-broker/src/test/java/com/senseidb/federated/broker/FederatedBrokerIntegrationTest.java delete mode 100644 sensei-federated-broker/src/test/resources/federatedBroker-context.xml diff --git a/sensei-core/src/main/java/com/senseidb/jmx/MockJMXServer.java b/sensei-core/src/main/java/com/senseidb/jmx/MockJMXServer.java deleted file mode 100644 index ce0302436..000000000 --- a/sensei-core/src/main/java/com/senseidb/jmx/MockJMXServer.java +++ /dev/null @@ -1,280 +0,0 @@ -/** - * This software is licensed to you under the Apache License, Version 2.0 (the - * "Apache License"). - * - * LinkedIn's contributions are made under the Apache License. If you contribute - * to the Software, the contributions will be deemed to have been made under the - * Apache License, unless you expressly indicate otherwise. Please do not make any - * contributions that would be inconsistent with the Apache License. - * - * You may obtain a copy of the Apache License at http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, this software - * distributed under the Apache License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the Apache - * License for the specific language governing permissions and limitations for the - * software governed under the Apache License. - * - * © 2012 LinkedIn Corp. All Rights Reserved. - */ -package com.senseidb.jmx; - -import java.io.ObjectInputStream; -import java.util.Set; - -import javax.management.Attribute; -import javax.management.AttributeList; -import javax.management.AttributeNotFoundException; -import javax.management.InstanceAlreadyExistsException; -import javax.management.InstanceNotFoundException; -import javax.management.IntrospectionException; -import javax.management.InvalidAttributeValueException; -import javax.management.ListenerNotFoundException; -import javax.management.MBeanException; -import javax.management.MBeanInfo; -import javax.management.MBeanRegistrationException; -import javax.management.MBeanServer; -import javax.management.NotCompliantMBeanException; -import javax.management.NotificationFilter; -import javax.management.NotificationListener; -import javax.management.ObjectInstance; -import javax.management.ObjectName; -import javax.management.OperationsException; -import javax.management.QueryExp; -import javax.management.ReflectionException; -import javax.management.loading.ClassLoaderRepository; - -public class MockJMXServer implements MBeanServer { - - @Override - public ObjectInstance createMBean(String className, ObjectName name) throws ReflectionException, InstanceAlreadyExistsException, - MBeanRegistrationException, MBeanException, NotCompliantMBeanException { - // TODO Auto-generated method stub - return null; - } - - @Override - public ObjectInstance createMBean(String className, ObjectName name, ObjectName loaderName) throws ReflectionException, - InstanceAlreadyExistsException, MBeanRegistrationException, MBeanException, NotCompliantMBeanException, InstanceNotFoundException { - // TODO Auto-generated method stub - return null; - } - - @Override - public ObjectInstance createMBean(String className, ObjectName name, Object[] params, String[] signature) throws ReflectionException, - InstanceAlreadyExistsException, MBeanRegistrationException, MBeanException, NotCompliantMBeanException { - // TODO Auto-generated method stub - return null; - } - - @Override - public ObjectInstance createMBean(String className, ObjectName name, ObjectName loaderName, Object[] params, String[] signature) - throws ReflectionException, InstanceAlreadyExistsException, MBeanRegistrationException, MBeanException, NotCompliantMBeanException, - InstanceNotFoundException { - // TODO Auto-generated method stub - return null; - } - - @Override - public ObjectInstance registerMBean(Object object, ObjectName name) throws InstanceAlreadyExistsException, MBeanRegistrationException, - NotCompliantMBeanException { - // TODO Auto-generated method stub - return null; - } - - @Override - public void unregisterMBean(ObjectName name) throws InstanceNotFoundException, MBeanRegistrationException { - // TODO Auto-generated method stub - - } - - @Override - public ObjectInstance getObjectInstance(ObjectName name) throws InstanceNotFoundException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Set queryMBeans(ObjectName name, QueryExp query) { - // TODO Auto-generated method stub - return null; - } - - @Override - public Set queryNames(ObjectName name, QueryExp query) { - // TODO Auto-generated method stub - return null; - } - - @Override - public boolean isRegistered(ObjectName name) { - // TODO Auto-generated method stub - return false; - } - - @Override - public Integer getMBeanCount() { - // TODO Auto-generated method stub - return null; - } - - @Override - public Object getAttribute(ObjectName name, String attribute) throws MBeanException, AttributeNotFoundException, - InstanceNotFoundException, ReflectionException { - // TODO Auto-generated method stub - return null; - } - - @Override - public AttributeList getAttributes(ObjectName name, String[] attributes) throws InstanceNotFoundException, ReflectionException { - // TODO Auto-generated method stub - return null; - } - - @Override - public void setAttribute(ObjectName name, Attribute attribute) throws InstanceNotFoundException, AttributeNotFoundException, - InvalidAttributeValueException, MBeanException, ReflectionException { - // TODO Auto-generated method stub - - } - - @Override - public AttributeList setAttributes(ObjectName name, AttributeList attributes) throws InstanceNotFoundException, ReflectionException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Object invoke(ObjectName name, String operationName, Object[] params, String[] signature) throws InstanceNotFoundException, - MBeanException, ReflectionException { - // TODO Auto-generated method stub - return null; - } - - @Override - public String getDefaultDomain() { - // TODO Auto-generated method stub - return null; - } - - @Override - public String[] getDomains() { - // TODO Auto-generated method stub - return null; - } - - @Override - public void addNotificationListener(ObjectName name, NotificationListener listener, NotificationFilter filter, Object handback) - throws InstanceNotFoundException { - // TODO Auto-generated method stub - - } - - @Override - public void addNotificationListener(ObjectName name, ObjectName listener, NotificationFilter filter, Object handback) - throws InstanceNotFoundException { - // TODO Auto-generated method stub - - } - - @Override - public void removeNotificationListener(ObjectName name, ObjectName listener) throws InstanceNotFoundException, ListenerNotFoundException { - // TODO Auto-generated method stub - - } - - @Override - public void removeNotificationListener(ObjectName name, ObjectName listener, NotificationFilter filter, Object handback) - throws InstanceNotFoundException, ListenerNotFoundException { - // TODO Auto-generated method stub - - } - - @Override - public void removeNotificationListener(ObjectName name, NotificationListener listener) throws InstanceNotFoundException, - ListenerNotFoundException { - // TODO Auto-generated method stub - - } - - @Override - public void removeNotificationListener(ObjectName name, NotificationListener listener, NotificationFilter filter, Object handback) - throws InstanceNotFoundException, ListenerNotFoundException { - // TODO Auto-generated method stub - - } - - @Override - public MBeanInfo getMBeanInfo(ObjectName name) throws InstanceNotFoundException, IntrospectionException, ReflectionException { - // TODO Auto-generated method stub - return null; - } - - @Override - public boolean isInstanceOf(ObjectName name, String className) throws InstanceNotFoundException { - // TODO Auto-generated method stub - return false; - } - - @Override - public Object instantiate(String className) throws ReflectionException, MBeanException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Object instantiate(String className, ObjectName loaderName) throws ReflectionException, MBeanException, InstanceNotFoundException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Object instantiate(String className, Object[] params, String[] signature) throws ReflectionException, MBeanException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Object instantiate(String className, ObjectName loaderName, Object[] params, String[] signature) throws ReflectionException, - MBeanException, InstanceNotFoundException { - // TODO Auto-generated method stub - return null; - } - - @Override - public ObjectInputStream deserialize(ObjectName name, byte[] data) throws InstanceNotFoundException, OperationsException { - // TODO Auto-generated method stub - return null; - } - - @Override - public ObjectInputStream deserialize(String className, byte[] data) throws OperationsException, ReflectionException { - // TODO Auto-generated method stub - return null; - } - - @Override - public ObjectInputStream deserialize(String className, ObjectName loaderName, byte[] data) throws InstanceNotFoundException, - OperationsException, ReflectionException { - // TODO Auto-generated method stub - return null; - } - - @Override - public ClassLoader getClassLoaderFor(ObjectName mbeanName) throws InstanceNotFoundException { - // TODO Auto-generated method stub - return null; - } - - @Override - public ClassLoader getClassLoader(ObjectName loaderName) throws InstanceNotFoundException { - // TODO Auto-generated method stub - return null; - } - - @Override - public ClassLoaderRepository getClassLoaderRepository() { - // TODO Auto-generated method stub - return null; - } - -} diff --git a/sensei-core/src/main/java/com/senseidb/search/node/inmemory/InMemorySenseiService.java b/sensei-core/src/main/java/com/senseidb/search/node/inmemory/InMemorySenseiService.java deleted file mode 100644 index a425e5a2d..000000000 --- a/sensei-core/src/main/java/com/senseidb/search/node/inmemory/InMemorySenseiService.java +++ /dev/null @@ -1,193 +0,0 @@ -/** - * This software is licensed to you under the Apache License, Version 2.0 (the - * "Apache License"). - * - * LinkedIn's contributions are made under the Apache License. If you contribute - * to the Software, the contributions will be deemed to have been made under the - * Apache License, unless you expressly indicate otherwise. Please do not make any - * contributions that would be inconsistent with the Apache License. - * - * You may obtain a copy of the Apache License at http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, this software - * distributed under the Apache License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the Apache - * License for the specific language governing permissions and limitations for the - * software governed under the Apache License. - * - * © 2012 LinkedIn Corp. All Rights Reserved. - */ -package com.senseidb.search.node.inmemory; - -import java.io.File; -import java.lang.management.ManagementFactory; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import javax.management.MBeanServer; - -import org.apache.commons.configuration.Configuration; -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.util.Version; -import org.json.JSONException; -import org.json.JSONObject; - -import proj.zoie.api.ZoieIndexReader; -import proj.zoie.api.ZoieMultiReader; -import proj.zoie.api.ZoieSegmentReader; -import proj.zoie.api.indexing.AbstractZoieIndexable; -import proj.zoie.api.indexing.ZoieIndexable; -import proj.zoie.impl.indexing.ZoieConfig; - -import com.browseengine.bobo.api.BoboIndexReader; -import com.browseengine.bobo.facets.FacetHandler; -import com.browseengine.bobo.facets.RuntimeFacetHandlerFactory; -import com.senseidb.conf.SenseiConfParams; -import com.senseidb.conf.SenseiFacetHandlerBuilder; -import com.senseidb.conf.SenseiSchema; -import com.senseidb.conf.SenseiServerBuilder; -import com.senseidb.indexing.DefaultJsonSchemaInterpreter; -import com.senseidb.indexing.ShardingStrategy; -import com.senseidb.indexing.activity.ActivityPersistenceFactory; -import com.senseidb.jmx.JmxUtil; -import com.senseidb.jmx.MockJMXServer; -import com.senseidb.plugin.SenseiPluginRegistry; -import com.senseidb.search.node.SenseiIndexReaderDecorator; -import com.senseidb.search.plugin.PluggableSearchEngineManager; -import com.senseidb.search.req.SenseiRequest; -import com.senseidb.search.req.SenseiResult; -import com.senseidb.search.req.SenseiSystemInfo; -import com.senseidb.svc.impl.CoreSenseiServiceImpl; - -public class InMemorySenseiService { - private DefaultJsonSchemaInterpreter defaultJsonSchemaInterpreter; - private List> facets; - private List> runtimeFacets; - - private CoreSenseiServiceImpl coreSenseiServiceImpl; - private PluggableSearchEngineManager pluggableSearchEngineManager; - private MockSenseiCore mockSenseiCore; - private SenseiSystemInfo senseiSystemInfo; - - - public InMemorySenseiService(SenseiSchema schema, SenseiPluginRegistry pluginRegistry) { - MBeanServer platformMBeanServer = ManagementFactory.getPlatformMBeanServer(); - schema.setCompressSrcData(false); - try { - platformMBeanServer = JmxUtil.registerNewJmxServer(new MockJMXServer()); - defaultJsonSchemaInterpreter = new DefaultJsonSchemaInterpreter(schema); - facets = new ArrayList>(); - runtimeFacets = new ArrayList>(); - - ShardingStrategy strategy = new ShardingStrategy() { - public int caculateShard(int maxShardId, JSONObject dataObj) throws JSONException { - return 0; - } - }; - ActivityPersistenceFactory.setOverrideForCurrentThread(ActivityPersistenceFactory.getInMemoryInstance()); - pluggableSearchEngineManager = new PluggableSearchEngineManager(); - pluggableSearchEngineManager.init("", 0, schema, ZoieConfig.DEFAULT_VERSION_COMPARATOR, pluginRegistry, strategy); - senseiSystemInfo = SenseiFacetHandlerBuilder.buildFacets(schema.getSchemaObj(), pluginRegistry, facets, runtimeFacets, - pluggableSearchEngineManager); - - int[] partitions = new int[]{0}; - mockSenseiCore = new MockSenseiCore(partitions); - pluggableSearchEngineManager.start(mockSenseiCore); - coreSenseiServiceImpl = new CoreSenseiServiceImpl(mockSenseiCore); - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - JmxUtil.registerNewJmxServer(platformMBeanServer); - ActivityPersistenceFactory.setOverrideForCurrentThread(null); - - } - } - - private void addDocuments(Directory directory, IndexWriter writer, List documents) { - try { - writer.deleteAll(); - for (JSONObject doc : documents) { - if (doc == null) - continue; - writer.addDocument(buildDoc(doc)); - pluggableSearchEngineManager.update(doc, ""); - } - writer.commit(); - - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @SuppressWarnings("unchecked") - public SenseiResult doQuery(SenseiRequest senseiRequest, List documents) { - Directory directory = null; - IndexWriter writer = null; - try { - directory = new RAMDirectory(); - writer = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_35, new StandardAnalyzer(Version.LUCENE_35))); - addDocuments(directory, writer, documents); - ZoieIndexReader zoieMultiReader = new ZoieMultiReader(IndexReader.open(directory), - new SenseiIndexReaderDecorator(facets, runtimeFacets)); - MockIndexReaderFactory mockIndexReaderFactory = new MockIndexReaderFactory>(Arrays.asList(zoieMultiReader)); - mockSenseiCore.setIndexReaderFactory(mockIndexReaderFactory); - SenseiResult result = coreSenseiServiceImpl.execute(senseiRequest); - mockSenseiCore.setIndexReaderFactory(null); - return result; - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - try { - if (writer != null) { - writer.close(); - } - if (directory != null) { - directory.close(); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } - - public Document buildDoc(JSONObject json) { - ZoieIndexable indexable = defaultJsonSchemaInterpreter.convertAndInterpret(json); - Document ret = indexable.buildIndexingReqs()[0].getDocument(); - ret.add(new Field(AbstractZoieIndexable.DOCUMENT_STORE_FIELD,indexable.getStoreValue())); - ZoieSegmentReader.fillDocumentID(ret, indexable.getUID()); - return ret; - } - - public static InMemorySenseiService valueOf(File confDir) { - try { - JSONObject schema = SenseiServerBuilder.loadSchema(confDir); - File senseiConfFile = new File(confDir, SenseiServerBuilder.SENSEI_PROPERTIES); - if (!senseiConfFile.exists()) { - throw new ConfigurationException("configuration file: " + senseiConfFile.getAbsolutePath() + " does not exist."); - } - Configuration senseiConf = new PropertiesConfiguration(); - ((PropertiesConfiguration) senseiConf).setDelimiterParsingDisabled(true); - ((PropertiesConfiguration) senseiConf).load(senseiConfFile); - return new InMemorySenseiService(SenseiSchema.build(schema), SenseiPluginRegistry.build(senseiConf)); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - } - - public SenseiSystemInfo getSenseiSystemInfo() { - return senseiSystemInfo; - } - - public void setSenseiSystemInfo(SenseiSystemInfo senseiSystemInfo) { - this.senseiSystemInfo = senseiSystemInfo; - } -} diff --git a/sensei-core/src/main/java/com/senseidb/search/node/inmemory/MockIndexReaderFactory.java b/sensei-core/src/main/java/com/senseidb/search/node/inmemory/MockIndexReaderFactory.java deleted file mode 100644 index 5c604e36b..000000000 --- a/sensei-core/src/main/java/com/senseidb/search/node/inmemory/MockIndexReaderFactory.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * This software is licensed to you under the Apache License, Version 2.0 (the - * "Apache License"). - * - * LinkedIn's contributions are made under the Apache License. If you contribute - * to the Software, the contributions will be deemed to have been made under the - * Apache License, unless you expressly indicate otherwise. Please do not make any - * contributions that would be inconsistent with the Apache License. - * - * You may obtain a copy of the Apache License at http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, this software - * distributed under the Apache License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the Apache - * License for the specific language governing permissions and limitations for the - * software governed under the Apache License. - * - * © 2012 LinkedIn Corp. All Rights Reserved. - */ -package com.senseidb.search.node.inmemory; - -import java.io.IOException; -import java.util.List; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.util.Version; - -import proj.zoie.api.IndexReaderFactory; -import proj.zoie.api.ZoieIndexReader; - -import com.browseengine.bobo.api.BoboIndexReader; - -public class MockIndexReaderFactory implements IndexReaderFactory > { - private Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_35); - - private final List> readers; - - public MockIndexReaderFactory(List> readers) { - this.readers = readers; - - } - - @Override - public List> getIndexReaders() throws IOException { - return readers; - - } - - @Override - public Analyzer getAnalyzer() { - - return analyzer; - } - - @Override - public void returnIndexReaders(List> r) { - - } - - @Override - public String getCurrentReaderVersion() { - - return null; - } - -} diff --git a/sensei-core/src/main/java/com/senseidb/search/node/inmemory/MockSenseiCore.java b/sensei-core/src/main/java/com/senseidb/search/node/inmemory/MockSenseiCore.java deleted file mode 100644 index e173d371a..000000000 --- a/sensei-core/src/main/java/com/senseidb/search/node/inmemory/MockSenseiCore.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * This software is licensed to you under the Apache License, Version 2.0 (the - * "Apache License"). - * - * LinkedIn's contributions are made under the Apache License. If you contribute - * to the Software, the contributions will be deemed to have been made under the - * Apache License, unless you expressly indicate otherwise. Please do not make any - * contributions that would be inconsistent with the Apache License. - * - * You may obtain a copy of the Apache License at http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, this software - * distributed under the Apache License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the Apache - * License for the specific language governing permissions and limitations for the - * software governed under the Apache License. - * - * © 2012 LinkedIn Corp. All Rights Reserved. - */ -package com.senseidb.search.node.inmemory; - - -import java.util.Collections; - -import java.util.List; - -import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.queryParser.QueryParser; -import org.apache.lucene.util.Version; - -import proj.zoie.api.IndexReaderFactory; -import proj.zoie.api.ZoieIndexReader; - -import com.browseengine.bobo.api.BoboIndexReader; -import com.senseidb.indexing.SenseiIndexPruner; -import com.senseidb.search.node.SenseiCore; -import com.senseidb.search.node.impl.DefaultJsonQueryBuilderFactory; - -public class MockSenseiCore extends SenseiCore { - - - private final ThreadLocal>> mockIndexReaderFactory = new ThreadLocal>>(); - private final int[] partitions; - private static MockIndexReaderFactory> emptyIndexFactory = new MockIndexReaderFactory>(Collections.EMPTY_LIST); - public MockSenseiCore(int[] partitions) { - super(0, new int[] { 0 }, null, null, new DefaultJsonQueryBuilderFactory(new QueryParser(Version.LUCENE_35, "contents", - new StandardAnalyzer(Version.LUCENE_35)))); - this.partitions = partitions; - setIndexPruner(new SenseiIndexPruner.DefaultSenseiIndexPruner()); - } - - @Override - public IndexReaderFactory> getIndexReaderFactory(int partition) { - - if (partition == partitions[0]) - return mockIndexReaderFactory.get(); - else { - return emptyIndexFactory; - } - } - public void setIndexReaderFactory(IndexReaderFactory> indexReaderFactory) { - mockIndexReaderFactory.set((MockIndexReaderFactory>)indexReaderFactory); - } - @Override - public int[] getPartitions() { - return partitions; - } -} diff --git a/sensei-core/src/test/java/com/senseidb/search/node/inmemory/InMemoryIndexPerfEval.java b/sensei-core/src/test/java/com/senseidb/search/node/inmemory/InMemoryIndexPerfEval.java deleted file mode 100644 index 214f446c0..000000000 --- a/sensei-core/src/test/java/com/senseidb/search/node/inmemory/InMemoryIndexPerfEval.java +++ /dev/null @@ -1,96 +0,0 @@ -/** - * This software is licensed to you under the Apache License, Version 2.0 (the - * "Apache License"). - * - * LinkedIn's contributions are made under the Apache License. If you contribute - * to the Software, the contributions will be deemed to have been made under the - * Apache License, unless you expressly indicate otherwise. Please do not make any - * contributions that would be inconsistent with the Apache License. - * - * You may obtain a copy of the Apache License at http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, this software - * distributed under the Apache License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the Apache - * License for the specific language governing permissions and limitations for the - * software governed under the Apache License. - * - * © 2012 LinkedIn Corp. All Rights Reserved. - */ - -package com.senseidb.search.node.inmemory; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.LineIterator; -import org.json.JSONObject; - -import com.browseengine.bobo.api.BrowseSelection; -import com.browseengine.bobo.api.FacetSpec; -import com.browseengine.bobo.api.FacetSpec.FacetSortSpec; -import com.senseidb.search.req.SenseiRequest; - -public class InMemoryIndexPerfEval { - - public static void main(String[] args) throws Exception { - final InMemorySenseiService memorySenseiService = InMemorySenseiService.valueOf(new File( - InMemoryIndexPerfEval.class.getClassLoader().getResource("test-conf/node1/").toURI())); - - - final List docs = new ArrayList(15000); - LineIterator lineIterator = FileUtils.lineIterator(new File(InMemoryIndexPerfEval.class.getClassLoader().getResource("data/test_data.json").toURI())); - int i = 0; - while(lineIterator.hasNext() && i < 100) { - String car = lineIterator.next(); - if (car != null && car.contains("{")) - docs.add(new JSONObject(car)); - i++; - } - Thread[] threads = new Thread[10]; - for (int k = 0 ; k < threads.length; k++) { - threads[k] = new Thread(new Runnable() { - public void run() { - long time = System.currentTimeMillis(); - //System.out.println("Start thread"); - for (int j = 0; j < 1000; j++) { - //System.out.println("Send request"); - memorySenseiService.doQuery(getRequest(), docs); - } - System.out.println("time = " + (System.currentTimeMillis() - time)); - } - }); - threads[k].start(); - } - Thread.sleep(500000); - } - private static void setspec(SenseiRequest req, FacetSpec spec) { - req.setFacetSpec("color", spec); - req.setFacetSpec("category", spec); - req.setFacetSpec("city", spec); - req.setFacetSpec("makemodel", spec); - req.setFacetSpec("year", spec); - req.setFacetSpec("price", spec); - req.setFacetSpec("mileage", spec); - req.setFacetSpec("tags", spec); - } - public static SenseiRequest getRequest() { - FacetSpec facetSpecall = new FacetSpec(); - facetSpecall.setMaxCount(1000000); - facetSpecall.setExpandSelection(true); - facetSpecall.setMinHitCount(0); - facetSpecall.setOrderBy(FacetSortSpec.OrderHitsDesc); - FacetSpec facetSpec = new FacetSpec(); - facetSpec.setMaxCount(5); - SenseiRequest req = new SenseiRequest(); - req.setCount(3); - facetSpecall.setMaxCount(3); - setspec(req, facetSpecall); - BrowseSelection sel = new BrowseSelection("year"); - String selVal = "[2001 TO 2002]"; - sel.addValue(selVal); - req.addSelection(sel); - return req; - } -} diff --git a/sensei-core/src/test/java/com/senseidb/search/node/inmemory/InMemorySenseiServiceTest.java b/sensei-core/src/test/java/com/senseidb/search/node/inmemory/InMemorySenseiServiceTest.java deleted file mode 100644 index 7c6385618..000000000 --- a/sensei-core/src/test/java/com/senseidb/search/node/inmemory/InMemorySenseiServiceTest.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * This software is licensed to you under the Apache License, Version 2.0 (the - * "Apache License"). - * - * LinkedIn's contributions are made under the Apache License. If you contribute - * to the Software, the contributions will be deemed to have been made under the - * Apache License, unless you expressly indicate otherwise. Please do not make any - * contributions that would be inconsistent with the Apache License. - * - * You may obtain a copy of the Apache License at http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, this software - * distributed under the Apache License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the Apache - * License for the specific language governing permissions and limitations for the - * software governed under the Apache License. - * - * © 2012 LinkedIn Corp. All Rights Reserved. - */ - -package com.senseidb.search.node.inmemory; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.LineIterator; -import org.json.JSONObject; - -import com.senseidb.search.req.SenseiResult; - -import junit.framework.TestCase; - -public class InMemorySenseiServiceTest extends TestCase { - private InMemorySenseiService inMemorySenseiService; - private List docs; - @Override - protected void setUp() throws Exception { - inMemorySenseiService = InMemorySenseiService.valueOf(new File( - InMemoryIndexPerfEval.class.getClassLoader().getResource("test-conf/node1/").toURI())); - LineIterator lineIterator = FileUtils.lineIterator(new File(InMemoryIndexPerfEval.class.getClassLoader().getResource("data/test_data.json").toURI())); - int i = 0; - docs = new ArrayList(); - while(lineIterator.hasNext() && i < 100) { - String car = lineIterator.next(); - if (car != null && car.contains("{")) - docs.add(new JSONObject(car)); - i++; - } - lineIterator.close(); - } - public void test1() { - SenseiResult result = inMemorySenseiService.doQuery(InMemoryIndexPerfEval.getRequest(), docs); - assertEquals(16, result.getNumHits()); - assertEquals(100, result.getTotalDocs()); - } -} diff --git a/sensei-federated-broker/src/main/java/com/senseidb/federated/broker/FederatedBroker.java b/sensei-federated-broker/src/main/java/com/senseidb/federated/broker/FederatedBroker.java index 0a599bf76..80fe7c6b9 100644 --- a/sensei-federated-broker/src/main/java/com/senseidb/federated/broker/FederatedBroker.java +++ b/sensei-federated-broker/src/main/java/com/senseidb/federated/broker/FederatedBroker.java @@ -37,7 +37,7 @@ import com.senseidb.search.node.Broker; import com.senseidb.search.node.ResultMerger; import com.senseidb.search.node.SenseiBroker; -import com.senseidb.search.node.inmemory.InMemorySenseiService; +//import com.senseidb.search.node.inmemory.InMemorySenseiService; import com.senseidb.search.req.SenseiRequest; import com.senseidb.search.req.SenseiResult; import com.senseidb.search.req.SenseiSystemInfo; @@ -113,13 +113,7 @@ public void run() { throw new RuntimeException(e); } } - - - public void setInMemorySenseiService(InMemorySenseiService inMemorySenseiService) { - if (inMemorySenseiService != null && inMemorySenseiService.getSenseiSystemInfo() != null) { - facetInfo = AbstractSenseiClientServlet.extractFacetInfo(inMemorySenseiService.getSenseiSystemInfo()); - } - } + public JSONObject query(JSONObject request) { try { SenseiRequest senseiRequest = RequestConverter2.fromJSON(request, facetInfo); diff --git a/sensei-federated-broker/src/main/java/com/senseidb/federated/broker/proxy/GenericBrokerProxy.java b/sensei-federated-broker/src/main/java/com/senseidb/federated/broker/proxy/GenericBrokerProxy.java deleted file mode 100644 index a6a964032..000000000 --- a/sensei-federated-broker/src/main/java/com/senseidb/federated/broker/proxy/GenericBrokerProxy.java +++ /dev/null @@ -1,73 +0,0 @@ -/** - * This software is licensed to you under the Apache License, Version 2.0 (the - * "Apache License"). - * - * LinkedIn's contributions are made under the Apache License. If you contribute - * to the Software, the contributions will be deemed to have been made under the - * Apache License, unless you expressly indicate otherwise. Please do not make any - * contributions that would be inconsistent with the Apache License. - * - * You may obtain a copy of the Apache License at http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, this software - * distributed under the Apache License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the Apache - * License for the specific language governing permissions and limitations for the - * software governed under the Apache License. - * - * © 2012 LinkedIn Corp. All Rights Reserved. - */ - -package com.senseidb.federated.broker.proxy; - -import java.util.Arrays; -import java.util.List; - -import org.json.JSONObject; -import org.springframework.util.Assert; - -import com.senseidb.search.node.inmemory.InMemorySenseiService; -import com.senseidb.search.req.SenseiRequest; -import com.senseidb.search.req.SenseiResult; - -public class GenericBrokerProxy implements BrokerProxy { - private InMemorySenseiService inMemorySenseiService; - private ProxyDataSource proxyDataSource; - private SenseiRequestFactory requestFactory; - public GenericBrokerProxy(InMemorySenseiService inMemorySenseiService, ProxyDataSource proxyDataSource) { - Assert.notNull(inMemorySenseiService); - Assert.notNull(proxyDataSource); - this.inMemorySenseiService = inMemorySenseiService; - this.proxyDataSource = proxyDataSource; - } - public GenericBrokerProxy() { - - } - @Override - public List doQuery(SenseiRequest senseiRequest) { - List documents = proxyDataSource.getData(senseiRequest); - if (requestFactory != null) { - senseiRequest = requestFactory.build(senseiRequest); - } - SenseiResult senseiResult = inMemorySenseiService.doQuery(senseiRequest, documents); - return Arrays.asList(senseiResult); - } - public InMemorySenseiService getInMemorySenseiService() { - return inMemorySenseiService; - } - public void setInMemorySenseiService(InMemorySenseiService inMemorySenseiService) { - this.inMemorySenseiService = inMemorySenseiService; - } - public ProxyDataSource getProxyDataSource() { - return proxyDataSource; - } - public void setProxyDataSource(ProxyDataSource proxyDataSource) { - this.proxyDataSource = proxyDataSource; - } - public SenseiRequestFactory getRequestFactory() { - return requestFactory; - } - public void setRequestFactory(SenseiRequestFactory requestFactory) { - this.requestFactory = requestFactory; - } - -} diff --git a/sensei-federated-broker/src/test/java/com/senseidb/federated/broker/FederatedBrokerIntegrationTest.java b/sensei-federated-broker/src/test/java/com/senseidb/federated/broker/FederatedBrokerIntegrationTest.java deleted file mode 100644 index b59fcf13f..000000000 --- a/sensei-federated-broker/src/test/java/com/senseidb/federated/broker/FederatedBrokerIntegrationTest.java +++ /dev/null @@ -1,94 +0,0 @@ -/** - * This software is licensed to you under the Apache License, Version 2.0 (the - * "Apache License"). - * - * LinkedIn's contributions are made under the Apache License. If you contribute - * to the Software, the contributions will be deemed to have been made under the - * Apache License, unless you expressly indicate otherwise. Please do not make any - * contributions that would be inconsistent with the Apache License. - * - * You may obtain a copy of the Apache License at http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, this software - * distributed under the Apache License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the Apache - * License for the specific language governing permissions and limitations for the - * software governed under the Apache License. - * - * © 2012 LinkedIn Corp. All Rights Reserved. - */ - -package com.senseidb.federated.broker; - -import java.io.File; -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; - -import junit.framework.TestCase; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.LineIterator; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.springframework.context.support.ClassPathXmlApplicationContext; - -import com.browseengine.bobo.api.BrowseSelection; -import com.senseidb.federated.broker.proxy.BrokerProxy; -import com.senseidb.search.req.SenseiRequest; -import com.senseidb.search.req.SenseiResult; - -import voldemort.client.StoreClient; - -public class FederatedBrokerIntegrationTest extends TestCase { - - private ClassPathXmlApplicationContext brokerContext; - private FederatedBroker federatedBroker; - private StoreClient storeClient; - private BrokerProxy senseiProxy; - @Override - protected void setUp() throws Exception { - SingleNodeStarter.start("conf", 15000); - brokerContext = new ClassPathXmlApplicationContext("federatedBroker-context.xml"); - federatedBroker = (FederatedBroker) brokerContext.getBean("federatedBroker", FederatedBroker.class); - storeClient = (StoreClient) brokerContext.getBean("storeClient"); - senseiProxy = (BrokerProxy) brokerContext.getBean("senseiProxy"); - JSONArray arr = readCarDocs(); - storeClient.put("test", arr.toString()); - } - private JSONArray readCarDocs() throws IOException, URISyntaxException, JSONException { - JSONArray arr = new JSONArray(); - LineIterator lineIterator = FileUtils.lineIterator(new File(FederatedBrokerIntegrationTest.class.getClassLoader().getResource("data/cars.json").toURI())); - while(lineIterator.hasNext()) { - String car = lineIterator.next(); - if (car != null && car.contains("{")) { - JSONObject carDoc = new JSONObject(car); - carDoc.put("id", carDoc.getLong("id") + 15000); - arr.put(carDoc); - } - - } - return arr; - } - public void test1SearchOnTwoClusters() throws Exception { - SenseiRequest req = new SenseiRequest(); - BrowseSelection sel = new BrowseSelection("year"); - String selVal = "[2001 TO 2002]"; - sel.addValue(selVal); - req .addSelection(sel); - SenseiResult result = federatedBroker.browse(req); - assertEquals(30000, result.getTotalDocs()); - assertEquals(5814, result.getNumHits()); - SenseiResult oneProxyResult = senseiProxy.doQuery(req).get(0); - assertEquals(15000, oneProxyResult.getTotalDocs()); - assertEquals(2907, oneProxyResult.getNumHits()); - - } - - - @Override - protected void tearDown() throws Exception { - brokerContext.close(); - } -} diff --git a/sensei-federated-broker/src/test/java/com/senseidb/federated/broker/SingleNodeStarter.java b/sensei-federated-broker/src/test/java/com/senseidb/federated/broker/SingleNodeStarter.java index 3c511058b..959fc07a2 100644 --- a/sensei-federated-broker/src/test/java/com/senseidb/federated/broker/SingleNodeStarter.java +++ b/sensei-federated-broker/src/test/java/com/senseidb/federated/broker/SingleNodeStarter.java @@ -39,10 +39,6 @@ public class SingleNodeStarter { private static Server jettyServer; private static SenseiServer server; - public static void start(String localPath, int expectedDocs) { - start(new File(getUri(localPath)), expectedDocs); - } - public static void start(File confDir, int expectedDocs) { if (!serverStarted) { try { diff --git a/sensei-federated-broker/src/test/resources/federatedBroker-context.xml b/sensei-federated-broker/src/test/resources/federatedBroker-context.xml deleted file mode 100644 index 9dfa84869..000000000 --- a/sensei-federated-broker/src/test/resources/federatedBroker-context.xml +++ /dev/null @@ -1,51 +0,0 @@ - - - - - - - - - - test - - - - - - src/test/resources/conf/sensei.properties - - - - - - - - - - - src/test/resources/conf - - - - - - - - - - - - - - - - \ No newline at end of file