Lucene7 (#63)

* wip

* Add missing parameter to make it compile (#62)

* update lucene 7.4.0

bin/clue.sh

@@ -9,8 +9,8 @@ dist=$bin/../target
 #HADOOP_CONF_DIR=
 
 HEAP_OPTS="-Xmx1g -Xms1g -XX:NewSize=256m"
-JAVA_OPTS="-server -d64 -Dhadoop.conf.dir=$HADOOP_CONF_DIR"
+JAVA_OPTS="-Dhadoop.conf.dir=$HADOOP_CONF_DIR"
 #JAVA_DEBUG="-Xdebug -Xrunjdwp:transport=dt_socket,address=1044,server=y,suspend=y"
 
-(cd $bin/..; java $JAVA_OPTS $JAVA_DEBUG $HEAP_OPTS -jar $dist/clue-6.6.2-1.0.0.jar $@)
+(cd $bin/..; java $JAVA_OPTS $JAVA_DEBUG $HEAP_OPTS -jar $dist/clue-*.jar $@)
 

pom.xml

@@ -6,7 +6,7 @@
     <artifactId>clue</artifactId>
     <packaging>jar</packaging>
     <name>clue</name>
-    <version>6.6.2-1.0.0</version>
+    <version>7.4.0-1.0.0</version>
     <description>command line client for Apache Lucene</description>
 
     <url>http://github.com/javasoze/clue</url>
@@ -36,7 +36,7 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <mainClass>io.dashbase.clue.ClueApplication</mainClass>
-        <lucene.version>6.6.2</lucene.version>
+        <lucene.version>7.4.0</lucene.version>
     </properties>
 
     <!-- Set the compiler to java6 -->

src/main/java/io/dashbase/clue/commands/DeleteUserCommitData.java

@@ -1,6 +1,9 @@
 package io.dashbase.clue.commands;
 
 import java.io.PrintStream;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Map;
 
 import io.dashbase.clue.ClueContext;
@@ -26,17 +29,19 @@ public String help() {
 	public void execute(String[] args, PrintStream out) throws Exception {
 		IndexWriter writer = ctx.getIndexWriter();
 		if (writer != null) {
-			if (args.length > 0) {				
-			  Map<String, String> commitData = writer.getCommitData();
-			  if (commitData != null && commitData.size() > 0) {
-			  	if (commitData.remove(args[0]) != null) {
-			      writer.setCommitData(commitData);
-			      writer.commit();
-			      ctx.refreshReader();
-			      out.println("commit data: " + args[0] +" removed.");
-			  	} else {
-			  		out.println("no commit data with the key: " + args[0] +", no action taken");
-			  	}
+			if (args.length > 0) {
+				Iterable<Map.Entry<String, String>> commitData = writer.getLiveCommitData();
+				List<Map.Entry<String, String>> commitList = new LinkedList<>();
+				for (Map.Entry<String, String> dataEntry : commitData) {
+					if (!dataEntry.equals(args[0])) {
+						commitList.add(dataEntry);
+					}
+				}
+			  if (commitList.size() > 0) {
+				  writer.setLiveCommitData(commitList);
+				  writer.commit();
+				  ctx.refreshReader();
+				  out.println("commit data: " + args[0] +" removed.");
 			  } else {
 			  	out.println("no commit data found, no action taken");
 			  }

src/main/java/io/dashbase/clue/commands/DocValCommand.java

@@ -44,68 +44,76 @@ private void showDocId(int docid, int docBase,
     int subid = docid - docBase;
     if (docVals != null) {
 
-      String val;
+      String val = null;
 
       switch (docValType) {
       case NUMERIC:
         NumericDocValues dv = (NumericDocValues)docVals;
-        val = String.valueOf(dv.get(subid));
+        if (dv.advanceExact(subid)) {
+          val = String.valueOf(dv.longValue());
+        }
         break;
       case BINARY:
         BinaryDocValues bv = (BinaryDocValues)docVals;
-        bytesRef = bv.get(subid);
-        val = bytesRef.utf8ToString();
+        if (bv.advanceExact(subid)) {
+          bytesRef = bv.binaryValue();
+          val = bytesRef.utf8ToString();
+        }
         break;
       case SORTED: {
         SortedDocValues sv = (SortedDocValues)docVals;
-        bytesRef = sv.get(subid);
-        StringBuilder sb = new StringBuilder();
-        sb.append(NUM_TERMS_IN_FIELD).append(sv.getValueCount()).append(", ");
-        sb.append("value: [");
-        sb.append(bytesRef.utf8ToString());
-        sb.append("]");
-        val = sb.toString();
+        if (sv.advanceExact(subid)) {
+          bytesRef = sv.binaryValue();
+          StringBuilder sb = new StringBuilder();
+          sb.append(NUM_TERMS_IN_FIELD).append(sv.getValueCount()).append(", ");
+          sb.append("value: [");
+          sb.append(bytesRef.utf8ToString());
+          sb.append("]");
+          val = sb.toString();
+        }
         break;
       }
       case SORTED_SET: {
         SortedSetDocValues sv = (SortedSetDocValues)docVals;
-        sv.setDocument(subid);
-        long nextOrd;
-        long count = sv.getValueCount();
-        StringBuilder sb = new StringBuilder();
-        sb.append(NUM_TERMS_IN_FIELD).append(count).append(", ");
-        sb.append("values: [");
-        boolean firstPass = true;
-        while ((nextOrd = sv.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
-          bytesRef = sv.lookupOrd(nextOrd);
-          if (!firstPass) {
-            sb.append(", ");
+        if (sv.advanceExact(subid)) {
+          long nextOrd;
+          long count = sv.getValueCount();
+          StringBuilder sb = new StringBuilder();
+          sb.append(NUM_TERMS_IN_FIELD).append(count).append(", ");
+          sb.append("values: [");
+          boolean firstPass = true;
+          while ((nextOrd = sv.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+            bytesRef = sv.lookupOrd(nextOrd);
+            if (!firstPass) {
+              sb.append(", ");
+            }
+            sb.append(bytesRef.utf8ToString());
+            firstPass = false;
           }
-          sb.append(bytesRef.utf8ToString());
-          firstPass = false;
+          sb.append("]");
+          val = sb.toString();
         }
-        sb.append("]");
-        val = sb.toString();
         break;
       }
       case SORTED_NUMERIC: {
         SortedNumericDocValues sv = (SortedNumericDocValues)docVals;
-        sv.setDocument(subid);        
-        int count = sv.count();
-        StringBuilder sb = new StringBuilder();
-        sb.append(NUM_TERMS_IN_FIELD).append(count).append(", ");
-        sb.append("values: [");
-        boolean firstPass = true;
-        for (int i = 0; i < count; ++i) {
-          long nextVal = sv.valueAt(i);
-          if (!firstPass) {
-            sb.append(", ");
+        if (sv.advanceExact(subid)) {
+          int count = sv.docValueCount();
+          StringBuilder sb = new StringBuilder();
+          sb.append(NUM_TERMS_IN_FIELD).append(count).append(", ");
+          sb.append("values: [");
+          boolean firstPass = true;
+          for (int i = 0; i < count; ++i) {
+            long nextVal = sv.nextValue();
+            if (!firstPass) {
+              sb.append(", ");
+            }
+            sb.append(String.valueOf(nextVal));
+            firstPass = false;
           }
-          sb.append(String.valueOf(nextVal));
-          firstPass = false;
-        }        
-        sb.append("]");
-        val = sb.toString();
+          sb.append("]");
+          val = sb.toString();
+        }
         break;
       }
       default:

src/main/java/io/dashbase/clue/commands/IndexTrimCommand.java

@@ -1,14 +1,14 @@
 package io.dashbase.clue.commands;
 
-import java.io.PrintStream;
-import java.util.Random;
-
+import io.dashbase.clue.ClueContext;
+import io.dashbase.clue.util.DocIdMatcher;
+import io.dashbase.clue.util.MatchSomeDocsQuery;
+import io.dashbase.clue.util.MatcherDocIdSetIterator;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.search.Query;
 
-import io.dashbase.clue.ClueContext;
-import io.dashbase.clue.util.MatchSomeDocsQuery;
+import java.io.PrintStream;
 
 public class IndexTrimCommand extends ClueCommand {
 
@@ -26,26 +26,9 @@ public String help() {
     return "trims the index, <TRIM PERCENTAGE>"; 
   }
 
-  private static Query buildDeleteQuery(final int percentToDelete) {
+  private static Query buildDeleteQuery(final int percentToDelete, int maxDoc) {
     assert percentToDelete >= 0 && percentToDelete <= 100;
-    final Random rand = new Random();
-    return new MatchSomeDocsQuery() {
-
-      @Override
-      public String toString(String field) {
-        return null;
-      }
-
-      @Override
-      protected boolean match(int docId) {
-        int guess = rand.nextInt(100);
-        if (guess < percentToDelete) {
-          return true;
-        }
-        return false;
-      }
-    };
-
+    return new MatchSomeDocsQuery(new MatcherDocIdSetIterator(DocIdMatcher.newRandomMatcher(percentToDelete), maxDoc));
   }
 
   @Override
@@ -65,7 +48,7 @@ public void execute(String[] args, PrintStream out) throws Exception {
     if (writer != null) {      
       IndexReader reader = ctx.getIndexReader();
 
-      writer.deleteDocuments(buildDeleteQuery(trimPercent));
+      writer.deleteDocuments(buildDeleteQuery(trimPercent, reader.maxDoc()));
       writer.commit();      
       ctx.refreshReader();
       reader = ctx.getIndexReader();

src/main/java/io/dashbase/clue/commands/InfoCommand.java

@@ -117,13 +117,11 @@ public void execute(String[] args, PrintStream out) throws Exception {
         LeafReader ar = leaf.reader();        
         FieldInfos fldInfos = ar.getFieldInfos();
         Iterator<FieldInfo> finfoIter = fldInfos.iterator();
-
-        Fields flds = ar.fields();
-
+
         while (finfoIter.hasNext()) {
           FieldInfo finfo = finfoIter.next();          
           Object[] data = fields.get(finfo.name);
-          Terms t = flds.terms(finfo.name);
+          Terms t = ar.terms(finfo.name);
           if (data == null) {
             data = new Object[2];
             LinkedList<Terms> termsList = new LinkedList<Terms>();
@@ -167,13 +165,12 @@ public void execute(String[] args, PrintStream out) throws Exception {
       out.println("num deleted docs:\t" + atomicReader.numDeletedDocs());
 
       FieldInfos fields = atomicReader.getFieldInfos();
-      Fields flds = atomicReader.fields();
 
       out.println("number of fields: " + fields.size());
 
       for (int i = 0; i < fields.size(); ++i) {
         FieldInfo finfo = fields.fieldInfo(i);
-        Terms te = flds.terms(finfo.name);
+        Terms te = atomicReader.terms(finfo.name);
         out.println("=================================== Field "+finfo.name+" ===================================");
         toString(new Object[] { finfo, Arrays.asList(te) }, out);
       }

src/main/java/io/dashbase/clue/commands/NormsCommand.java

@@ -33,7 +33,11 @@ private void showDocId(int docid, int docBase,
       PrintStream out, int segmentid) throws Exception {
     int subid = docid - docBase;
     if (docVals != null) {
-      String val = String.valueOf(docVals.get(subid));
+      String val = null;
+
+      if (docVals.advanceExact(subid)) {
+        val = String.valueOf(docVals.longValue());
+      }
 
       if (val == null) {
         out.println("cannot read norm for docid: " + docid + ", subid: " + subid);

src/main/java/io/dashbase/clue/commands/SaveUserCommitData.java

@@ -1,9 +1,7 @@
 package io.dashbase.clue.commands;
 
 import java.io.PrintStream;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.*;
 
 import org.apache.lucene.index.IndexWriter;
 
@@ -32,12 +30,14 @@ public void execute(String[] args, PrintStream out) throws Exception {
 			if (args.length != 2) {
 				throw new IllegalArgumentException("expected 2 arguments indicating key and value");
 			}
-			Map<String, String> commitData = writer.getCommitData();
-			if (commitData == null) {
-				commitData= new HashMap<String, String>();
+			Iterable<Map.Entry<String, String>> commitData = writer.getLiveCommitData();
+			HashMap<String, String> commitMap = new HashMap<>();
+			if (commitData != null) {
+				for (Map.Entry<String, String> entry : commitData) {
+					commitMap.put(entry.getKey(), entry.getValue());
+				}
 			}
-			commitData.put(args[0], args[1]);
-			writer.setCommitData(commitData);
+			writer.setLiveCommitData(commitData);
 			writer.commit();
 			ctx.refreshReader();
 			out.println("commit data: " + Arrays.toString(args) +" saved.");

src/main/java/io/dashbase/clue/commands/TermsCommand.java

@@ -79,7 +79,7 @@ public void execute(String[] args, PrintStream out) throws Exception {
     for (LeafReaderContext leaf : leaves){
       LeafReader atomicReader = leaf.reader();
 
-      Terms terms = atomicReader.fields().terms(field);
+      Terms terms = atomicReader.terms(field);
 
       if (terms == null) {
         continue;

src/main/java/io/dashbase/clue/util/DocIdMatcher.java

@@ -0,0 +1,24 @@
+package io.dashbase.clue.util;
+
+import java.util.Random;
+
+public interface DocIdMatcher {
+    boolean match(int docid);
+
+    static DocIdMatcher newRandomMatcher(int percent) {
+
+        return new DocIdMatcher() {
+            Random rand = new Random();
+
+            @Override
+            public boolean match(int docid) {
+
+                int guess = rand.nextInt(100);
+                if (guess < percent) {
+                    return true;
+                }
+                return false;
+            }
+        };
+    }
+}