release:1.18.2, keep version same as leveldbjni

 1. add batch mode for recovery from manifest

.github/workflows/maven.yml

@@ -0,0 +1,26 @@
+# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time
+# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+
+name: Java CI with Maven
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up JDK 11
+      uses: actions/setup-java@v3
+      with:
+        java-version: '11'
+        distribution: 'temurin'
+        cache: maven
+    - name: Build with Maven
+      run: mvn clean package -P release

leveldb-api/pom.xml

@@ -3,9 +3,9 @@
     <modelVersion>4.0.0</modelVersion>
 
     <parent>
-        <groupId>org.iq80.leveldb</groupId>
+        <groupId>com.halibobor</groupId>
         <artifactId>leveldb-project</artifactId>
-        <version>0.13-SNAPSHOT</version>
+        <version>1.18.2</version>
     </parent>
 
     <artifactId>leveldb-api</artifactId>

leveldb-api/src/license/LICENSE-HEADER.txt

@@ -0,0 +1,15 @@
+Copyright (C) 2011 the original author or authors.
+See the notice.md file distributed with this work for additional
+information regarding copyright ownership.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

leveldb-api/src/main/java/org/iq80/leveldb/Options.java

@@ -33,6 +33,8 @@ public class Options
     private DBComparator comparator;
     private Logger logger;
     private long cacheSize;
+    private int maxBatchSize = 80_000;
+    private int maxManifestSize = 0; //M
 
     static void checkArgNotNull(Object value, String name)
     {
@@ -173,4 +175,33 @@ public Options paranoidChecks(boolean paranoidChecks)
         this.paranoidChecks = paranoidChecks;
         return this;
     }
+
+    public int maxBatchSize()
+    {
+        return maxBatchSize;
+    }
+
+    public Options maxBatchSize(int maxBatchSize)
+    {
+        if (maxBatchSize < 0) {
+            maxBatchSize = Integer.MAX_VALUE;
+        }
+        this.maxBatchSize = maxBatchSize;
+        return this;
+    }
+
+    public int maxManifestSize()
+    {
+        return maxManifestSize;
+    }
+
+    public Options maxManifestSize(int maxManifestSize)
+    {
+        if (maxManifestSize < 0) {
+            maxManifestSize = -1;
+            maxBatchSize(-1);
+        }
+        this.maxManifestSize = maxManifestSize;
+        return this;
+    }
 }

leveldb-benchmark/pom.xml

@@ -19,9 +19,9 @@
     <modelVersion>4.0.0</modelVersion>
 
     <parent>
+        <groupId>com.halibobor</groupId>
         <artifactId>leveldb-project</artifactId>
-        <groupId>org.iq80.leveldb</groupId>
-        <version>0.13-SNAPSHOT</version>
+        <version>1.18.2</version>
     </parent>
 
     <artifactId>leveldb-benchmark</artifactId>
@@ -34,11 +34,11 @@
 
     <dependencies>
         <dependency>
-            <groupId>org.iq80.leveldb</groupId>
+            <groupId>com.halibobor</groupId>
             <artifactId>leveldb-api</artifactId>
         </dependency>
         <dependency>
-            <groupId>org.iq80.leveldb</groupId>
+            <groupId>com.halibobor</groupId>
             <artifactId>leveldb</artifactId>
         </dependency>
         <dependency>

leveldb/pom.xml

@@ -3,9 +3,9 @@
     <modelVersion>4.0.0</modelVersion>
 
     <parent>
-        <groupId>org.iq80.leveldb</groupId>
+        <groupId>com.halibobor</groupId>
         <artifactId>leveldb-project</artifactId>
-        <version>0.13-SNAPSHOT</version>
+        <version>1.18.2</version>
     </parent>
 
     <artifactId>leveldb</artifactId>
@@ -20,14 +20,14 @@
     <dependencies>
 
         <dependency>
-            <groupId>org.iq80.leveldb</groupId>
+            <groupId>com.halibobor</groupId>
             <artifactId>leveldb-api</artifactId>
         </dependency>
 
         <dependency>
             <groupId>org.xerial.snappy</groupId>
             <artifactId>snappy-java</artifactId>
-            <version>1.1.2.6 </version>
+            <version>1.1.2.6</version>
             <optional>true</optional>
         </dependency>
 
@@ -60,7 +60,13 @@
         <dependency>
             <groupId>org.fusesource.leveldbjni</groupId>
             <artifactId>leveldbjni</artifactId>
-            <version>1.1</version>
+            <version>1.8</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.iq80.leveldb</groupId>
+                    <artifactId>leveldb-api</artifactId>
+                </exclusion>
+            </exclusions>
             <scope>test</scope>
         </dependency>
 

leveldb/src/main/java/org/iq80/leveldb/impl/DbImpl.java

@@ -177,7 +177,7 @@ public void uncaughtException(Thread t, Throwable e)
                 checkArgument(!options.errorIfExists(), "Database '%s' exists and the error if exists option is enabled", databaseDir);
             }
 
-            versions = new VersionSet(databaseDir, tableCache, internalKeyComparator);
+            versions = new VersionSet(databaseDir, tableCache, options, internalKeyComparator);
 
             // load  (and recover) current version
             versions.recover();

leveldb/src/main/java/org/iq80/leveldb/impl/VersionSet.java

@@ -24,6 +24,7 @@
 import com.google.common.collect.MapMaker;
 import com.google.common.collect.Maps;
 import com.google.common.io.Files;
+import org.iq80.leveldb.Options;
 import org.iq80.leveldb.table.UserComparator;
 import org.iq80.leveldb.util.InternalIterator;
 import org.iq80.leveldb.util.Level0Iterator;
@@ -43,6 +44,7 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.SortedMap;
 import java.util.SortedSet;
 import java.util.TreeMap;
 import java.util.TreeSet;
@@ -80,12 +82,14 @@ public class VersionSet
 
     private LogWriter descriptorLog;
     private final Map<Integer, InternalKey> compactPointers = new TreeMap<>();
+    private final Options options;
 
-    public VersionSet(File databaseDir, TableCache tableCache, InternalKeyComparator internalKeyComparator)
-            throws IOException
+    public VersionSet(File databaseDir, TableCache tableCache, Options options, InternalKeyComparator internalKeyComparator)
+        throws IOException
     {
         this.databaseDir = databaseDir;
         this.tableCache = tableCache;
+        this.options = options;
         this.internalKeyComparator = internalKeyComparator;
         appendVersion(new Version(this));
 
@@ -353,9 +357,10 @@ public void recover()
             Builder builder = new Builder(this, current);
 
             LogReader reader = new LogReader(fileChannel, throwExceptionMonitor(), true, 0);
+            VersionEdit edit;
             for (Slice record = reader.readRecord(); record != null; record = reader.readRecord()) {
                 // read version edit
-                VersionEdit edit = new VersionEdit(record);
+                edit = new VersionEdit(record);
 
                 // verify comparator
                 // todo implement user comparator
@@ -366,7 +371,16 @@ public void recover()
 
                 // apply edit
                 builder.apply(edit);
+                if (builder.batchSize > options.maxBatchSize()) {
+                    Version version = new Version(this);
+                    builder.saveTo(version);
+
+                    // Install recovered version
+                    finalizeVersion(version);
 
+                    appendVersion(version);
+                    builder = new Builder(this, current);
+                }
                 // save edit values for verification below
                 logNumber = coalesce(edit.getLogNumber(), logNumber);
                 prevLogNumber = coalesce(edit.getPreviousLogNumber(), prevLogNumber);
@@ -687,6 +701,7 @@ private static class Builder
         private final VersionSet versionSet;
         private final Version baseVersion;
         private final List<LevelState> levels;
+        private int batchSize;
 
         private Builder(VersionSet versionSet, Version baseVersion)
         {
@@ -716,7 +731,11 @@ public void apply(VersionEdit edit)
                 Integer level = entry.getKey();
                 Long fileNumber = entry.getValue();
                 levels.get(level).deletedFiles.add(fileNumber);
-                // todo missing update to addedFiles?
+                batchSize++;
+                // missing update to addedFiles for open db to release resource
+                if (levels.get(level).addedFilesMap.remove(fileNumber) != null) {
+                    batchSize--;
+                }
             }
 
             // Add new files
@@ -743,8 +762,11 @@ public void apply(VersionEdit edit)
                 }
                 fileMetaData.setAllowedSeeks(allowedSeeks);
 
-                levels.get(level).deletedFiles.remove(fileMetaData.getNumber());
-                levels.get(level).addedFiles.add(fileMetaData);
+                if (levels.get(level).deletedFiles.remove(fileMetaData.getNumber())) {
+                    batchSize--;
+                }
+                levels.get(level).addedFilesMap.put(fileMetaData.getNumber(), fileMetaData);
+                batchSize++;
             }
         }
 
@@ -763,6 +785,7 @@ public void saveTo(Version version)
                 if (baseFiles == null) {
                     baseFiles = ImmutableList.of();
                 }
+                levels.get(level).addedFiles.addAll(levels.get(level).addedFilesMap.values());
                 SortedSet<FileMetaData> addedFiles = levels.get(level).addedFiles;
                 if (addedFiles == null) {
                     addedFiles = ImmutableSortedSet.of();
@@ -833,11 +856,13 @@ public int compare(FileMetaData f1, FileMetaData f2)
         private static class LevelState
         {
             private final SortedSet<FileMetaData> addedFiles;
+            private final SortedMap<Long, FileMetaData> addedFilesMap;
             private final Set<Long> deletedFiles = new HashSet<Long>();
 
             public LevelState(InternalKeyComparator internalKeyComparator)
             {
                 addedFiles = new TreeSet<FileMetaData>(new FileMetaDataBySmallestKey(internalKeyComparator));
+                addedFilesMap = new TreeMap<>();
             }
 
             @Override

leveldb/src/test/java/org/iq80/leveldb/impl/DbImplTest.java

@@ -20,6 +20,9 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.primitives.Ints;
 import com.google.common.primitives.UnsignedBytes;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import org.iq80.leveldb.CompressionType;
 import org.iq80.leveldb.DB;
 import org.iq80.leveldb.DBComparator;
 import org.iq80.leveldb.DBIterator;
@@ -49,6 +52,7 @@
 import static com.google.common.collect.Maps.immutableEntry;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.util.Arrays.asList;
+import static org.iq80.leveldb.impl.Iq80DBFactory.factory;
 import static org.iq80.leveldb.CompressionType.NONE;
 import static org.iq80.leveldb.impl.DbConstants.NUM_LEVELS;
 import static org.iq80.leveldb.table.BlockHelper.afterString;
@@ -862,6 +866,36 @@ public void testCustomComparator()
         assertFalse(seekingIterator.hasNext());
     }
 
+    @Test
+    // test for local
+    public void testHugeManifest()
+    {
+        DB database;
+        Path db = Paths.get("/tmp/leveldb", "account");
+        File file = db.toFile();
+        org.iq80.leveldb.Options dbOptions = new org.iq80.leveldb.Options();
+        dbOptions.createIfMissing(true);
+        dbOptions.paranoidChecks(true);
+        dbOptions.verifyChecksums(true);
+        dbOptions.maxBatchSize(64_000);
+        dbOptions.compressionType(CompressionType.SNAPPY);
+        dbOptions.blockSize(4 * 1024);
+        dbOptions.writeBufferSize(10 * 1024 * 1024);
+        dbOptions.cacheSize(10 * 1024 * 1024L);
+        dbOptions.maxOpenFiles(100_000);
+        dbOptions.maxManifestSize(256);
+        try {
+            long s = System.currentTimeMillis();
+            database = factory.open(file, dbOptions);
+            long e = System.currentTimeMillis();
+            System.out.println("open cost :" + (e - s));
+            database.close();
+        }
+        catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
     @SafeVarargs
     private final void testDb(DbStringWrapper db, Entry<String, String>... entries)
             throws IOException