From bec24e9c18c65591d65aa434e037d85ec2ef7e53 Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Fri, 18 Nov 2022 11:55:11 +0800 Subject: [PATCH] release:1.18.2, keep version same as leveldbjni 1. add batch mode for recovery from manifest --- .github/workflows/maven.yml | 26 ++++ leveldb-api/pom.xml | 4 +- leveldb-api/src/license/LICENSE-HEADER.txt | 15 ++ .../main/java/org/iq80/leveldb/Options.java | 31 ++++ leveldb-benchmark/pom.xml | 8 +- leveldb/pom.xml | 16 +- .../java/org/iq80/leveldb/impl/DbImpl.java | 2 +- .../org/iq80/leveldb/impl/VersionSet.java | 37 ++++- .../org/iq80/leveldb/impl/DbImplTest.java | 34 +++++ pom.xml | 144 +++++++++++++++++- 10 files changed, 291 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/maven.yml create mode 100644 leveldb-api/src/license/LICENSE-HEADER.txt diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml new file mode 100644 index 00000000..dbf1d0a2 --- /dev/null +++ b/.github/workflows/maven.yml @@ -0,0 +1,26 @@ +# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven + +name: Java CI with Maven + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + cache: maven + - name: Build with Maven + run: mvn clean package -P release diff --git a/leveldb-api/pom.xml b/leveldb-api/pom.xml index 32cc1715..8901d495 100644 --- a/leveldb-api/pom.xml +++ b/leveldb-api/pom.xml @@ -3,9 +3,9 @@ 4.0.0 - org.iq80.leveldb + com.halibobor leveldb-project - 0.13-SNAPSHOT + 1.18.2 leveldb-api diff --git a/leveldb-api/src/license/LICENSE-HEADER.txt b/leveldb-api/src/license/LICENSE-HEADER.txt new file mode 100644 index 00000000..1de76d19 --- /dev/null +++ b/leveldb-api/src/license/LICENSE-HEADER.txt @@ -0,0 +1,15 @@ +Copyright (C) 2011 the original author or authors. +See the notice.md file distributed with this work for additional +information regarding copyright ownership. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/leveldb-api/src/main/java/org/iq80/leveldb/Options.java b/leveldb-api/src/main/java/org/iq80/leveldb/Options.java index 630753cc..b3bd03f2 100644 --- a/leveldb-api/src/main/java/org/iq80/leveldb/Options.java +++ b/leveldb-api/src/main/java/org/iq80/leveldb/Options.java @@ -33,6 +33,8 @@ public class Options private DBComparator comparator; private Logger logger; private long cacheSize; + private int maxBatchSize = 80_000; + private int maxManifestSize = 0; //M static void checkArgNotNull(Object value, String name) { @@ -173,4 +175,33 @@ public Options paranoidChecks(boolean paranoidChecks) this.paranoidChecks = paranoidChecks; return this; } + + public int maxBatchSize() + { + return maxBatchSize; + } + + public Options maxBatchSize(int maxBatchSize) + { + if (maxBatchSize < 0) { + maxBatchSize = Integer.MAX_VALUE; + } + this.maxBatchSize = maxBatchSize; + return this; + } + + public int maxManifestSize() + { + return maxManifestSize; + } + + public Options maxManifestSize(int maxManifestSize) + { + if (maxManifestSize < 0) { + maxManifestSize = -1; + maxBatchSize(-1); + } + this.maxManifestSize = maxManifestSize; + return this; + } } diff --git a/leveldb-benchmark/pom.xml b/leveldb-benchmark/pom.xml index 0262f0b8..481e2330 100644 --- a/leveldb-benchmark/pom.xml +++ b/leveldb-benchmark/pom.xml @@ -19,9 +19,9 @@ 4.0.0 + com.halibobor leveldb-project - org.iq80.leveldb - 0.13-SNAPSHOT + 1.18.2 leveldb-benchmark @@ -34,11 +34,11 @@ - org.iq80.leveldb + com.halibobor leveldb-api - org.iq80.leveldb + com.halibobor leveldb diff --git a/leveldb/pom.xml b/leveldb/pom.xml index ebd55194..62a9999c 100644 --- a/leveldb/pom.xml +++ b/leveldb/pom.xml @@ -3,9 +3,9 @@ 4.0.0 - org.iq80.leveldb + com.halibobor leveldb-project - 0.13-SNAPSHOT + 1.18.2 leveldb @@ -20,14 +20,14 @@ - org.iq80.leveldb + com.halibobor leveldb-api org.xerial.snappy snappy-java - 1.1.2.6 + 1.1.2.6 true @@ -60,7 +60,13 @@ org.fusesource.leveldbjni leveldbjni - 1.1 + 1.8 + + + org.iq80.leveldb + leveldb-api + + test diff --git a/leveldb/src/main/java/org/iq80/leveldb/impl/DbImpl.java b/leveldb/src/main/java/org/iq80/leveldb/impl/DbImpl.java index d0cafac4..426fd998 100755 --- a/leveldb/src/main/java/org/iq80/leveldb/impl/DbImpl.java +++ b/leveldb/src/main/java/org/iq80/leveldb/impl/DbImpl.java @@ -177,7 +177,7 @@ public void uncaughtException(Thread t, Throwable e) checkArgument(!options.errorIfExists(), "Database '%s' exists and the error if exists option is enabled", databaseDir); } - versions = new VersionSet(databaseDir, tableCache, internalKeyComparator); + versions = new VersionSet(databaseDir, tableCache, options, internalKeyComparator); // load (and recover) current version versions.recover(); diff --git a/leveldb/src/main/java/org/iq80/leveldb/impl/VersionSet.java b/leveldb/src/main/java/org/iq80/leveldb/impl/VersionSet.java index 22f9bc57..a7464944 100755 --- a/leveldb/src/main/java/org/iq80/leveldb/impl/VersionSet.java +++ b/leveldb/src/main/java/org/iq80/leveldb/impl/VersionSet.java @@ -24,6 +24,7 @@ import com.google.common.collect.MapMaker; import com.google.common.collect.Maps; import com.google.common.io.Files; +import org.iq80.leveldb.Options; import org.iq80.leveldb.table.UserComparator; import org.iq80.leveldb.util.InternalIterator; import org.iq80.leveldb.util.Level0Iterator; @@ -43,6 +44,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.SortedMap; import java.util.SortedSet; import java.util.TreeMap; import java.util.TreeSet; @@ -80,12 +82,14 @@ public class VersionSet private LogWriter descriptorLog; private final Map compactPointers = new TreeMap<>(); + private final Options options; - public VersionSet(File databaseDir, TableCache tableCache, InternalKeyComparator internalKeyComparator) - throws IOException + public VersionSet(File databaseDir, TableCache tableCache, Options options, InternalKeyComparator internalKeyComparator) + throws IOException { this.databaseDir = databaseDir; this.tableCache = tableCache; + this.options = options; this.internalKeyComparator = internalKeyComparator; appendVersion(new Version(this)); @@ -353,9 +357,10 @@ public void recover() Builder builder = new Builder(this, current); LogReader reader = new LogReader(fileChannel, throwExceptionMonitor(), true, 0); + VersionEdit edit; for (Slice record = reader.readRecord(); record != null; record = reader.readRecord()) { // read version edit - VersionEdit edit = new VersionEdit(record); + edit = new VersionEdit(record); // verify comparator // todo implement user comparator @@ -366,7 +371,16 @@ public void recover() // apply edit builder.apply(edit); + if (builder.batchSize > options.maxBatchSize()) { + Version version = new Version(this); + builder.saveTo(version); + + // Install recovered version + finalizeVersion(version); + appendVersion(version); + builder = new Builder(this, current); + } // save edit values for verification below logNumber = coalesce(edit.getLogNumber(), logNumber); prevLogNumber = coalesce(edit.getPreviousLogNumber(), prevLogNumber); @@ -687,6 +701,7 @@ private static class Builder private final VersionSet versionSet; private final Version baseVersion; private final List levels; + private int batchSize; private Builder(VersionSet versionSet, Version baseVersion) { @@ -716,7 +731,11 @@ public void apply(VersionEdit edit) Integer level = entry.getKey(); Long fileNumber = entry.getValue(); levels.get(level).deletedFiles.add(fileNumber); - // todo missing update to addedFiles? + batchSize++; + // missing update to addedFiles for open db to release resource + if (levels.get(level).addedFilesMap.remove(fileNumber) != null) { + batchSize--; + } } // Add new files @@ -743,8 +762,11 @@ public void apply(VersionEdit edit) } fileMetaData.setAllowedSeeks(allowedSeeks); - levels.get(level).deletedFiles.remove(fileMetaData.getNumber()); - levels.get(level).addedFiles.add(fileMetaData); + if (levels.get(level).deletedFiles.remove(fileMetaData.getNumber())) { + batchSize--; + } + levels.get(level).addedFilesMap.put(fileMetaData.getNumber(), fileMetaData); + batchSize++; } } @@ -763,6 +785,7 @@ public void saveTo(Version version) if (baseFiles == null) { baseFiles = ImmutableList.of(); } + levels.get(level).addedFiles.addAll(levels.get(level).addedFilesMap.values()); SortedSet addedFiles = levels.get(level).addedFiles; if (addedFiles == null) { addedFiles = ImmutableSortedSet.of(); @@ -833,11 +856,13 @@ public int compare(FileMetaData f1, FileMetaData f2) private static class LevelState { private final SortedSet addedFiles; + private final SortedMap addedFilesMap; private final Set deletedFiles = new HashSet(); public LevelState(InternalKeyComparator internalKeyComparator) { addedFiles = new TreeSet(new FileMetaDataBySmallestKey(internalKeyComparator)); + addedFilesMap = new TreeMap<>(); } @Override diff --git a/leveldb/src/test/java/org/iq80/leveldb/impl/DbImplTest.java b/leveldb/src/test/java/org/iq80/leveldb/impl/DbImplTest.java index cc833cbe..3e04670c 100644 --- a/leveldb/src/test/java/org/iq80/leveldb/impl/DbImplTest.java +++ b/leveldb/src/test/java/org/iq80/leveldb/impl/DbImplTest.java @@ -20,6 +20,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.primitives.Ints; import com.google.common.primitives.UnsignedBytes; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.iq80.leveldb.CompressionType; import org.iq80.leveldb.DB; import org.iq80.leveldb.DBComparator; import org.iq80.leveldb.DBIterator; @@ -49,6 +52,7 @@ import static com.google.common.collect.Maps.immutableEntry; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Arrays.asList; +import static org.iq80.leveldb.impl.Iq80DBFactory.factory; import static org.iq80.leveldb.CompressionType.NONE; import static org.iq80.leveldb.impl.DbConstants.NUM_LEVELS; import static org.iq80.leveldb.table.BlockHelper.afterString; @@ -862,6 +866,36 @@ public void testCustomComparator() assertFalse(seekingIterator.hasNext()); } + @Test + // test for local + public void testHugeManifest() + { + DB database; + Path db = Paths.get("/tmp/leveldb", "account"); + File file = db.toFile(); + org.iq80.leveldb.Options dbOptions = new org.iq80.leveldb.Options(); + dbOptions.createIfMissing(true); + dbOptions.paranoidChecks(true); + dbOptions.verifyChecksums(true); + dbOptions.maxBatchSize(64_000); + dbOptions.compressionType(CompressionType.SNAPPY); + dbOptions.blockSize(4 * 1024); + dbOptions.writeBufferSize(10 * 1024 * 1024); + dbOptions.cacheSize(10 * 1024 * 1024L); + dbOptions.maxOpenFiles(100_000); + dbOptions.maxManifestSize(256); + try { + long s = System.currentTimeMillis(); + database = factory.open(file, dbOptions); + long e = System.currentTimeMillis(); + System.out.println("open cost :" + (e - s)); + database.close(); + } + catch (Exception e) { + e.printStackTrace(); + } + } + @SafeVarargs private final void testDb(DbStringWrapper db, Entry... entries) throws IOException diff --git a/pom.xml b/pom.xml index 223d88e2..e11eecb1 100644 --- a/pom.xml +++ b/pom.xml @@ -8,13 +8,13 @@ 95 - org.iq80.leveldb + com.halibobor leveldb-project - 0.13-SNAPSHOT + 1.18.2 pom Port of LevelDB to Java - http://github.com/dain/leveldb + http://github.com/halibobor/leveldb-java leveldb-api @@ -45,12 +45,21 @@ http://hiramchirino.com -5 + + halibobo + halibobo1205 + halibobo1205@gmail.com + + Developer + + +8 + - scm:git:git://github.com/dain/leveldb.git - scm:git:git@github.com:dain/leveldb.git - http://github.com/dain/leveldb/tree/master + scm:https://github.com/halibobor/leveldb-java.git + scm:https://github.com/halibobor/leveldb-java.git + https://github.com/halibobor/leveldb-java/tree/master HEAD @@ -65,18 +74,20 @@ -missing src/license/LICENSE-HEADER.txt ${air.main.basedir}/src/checkstyle/checks.xml + UTF-8 + 0xEF7F2D6C - org.iq80.leveldb + com.halibobor leveldb-api ${project.version} - org.iq80.leveldb + com.halibobor leveldb ${project.version} @@ -108,4 +119,121 @@ + + + + + release + + + + + org.apache.maven.plugins + maven-gpg-plugin + 3.0.1 + + + sign-artifacts + verify + + sign + + + ${gpg.keyname} + ${gpg.keyname} + + + + + + + org.apache.maven.plugins + maven-source-plugin + + + attach-sources + + jar-no-fork + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + ${project.build.sourceEncoding} + + + + attach-javadocs + + jar + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + true + true + UTF-8 + false + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.7 + true + + ossrh + https://s01.oss.sonatype.org/ + true + + + + + + + + + + enforce-no-snapshots + + + + org.apache.maven.plugins + maven-enforcer-plugin + 1.0 + + + enforce-no-snapshots + enforce + + + + No Snapshots Allowed! + + + No Snapshots Allowed! + + + + + + + + + +