Skip to content

Commit

Permalink
Improved performance of IOUtils.contentEquals(InputStream, InputStream).
Browse files Browse the repository at this point in the history 
This is based on the PR #118 by
XenoAmess but only for this one method.
  • Loading branch information
Gary Gregory committed Jan 24, 2021
1 parent b6bca11 commit 5bed26f
Show file tree
Hide file tree
Showing 6 changed files with 781 additions and 13 deletions.
3 changes: 3 additions & 0 deletions src/changes/changes.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,9 @@ The <action> type attribute can be add,update,fix,remove.
<action dev="ggregory" type="update" due-to="Dependabot">
Bump jimfs from 1.1 to 1.2 #183.
</action>
<action dev="ggregory" type="update" due-to="XenoAmess, Gary Gregory">
Improved performance of IOUtils.contentEquals(InputStream, InputStream).
</action>
</release>
<!-- The release date is the date RC is cut -->
<release version="2.8.0" date="2020-09-05" description="Java 8 required.">
Expand Down
49 changes: 36 additions & 13 deletions src/main/java/org/apache/commons/io/IOUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -744,26 +744,49 @@ public static long consume(final InputStream input)
* @throws NullPointerException if either input is null
* @throws IOException if an I/O error occurs
*/
@SuppressWarnings("resource")
public static boolean contentEquals(final InputStream input1, final InputStream input2)
throws IOException {
public static boolean contentEquals(final InputStream input1, final InputStream input2) throws IOException {
// Before making any changes, please test with
// org.apache.commons.io.jmh.IOUtilsContentEqualsInputStreamsBenchmark
if (input1 == input2) {
return true;
}
if (input1 == null ^ input2 == null) {
if (input1 == null || input2 == null) {
return false;
}
final BufferedInputStream bufferedInput1 = buffer(input1);
final BufferedInputStream bufferedInput2 = buffer(input2);
int ch = bufferedInput1.read();
while (EOF != ch) {
final int ch2 = bufferedInput2.read();
if (ch != ch2) {
return false;

final byte[] array1 = new byte[DEFAULT_BUFFER_SIZE];
final byte[] array2 = new byte[DEFAULT_BUFFER_SIZE];
int pos1;
int pos2;
int count1;
int count2;
while (true) {
pos1 = 0;
pos2 = 0;
for (int index = 0; index < DEFAULT_BUFFER_SIZE; index++) {
if (pos1 == index) {
do {
count1 = input1.read(array1, pos1, DEFAULT_BUFFER_SIZE - pos1);
} while (count1 == 0);
if (count1 == EOF) {
return pos2 == index && input2.read() == EOF;
}
pos1 += count1;
}
if (pos2 == index) {
do {
count2 = input2.read(array2, pos2, DEFAULT_BUFFER_SIZE - pos2);
} while (count2 == 0);
if (count2 == EOF) {
return pos1 == index && input1.read() == EOF;
}
pos2 += count2;
}
if (array1[index] != array2[index]) {
return false;
}
}
ch = bufferedInput1.read();
}
return bufferedInput2.read() == EOF;
}

/**
Expand Down
8 changes: 8 additions & 0 deletions src/test/java/org/apache/commons/io/IOUtilsTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -548,6 +548,14 @@ public void testContentEquals_InputStream_InputStream() throws Exception {
new ByteArrayInputStream(bytes2XDefaultA2)));
assertTrue(IOUtils.contentEquals(new ByteArrayInputStream(bytes2XDefaultA),
new ByteArrayInputStream(bytes2XDefaultA)));
// FileInputStream a bit more than 16 k.
try (
final FileInputStream input1 = new FileInputStream(
"src/test/resources/org/apache/commons/io/abitmorethan16k.txt");
final FileInputStream input2 = new FileInputStream(
"src/test/resources/org/apache/commons/io/abitmorethan16kcopy.txt")) {
assertTrue(IOUtils.contentEquals(input1, input1));
}
}

@Test
Expand Down
238 changes: 238 additions & 0 deletions src/test/java/org/apache/commons/io/jmh/IOUtilsContentEqualsInputStreamsBenchmark.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,238 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.commons.io.jmh;

import static org.apache.commons.io.IOUtils.DEFAULT_BUFFER_SIZE;
import static org.apache.commons.io.IOUtils.EOF;
import static org.apache.commons.io.IOUtils.buffer;

import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.concurrent.TimeUnit;

import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;

/**
* Test different implementations of {@link IOUtils#contentEquals(InputStream, InputStream)}.
*
* <pre>
* Benchmark Mode Cnt Score Error Units
* IOUtilsContentEqualsInputStreamsBenchmark.testFileCurrent avgt 5 1518342.821 ▒ 201890.705 ns/op
* IOUtilsContentEqualsInputStreamsBenchmark.testFilePr118 avgt 5 1578606.938 ▒ 66980.718 ns/op
* IOUtilsContentEqualsInputStreamsBenchmark.testFileRelease_2_8_0 avgt 5 2439163.068 ▒ 265765.294 ns/op
* IOUtilsContentEqualsInputStreamsBenchmark.testStringCurrent avgt 5 10389834700.000 ▒ 330301175.219 ns/op
* IOUtilsContentEqualsInputStreamsBenchmark.testStringPr118 avgt 5 10890915400.000 ▒ 3251289634.067 ns/op
* IOUtilsContentEqualsInputStreamsBenchmark.testStringRelease_2_8_0 avgt 5 12522802960.000 ▒ 111147669.527 ns/op
* </pre>
*/
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
@State(Scope.Thread)
@Warmup(iterations = 5, time = 10, timeUnit = TimeUnit.SECONDS)
@Measurement(iterations = 5, time = 10, timeUnit = TimeUnit.SECONDS)
@Fork(value = 1, jvmArgs = {"-server"})
public class IOUtilsContentEqualsInputStreamsBenchmark {

private static final String TEST_PATH_A = "/org/apache/commons/io/testfileBOM.xml";
private static final String TEST_PATH_16K_A = "/org/apache/commons/io/abitmorethan16k.txt";
private static final String TEST_PATH_16K_A_COPY = "/org/apache/commons/io/abitmorethan16kcopy.txt";
private static final String TEST_PATH_B = "/org/apache/commons/io/testfileNoBOM.xml";
private static final Charset DEFAULT_CHARSET = Charset.defaultCharset();
static String[] STRINGS = new String[5];

static {
STRINGS[0] = StringUtils.repeat("ab", 1 << 24);
STRINGS[1] = STRINGS[0] + 'c';
STRINGS[2] = STRINGS[0] + 'd';
STRINGS[3] = StringUtils.repeat("ab\rab\n", 1 << 24);
STRINGS[4] = StringUtils.repeat("ab\r\nab\r", 1 << 24);
}

static String SPECIAL_CASE_STRING_0 = StringUtils.repeat(StringUtils.repeat("ab", 1 << 24) + '\n', 2);
static String SPECIAL_CASE_STRING_1 = StringUtils.repeat(StringUtils.repeat("cd", 1 << 24) + '\n', 2);

@SuppressWarnings("resource")
public static boolean contentEquals_release_2_8_0(final InputStream input1, final InputStream input2)
throws IOException {
if (input1 == input2) {
return true;
}
if (input1 == null ^ input2 == null) {
return false;
}
final BufferedInputStream bufferedInput1 = buffer(input1);
final BufferedInputStream bufferedInput2 = buffer(input2);
int ch = bufferedInput1.read();
while (EOF != ch) {
final int ch2 = bufferedInput2.read();
if (ch != ch2) {
return false;
}
ch = bufferedInput1.read();
}
return bufferedInput2.read() == EOF;

}

public static boolean contentEqualsPr118(final InputStream input1, final InputStream input2) throws IOException {
if (input1 == input2) {
return true;
}
if (input1 == null || input2 == null) {
return false;
}

final byte[] array1 = new byte[DEFAULT_BUFFER_SIZE];
final byte[] array2 = new byte[DEFAULT_BUFFER_SIZE];
int pos1;
int pos2;
int count1;
int count2;
while (true) {
pos1 = 0;
pos2 = 0;
for (int index = 0; index < DEFAULT_BUFFER_SIZE; index++) {
if (pos1 == index) {
do {
count1 = input1.read(array1, pos1, DEFAULT_BUFFER_SIZE - pos1);
} while (count1 == 0);
if (count1 == EOF) {
return pos2 == index && input2.read() == EOF;
}
pos1 += count1;
}
if (pos2 == index) {
do {
count2 = input2.read(array2, pos2, DEFAULT_BUFFER_SIZE - pos2);
} while (count2 == 0);
if (count2 == EOF) {
return pos1 == index && input1.read() == EOF;
}
pos2 += count2;
}
if (array1[index] != array2[index]) {
return false;
}
}
}
}

@Benchmark
public boolean[] testFileCurrent() throws IOException {
final boolean[] res = new boolean[3];
try (InputStream inputStream1 = this.getClass().getResourceAsStream(TEST_PATH_A);
InputStream inputStream2 = this.getClass().getResourceAsStream(TEST_PATH_B)) {
res[0] = IOUtils.contentEquals(inputStream1, inputStream1);
}
try (InputStream inputStream1 = this.getClass().getResourceAsStream(TEST_PATH_A);
InputStream inputStream2 = this.getClass().getResourceAsStream(TEST_PATH_A);) {
res[1] = IOUtils.contentEquals(inputStream1, inputStream2);
}
try (InputStream inputStream1 = this.getClass().getResourceAsStream(TEST_PATH_16K_A);
InputStream inputStream2 = this.getClass().getResourceAsStream(TEST_PATH_16K_A_COPY);) {
res[2] = IOUtils.contentEquals(inputStream1, inputStream2);
}
return res;
}

@Benchmark
public boolean[] testFilePr118() throws IOException {
final boolean[] res = new boolean[3];
try (InputStream inputStream1 = this.getClass().getResourceAsStream(TEST_PATH_A);
InputStream inputStream2 = this.getClass().getResourceAsStream(TEST_PATH_B)) {
res[0] = contentEqualsPr118(inputStream1, inputStream1);
}
try (InputStream inputStream1 = this.getClass().getResourceAsStream(TEST_PATH_A);
InputStream inputStream2 = this.getClass().getResourceAsStream(TEST_PATH_A);) {
res[1] = contentEqualsPr118(inputStream1, inputStream2);
}
try (InputStream inputStream1 = this.getClass().getResourceAsStream(TEST_PATH_16K_A);
InputStream inputStream2 = this.getClass().getResourceAsStream(TEST_PATH_16K_A_COPY);) {
res[2] = contentEqualsPr118(inputStream1, inputStream2);
}
return res;
}

@Benchmark
public boolean[] testFileRelease_2_8_0() throws IOException {
final boolean[] res = new boolean[3];
try (InputStream inputStream1 = this.getClass().getResourceAsStream(TEST_PATH_A);
InputStream inputStream2 = this.getClass().getResourceAsStream(TEST_PATH_B)) {
res[0] = contentEquals_release_2_8_0(inputStream1, inputStream1);
}
try (InputStream inputStream1 = this.getClass().getResourceAsStream(TEST_PATH_A);
InputStream inputStream2 = this.getClass().getResourceAsStream(TEST_PATH_A);) {
res[1] = contentEquals_release_2_8_0(inputStream1, inputStream2);
}
try (InputStream inputStream1 = this.getClass().getResourceAsStream(TEST_PATH_16K_A);
InputStream inputStream2 = this.getClass().getResourceAsStream(TEST_PATH_16K_A_COPY);) {
res[2] = contentEquals_release_2_8_0(inputStream1, inputStream2);
}
return res;
}

@Benchmark
public void testStringCurrent(final Blackhole blackhole) throws IOException {
for (int i = 0; i < 5; i++) {
for (int j = 0; j < 5; j++) {
try (InputStream inputReader1 = IOUtils.toInputStream(STRINGS[i], DEFAULT_CHARSET);
InputStream inputReader2 = IOUtils.toInputStream(STRINGS[j], DEFAULT_CHARSET)) {
blackhole.consume(IOUtils.contentEquals(inputReader1, inputReader2));
}
}
}
}

@Benchmark
public void testStringPr118(final Blackhole blackhole) throws IOException {
for (int i = 0; i < 5; i++) {
for (int j = 0; j < 5; j++) {
try (InputStream inputReader1 = IOUtils.toInputStream(STRINGS[i], DEFAULT_CHARSET);
InputStream inputReader2 = IOUtils.toInputStream(STRINGS[j], DEFAULT_CHARSET)) {
blackhole.consume(contentEqualsPr118(inputReader1, inputReader2));
}
}
}
}

@Benchmark
public void testStringRelease_2_8_0(final Blackhole blackhole) throws IOException {
for (int i = 0; i < 5; i++) {
for (int j = 0; j < 5; j++) {
try (InputStream inputReader1 = IOUtils.toInputStream(STRINGS[i], DEFAULT_CHARSET);
InputStream inputReader2 = IOUtils.toInputStream(STRINGS[j], DEFAULT_CHARSET)) {
blackhole.consume(contentEquals_release_2_8_0(inputReader1, inputReader2));
}
}
}
}

}
Loading

0 comments on commit 5bed26f

Please sign in to comment.