Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

VFS-824 HttpFileSystem free Unused Resources lead to HttpClient Conn… #300

Open
wants to merge 15 commits into
base: master
Choose a base branch
from
Open

VFS-824 HttpFileSystem free Unused Resources lead to HttpClient Conn… #300

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
1a6685f
VFS-824 HttpFileSystem free Unused Resources lead to HttpClient Conn…
Sep 19, 2022
7d263ee
VFS-824 Fix HttpFileSystem free Unused Resources lead to HttpClient C…
Sep 19, 2022
7c09877
VFS-824 Fix HttpFileSystem free Unused Resources lead to HttpClient C…
Sep 19, 2022
ddd84bc
fix: (javadoc) JavadocStyle: First sentence should end with a period.
Sep 19, 2022
88b5046
review fix
Sep 20, 2022
66b0b91
review fix
Sep 21, 2022
5b05197
review fix
Sep 21, 2022
5fee9d8
review fix
Sep 21, 2022
852c9bf
Revert "VFS-824 Fix HttpFileSystem free Unused Resources lead to Http…
Sep 21, 2022
5d0f199
unit test: sleep 20 millis
Sep 22, 2022
4853cca
unit test: fix:decrease AbstractFileSystem.useCount directly,make Fil…
Jan 14, 2023
63017d9
unit test: fix:decrease AbstractFileSystem.useCount directly,make Fil…
Jan 14, 2023
ed09716
Remove extra empty lines
garydgregory Jan 18, 2023
fa05f61
Remove extra empty lines
garydgregory Jan 18, 2023
27e842a
Clean up and fix Javadoc
garydgregory Jan 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions commons-vfs2/src/main/java/org/apache/commons/vfs2/provider/AbstractFileProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,21 @@ public void freeUnusedResources() {
.forEach(AbstractFileSystem::closeCommunicationLink);
}

/**
* get snapshot of all fileSystem
mind-echo marked this conversation as resolved.
Show resolved Hide resolved
*
* @return FileSystem Array
mind-echo marked this conversation as resolved.
Show resolved Hide resolved
*/
public AbstractFileSystem[] getAllFileSystemSnapshot() {
mind-echo marked this conversation as resolved.
Show resolved Hide resolved
final AbstractFileSystem[] abstractFileSystems;
synchronized (fileSystemMap) {
// create snapshot under lock
abstractFileSystems = fileSystemMap.values().toArray(EMPTY_ABSTRACT_FILE_SYSTEMS);
}
return abstractFileSystems;
}


/**
* Gets the FileSystemConfigBuilder.
*
Expand Down
17 changes: 17 additions & 0 deletions commons-vfs2/src/main/java/org/apache/commons/vfs2/provider/http/HttpFileProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.stream.Stream;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.vfs2.Capability;
Expand All @@ -28,6 +29,7 @@
import org.apache.commons.vfs2.FileSystemException;
import org.apache.commons.vfs2.FileSystemOptions;
import org.apache.commons.vfs2.UserAuthenticationData;
import org.apache.commons.vfs2.provider.AbstractFileSystem;
import org.apache.commons.vfs2.provider.AbstractOriginatingFileProvider;
import org.apache.commons.vfs2.provider.GenericFileName;
import org.apache.commons.vfs2.util.UserAuthenticatorUtils;
Expand Down Expand Up @@ -93,4 +95,19 @@ public Collection<Capability> getCapabilities() {
public FileSystemConfigBuilder getConfigBuilder() {
return HttpFileSystemConfigBuilder.getInstance();
}


/**
* Frees unused resources and close HttpFileSystem.
*/
@Override
public void freeUnusedResources() {
final AbstractFileSystem[] abstractFileSystems = getAllFileSystemSnapshot();
// process snapshot outside lock
Stream.of(abstractFileSystems).filter(AbstractFileSystem::isReleaseable)
.forEach(system -> {
system.closeCommunicationLink();
closeFileSystem(system);
});
}
}
15 changes: 15 additions & 0 deletions commons-vfs2/src/main/java/org/apache/commons/vfs2/provider/http4/Http4FileProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import org.apache.commons.vfs2.FileSystemOptions;
import org.apache.commons.vfs2.UserAuthenticationData;
import org.apache.commons.vfs2.UserAuthenticator;
import org.apache.commons.vfs2.provider.AbstractFileSystem;
import org.apache.commons.vfs2.provider.AbstractOriginatingFileProvider;
import org.apache.commons.vfs2.provider.GenericFileName;
import org.apache.commons.vfs2.util.UserAuthenticatorUtils;
Expand Down Expand Up @@ -378,4 +379,18 @@ private HttpHost getProxyHttpHost(final Http4FileSystemConfigBuilder builder,
return null;
}


/**
* Frees unused resources and close HttpFileSystem.
mind-echo marked this conversation as resolved.
Show resolved Hide resolved
*/
@Override
public void freeUnusedResources() {
final AbstractFileSystem[] abstractFileSystems = getAllFileSystemSnapshot();
// process snapshot outside lock
Stream.of(abstractFileSystems).filter(AbstractFileSystem::isReleaseable)
.forEach(system -> {
system.closeCommunicationLink();
closeFileSystem(system);
});
}
}
16 changes: 16 additions & 0 deletions commons-vfs2/src/main/java/org/apache/commons/vfs2/provider/http5/Http5FileProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import org.apache.commons.vfs2.FileSystemOptions;
import org.apache.commons.vfs2.UserAuthenticationData;
import org.apache.commons.vfs2.UserAuthenticator;
import org.apache.commons.vfs2.provider.AbstractFileSystem;
import org.apache.commons.vfs2.provider.AbstractOriginatingFileProvider;
import org.apache.commons.vfs2.provider.GenericFileName;
import org.apache.commons.vfs2.util.UserAuthenticatorUtils;
Expand Down Expand Up @@ -375,4 +376,19 @@ private HttpHost getProxyHttpHost(final Http5FileSystemConfigBuilder builder,
return null;
}


/**
* Frees unused resources and close HttpFileSystem.
mind-echo marked this conversation as resolved.
Show resolved Hide resolved
*/
@Override
public void freeUnusedResources() {
final AbstractFileSystem[] abstractFileSystems = getAllFileSystemSnapshot();
// process snapshot outside lock
Stream.of(abstractFileSystems).filter(AbstractFileSystem::isReleaseable)
.forEach(system -> {
system.closeCommunicationLink();
closeFileSystem(system);
});
}

}
35 changes: 35 additions & 0 deletions commons-vfs2/src/test/java/org/apache/commons/vfs2/provider/http/HttpProviderTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,10 @@
import org.apache.commons.vfs2.FileSystemOptions;
import org.apache.commons.vfs2.ProviderTestSuite;
import org.apache.commons.vfs2.VFS;
import org.apache.commons.vfs2.cache.WeakRefFilesCache;
import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
import org.apache.commons.vfs2.impl.StandardFileSystemManager;
import org.apache.commons.vfs2.provider.AbstractFileSystem;
import org.apache.commons.vfs2.util.NHttpFileServer;
import org.junit.Test;
import org.junit.jupiter.api.Assertions;
Expand Down Expand Up @@ -212,4 +215,36 @@ public void testResolveFolderSlashYesRedirectOn() throws FileSystemException {
testResolveFolderSlash(ConnectionUri + "/read-tests/", true);
}

@Test
public void testHttpFileSystemFreeUnusedResources() throws Exception {
try (StandardFileSystemManager fileSystemManager = new StandardFileSystemManager()) {
fileSystemManager.setConfiguration(StandardFileSystemManager.class.getResource("providers.xml"));
// use WeakRef
fileSystemManager.setFilesCache(new WeakRefFilesCache());
fileSystemManager.init();

String path = "http://www.w3schools.com/webservices/tempconvert.asmx?action=WSDL";
mind-echo marked this conversation as resolved.
Show resolved Hide resolved
AbstractFileSystem http4FileSystem = getFile(fileSystemManager, path);
mind-echo marked this conversation as resolved.
Show resolved Hide resolved
http4FileSystem.isReleaseable();

while (!http4FileSystem.isReleaseable()) {
mind-echo marked this conversation as resolved.
Show resolved Hide resolved
// Try GC
System.gc();
}
// free resource
// http4FileSystem.httpclient is closed
fileSystemManager.freeUnusedResources();

// get file again
getFile(fileSystemManager, path);
}
}

private static AbstractFileSystem getFile(FileSystemManager fileSystemManager, String path) throws FileSystemException {
FileObject fileObject = fileSystemManager.resolveFile(path);
// send
fileObject.getType();
return (AbstractFileSystem) fileObject.getFileSystem();
}

}
34 changes: 34 additions & 0 deletions commons-vfs2/src/test/java/org/apache/commons/vfs2/provider/http4/Http4ProviderTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,10 @@
import org.apache.commons.vfs2.FileSystemOptions;
import org.apache.commons.vfs2.ProviderTestSuite;
import org.apache.commons.vfs2.VFS;
import org.apache.commons.vfs2.cache.WeakRefFilesCache;
import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
import org.apache.commons.vfs2.impl.StandardFileSystemManager;
import org.apache.commons.vfs2.provider.AbstractFileSystem;
import org.apache.commons.vfs2.util.NHttpFileServer;
import org.junit.jupiter.api.Assertions;
import org.junit.Test;
Expand Down Expand Up @@ -216,4 +219,35 @@ public void testResolveFolderSlashYesRedirectOn() throws FileSystemException {
testResolveFolderSlash(ConnectionUri + "/read-tests/", true);
}

@Test
public void testHttp4FileSystemFreeUnusedResources() throws Exception {
try (StandardFileSystemManager fileSystemManager = new StandardFileSystemManager()) {
fileSystemManager.setConfiguration(StandardFileSystemManager.class.getResource("providers.xml"));
// use WeakRef
fileSystemManager.setFilesCache(new WeakRefFilesCache());
fileSystemManager.init();

String path = "http4://www.w3schools.com/webservices/tempconvert.asmx?action=WSDL";
mind-echo marked this conversation as resolved.
Show resolved Hide resolved
AbstractFileSystem http4FileSystem = getFile(fileSystemManager, path);
http4FileSystem.isReleaseable();

while (!http4FileSystem.isReleaseable()) {
mind-echo marked this conversation as resolved.
Show resolved Hide resolved
// Try GC
System.gc();
}
// free resource
// http4FileSystem.httpclient is closed
fileSystemManager.freeUnusedResources();

// get file again
getFile(fileSystemManager, path);
}
}

private static AbstractFileSystem getFile(FileSystemManager fileSystemManager, String path) throws FileSystemException {
FileObject fileObject = fileSystemManager.resolveFile(path);
// send
fileObject.getType();
return (AbstractFileSystem) fileObject.getFileSystem();
}
}
35 changes: 35 additions & 0 deletions commons-vfs2/src/test/java/org/apache/commons/vfs2/provider/http5/Http5ProviderTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,10 @@
import org.apache.commons.vfs2.FileSystemOptions;
import org.apache.commons.vfs2.ProviderTestSuite;
import org.apache.commons.vfs2.VFS;
import org.apache.commons.vfs2.cache.WeakRefFilesCache;
import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
import org.apache.commons.vfs2.impl.StandardFileSystemManager;
import org.apache.commons.vfs2.provider.AbstractFileSystem;
import org.apache.commons.vfs2.util.NHttpFileServer;
import org.junit.Test;
import org.junit.jupiter.api.Assertions;
Expand Down Expand Up @@ -210,4 +213,36 @@ public void testResolveFolderSlashYesRedirectOn() throws FileSystemException {
testResolveFolderSlash(ConnectionUri + "/read-tests/", true);
}

@Test
public void testHttp5FileSystemFreeUnusedResources() throws Exception {
try (StandardFileSystemManager fileSystemManager = new StandardFileSystemManager()) {
fileSystemManager.setConfiguration(StandardFileSystemManager.class.getResource("providers.xml"));
// use WeakRef
fileSystemManager.setFilesCache(new WeakRefFilesCache());
fileSystemManager.init();

String path = "http5://www.w3schools.com/webservices/tempconvert.asmx?action=WSDL";
mind-echo marked this conversation as resolved.
Show resolved Hide resolved
AbstractFileSystem http4FileSystem = getFile(fileSystemManager, path);
mind-echo marked this conversation as resolved.
Show resolved Hide resolved
http4FileSystem.isReleaseable();

while (!http4FileSystem.isReleaseable()) {
mind-echo marked this conversation as resolved.
Show resolved Hide resolved
// Try GC
System.gc();
}
// free resource
// http4FileSystem.httpclient is closed
fileSystemManager.freeUnusedResources();

// get file again
getFile(fileSystemManager, path);
}
}

private static AbstractFileSystem getFile(FileSystemManager fileSystemManager, String path) throws FileSystemException {
FileObject fileObject = fileSystemManager.resolveFile(path);
// send
fileObject.getType();
return (AbstractFileSystem) fileObject.getFileSystem();
}

}