diff --git a/databricks-sdk-java/src/main/java/com/databricks/sdk/core/DatabricksConfig.java b/databricks-sdk-java/src/main/java/com/databricks/sdk/core/DatabricksConfig.java index 221c32ffa..1bc47ee04 100644 --- a/databricks-sdk-java/src/main/java/com/databricks/sdk/core/DatabricksConfig.java +++ b/databricks-sdk-java/src/main/java/com/databricks/sdk/core/DatabricksConfig.java @@ -193,7 +193,7 @@ private void initHttp() { return; } // eventually it'll get decoupled from config. - httpClient = new CommonsHttpClient(this); + httpClient = new CommonsHttpClient.Builder().withDatabricksConfig(this).build(); } public synchronized Map authenticate() throws DatabricksException { diff --git a/databricks-sdk-java/src/main/java/com/databricks/sdk/core/ProxyConfig.java b/databricks-sdk-java/src/main/java/com/databricks/sdk/core/ProxyConfig.java index 7eacf2816..92e37a4b9 100644 --- a/databricks-sdk-java/src/main/java/com/databricks/sdk/core/ProxyConfig.java +++ b/databricks-sdk-java/src/main/java/com/databricks/sdk/core/ProxyConfig.java @@ -7,6 +7,13 @@ public class ProxyConfig { private String password; private ProxyAuthType proxyAuthType; private Boolean useSystemProperties; + // a list of hosts that should be reached directly, bypassing the proxy. + // This is a list of patterns separated by '|'. The patterns may start or end with a '*' for + // wildcards. + // Any host matching one of these patterns will be reached through a direct connection instead of + // through a proxy. + // More info here: https://docs.oracle.com/javase/8/docs/technotes/guides/net/proxies.html + private String nonProxyHosts; public enum ProxyAuthType { // Currently we only support BASIC and SPNEGO @@ -80,4 +87,19 @@ public ProxyConfig setUseSystemProperties(Boolean useSystemProperties) { this.useSystemProperties = useSystemProperties; return this; } + + public String getNonProxyHosts() { + return nonProxyHosts; + } + + /** + * @param nonProxyHosts a list of hosts that should be reached directly, bypassing the proxy. This + * is a list of patterns separated by '|'. The patterns may start or end with a '*' for + * wildcards. + * @return the current ProxyConfig object + */ + public ProxyConfig setNonProxyHosts(String nonProxyHosts) { + this.nonProxyHosts = nonProxyHosts; + return this; + } } diff --git a/databricks-sdk-java/src/main/java/com/databricks/sdk/core/commons/CommonsHttpClient.java b/databricks-sdk-java/src/main/java/com/databricks/sdk/core/commons/CommonsHttpClient.java index 7d5e0ec86..cc176169d 100644 --- a/databricks-sdk-java/src/main/java/com/databricks/sdk/core/commons/CommonsHttpClient.java +++ b/databricks-sdk-java/src/main/java/com/databricks/sdk/core/commons/CommonsHttpClient.java @@ -25,6 +25,7 @@ import org.apache.http.*; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.*; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.entity.InputStreamEntity; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; @@ -36,18 +37,101 @@ import org.slf4j.LoggerFactory; public class CommonsHttpClient implements HttpClient { + /** + * Builder for CommonsHttpClient. This class is used to construct instances of CommonsHttpClient + * with configurable parameters for the underlying Apache HttpClient. + */ + public static class Builder { + private DatabricksConfig databricksConfig; + private Integer timeoutSeconds; + private ProxyConfig proxyConfig; + private SSLConnectionSocketFactory sslSocketFactory; + + /** + * @param databricksConfig The DatabricksConfig to use for the HttpClient. If the + * DatabricksConfig has an httpTimeoutSeconds set, it will be used as the default timeout + * for the HttpClient. + * @return This builder. + */ + public Builder withDatabricksConfig(DatabricksConfig databricksConfig) { + this.databricksConfig = databricksConfig; + return this; + } + + /** + * @param timeoutSeconds The timeout in seconds to use for the HttpClient. This will override + * any timeout set in the DatabricksConfig. + * @return This builder. + */ + public Builder withTimeoutSeconds(int timeoutSeconds) { + this.timeoutSeconds = timeoutSeconds; + return this; + } + + /** + * @param proxyConfig the proxy configuration to use for the HttpClient. + * @return This builder. + */ + public Builder withProxyConfig(ProxyConfig proxyConfig) { + this.proxyConfig = proxyConfig; + return this; + } + + /** + * @param sslSocketFactory the SSLConnectionSocketFactory to use for the HttpClient. + * @return This builder. + */ + public Builder withSslSocketFactory(SSLConnectionSocketFactory sslSocketFactory) { + this.sslSocketFactory = sslSocketFactory; + return this; + } + + /** Builds a new instance of CommonsHttpClient with the configured parameters. */ + public CommonsHttpClient build() { + return new CommonsHttpClient(this); + } + } + private static final Logger LOG = LoggerFactory.getLogger(CommonsHttpClient.class); private final PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(); private final CloseableHttpClient hc; private int timeout; + private CommonsHttpClient(Builder builder) { + int timeoutSeconds = 300; + if (builder.databricksConfig != null + && builder.databricksConfig.getHttpTimeoutSeconds() != null) { + timeoutSeconds = builder.databricksConfig.getHttpTimeoutSeconds(); + } + if (builder.timeoutSeconds != null) { + timeoutSeconds = builder.timeoutSeconds; + } + timeout = timeoutSeconds * 1000; + connectionManager.setMaxTotal(100); + HttpClientBuilder httpClientBuilder = + HttpClientBuilder.create() + .setConnectionManager(connectionManager) + .setDefaultRequestConfig(makeRequestConfig()); + if (builder.proxyConfig != null) { + ProxyUtils.setupProxy(builder.proxyConfig, httpClientBuilder); + } + if (builder.sslSocketFactory != null) { + httpClientBuilder.setSSLSocketFactory(builder.sslSocketFactory); + } + hc = httpClientBuilder.build(); + } + + // These constructors have been deprecate in favour of a builder pattern. + // They will be removed in a future release. + @Deprecated public CommonsHttpClient(int timeoutSeconds) { timeout = timeoutSeconds * 1000; connectionManager.setMaxTotal(100); hc = makeClosableHttpClient(); } + @Deprecated public CommonsHttpClient(DatabricksConfig databricksConfig) { this( databricksConfig.getHttpTimeoutSeconds() == null @@ -56,6 +140,7 @@ public CommonsHttpClient(DatabricksConfig databricksConfig) { new ProxyConfig(databricksConfig)); } + @Deprecated public CommonsHttpClient(int timeoutSeconds, ProxyConfig proxyConfig) { timeout = timeoutSeconds * 1000; connectionManager.setMaxTotal(100); @@ -70,6 +155,7 @@ private RequestConfig makeRequestConfig() { .build(); } + @Deprecated private CloseableHttpClient makeClosableHttpClient() { return HttpClientBuilder.create() .setConnectionManager(connectionManager) @@ -77,6 +163,7 @@ private CloseableHttpClient makeClosableHttpClient() { .build(); } + @Deprecated private CloseableHttpClient makeClosableHttpClient(ProxyConfig proxyConfig) { HttpClientBuilder builder = HttpClientBuilder.create() diff --git a/databricks-sdk-java/src/main/java/com/databricks/sdk/core/oauth/ClientCredentials.java b/databricks-sdk-java/src/main/java/com/databricks/sdk/core/oauth/ClientCredentials.java index 80435b4e2..7709a7b10 100644 --- a/databricks-sdk-java/src/main/java/com/databricks/sdk/core/oauth/ClientCredentials.java +++ b/databricks-sdk-java/src/main/java/com/databricks/sdk/core/oauth/ClientCredentials.java @@ -17,7 +17,7 @@ public static class Builder { private String clientId; private String clientSecret; private String tokenUrl; - private HttpClient hc = new CommonsHttpClient(30); + private HttpClient hc = new CommonsHttpClient.Builder().withTimeoutSeconds(30).build(); private Map endpointParams = Collections.emptyMap(); private List scopes = Collections.emptyList(); private AuthParameterPosition position = AuthParameterPosition.BODY; diff --git a/databricks-sdk-java/src/main/java/com/databricks/sdk/core/oauth/Consent.java b/databricks-sdk-java/src/main/java/com/databricks/sdk/core/oauth/Consent.java index 7c1ed7ba2..77045df97 100644 --- a/databricks-sdk-java/src/main/java/com/databricks/sdk/core/oauth/Consent.java +++ b/databricks-sdk-java/src/main/java/com/databricks/sdk/core/oauth/Consent.java @@ -55,7 +55,7 @@ public class Consent implements Serializable { private final String clientSecret; public static class Builder { - private HttpClient hc = new CommonsHttpClient(30); + private HttpClient hc = new CommonsHttpClient.Builder().withTimeoutSeconds(30).build(); private String authUrl; private String verifier; private String state; diff --git a/databricks-sdk-java/src/main/java/com/databricks/sdk/core/utils/CustomRoutePlanner.java b/databricks-sdk-java/src/main/java/com/databricks/sdk/core/utils/CustomRoutePlanner.java new file mode 100644 index 000000000..f5a63c5a9 --- /dev/null +++ b/databricks-sdk-java/src/main/java/com/databricks/sdk/core/utils/CustomRoutePlanner.java @@ -0,0 +1,46 @@ +package com.databricks.sdk.core.utils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import org.apache.http.HttpException; +import org.apache.http.HttpHost; +import org.apache.http.HttpRequest; +import org.apache.http.conn.routing.HttpRoute; +import org.apache.http.conn.routing.HttpRoutePlanner; +import org.apache.http.impl.conn.DefaultProxyRoutePlanner; +import org.apache.http.protocol.HttpContext; + +/** + * Custom route planner that routes requests via a proxy, except for hosts that match a list of + * non-proxy hosts. + */ +public class CustomRoutePlanner implements HttpRoutePlanner { + private final DefaultProxyRoutePlanner defaultRoutePlanner; + private final List nonProxyHostRegex; + + public CustomRoutePlanner(HttpHost proxy, String nonProxyHosts) { + this.defaultRoutePlanner = new DefaultProxyRoutePlanner(proxy); + if (nonProxyHosts == null || nonProxyHosts.isEmpty()) { + this.nonProxyHostRegex = new ArrayList<>(); + } else { + this.nonProxyHostRegex = + Arrays.stream(nonProxyHosts.split("\\|")) + .map(host -> host.replace(".", "\\.").replace("*", ".*")) + .map(Pattern::compile) + .collect(Collectors.toList()); + } + } + + @Override + public HttpRoute determineRoute(HttpHost target, HttpRequest request, HttpContext context) + throws HttpException { + String targetHostName = target.getHostName(); + if (nonProxyHostRegex.stream().anyMatch(pattern -> pattern.matcher(targetHostName).matches())) { + return new HttpRoute(target); // Direct route, no proxy + } + return defaultRoutePlanner.determineRoute(target, request, context); // Route via proxy + } +} diff --git a/databricks-sdk-java/src/main/java/com/databricks/sdk/core/utils/ProxyUtils.java b/databricks-sdk-java/src/main/java/com/databricks/sdk/core/utils/ProxyUtils.java index b51caa11c..b02a38d39 100644 --- a/databricks-sdk-java/src/main/java/com/databricks/sdk/core/utils/ProxyUtils.java +++ b/databricks-sdk-java/src/main/java/com/databricks/sdk/core/utils/ProxyUtils.java @@ -52,6 +52,14 @@ public static void setupProxy(ProxyConfig config, HttpClientBuilder builder) { proxyAuthType = config.getProxyAuthType(); builder.setProxy(new HttpHost(proxyHost, proxyPort)); } + if (proxyHost == null) { + // No proxy is set in system properties or in the config + return; + } + if (config.getNonProxyHosts() != null) { + builder.setRoutePlanner( + new CustomRoutePlanner(new HttpHost(proxyHost, proxyPort), config.getNonProxyHosts())); + } setupProxyAuth(proxyHost, proxyPort, proxyAuthType, proxyUser, proxyPassword, builder); } diff --git a/databricks-sdk-java/src/test/java/com/databricks/sdk/core/commons/CommonsHttpClientTest.java b/databricks-sdk-java/src/test/java/com/databricks/sdk/core/commons/CommonsHttpClientTest.java index 58626920c..7ef523b79 100644 --- a/databricks-sdk-java/src/test/java/com/databricks/sdk/core/commons/CommonsHttpClientTest.java +++ b/databricks-sdk-java/src/test/java/com/databricks/sdk/core/commons/CommonsHttpClientTest.java @@ -16,7 +16,7 @@ class CommonsHttpClientTest { @Test public void itWorks() throws IOException { try (FixtureServer fixtures = new FixtureServer().with("GET", "/foo?x=y", "bar")) { - HttpClient httpClient = new CommonsHttpClient(30); + HttpClient httpClient = new CommonsHttpClient.Builder().withTimeoutSeconds(30).build(); Request in = new Request("GET", fixtures.getUrl() + "/foo").withQueryParam("x", "y"); Response out = httpClient.execute(in); assertEquals("bar", out.getDebugBody().trim()); @@ -37,7 +37,7 @@ public void testStringBody() throws IOException { .withResponse("quux") .build(); try (FixtureServer fixtures = new FixtureServer().with(fixture)) { - HttpClient httpClient = new CommonsHttpClient(30); + HttpClient httpClient = new CommonsHttpClient.Builder().withTimeoutSeconds(30).build(); Request in = new Request("POST", fixtures.getUrl() + "/foo", "bar"); Response out = httpClient.execute(in); assertEquals("quux", out.getDebugBody().trim()); @@ -58,7 +58,7 @@ public void testInputStreamBody() throws IOException { .withResponse("quux") .build(); try (FixtureServer fixtures = new FixtureServer().with(fixture)) { - HttpClient httpClient = new CommonsHttpClient(30); + HttpClient httpClient = new CommonsHttpClient.Builder().withTimeoutSeconds(30).build(); Request in = new Request("POST", fixtures.getUrl() + "/foo", IOUtils.toInputStream("bar", "UTF-8")); Response out = httpClient.execute(in); @@ -76,7 +76,7 @@ public void testNoRedirection() throws IOException { .build(); try (FixtureServer fixtures = new FixtureServer().with(fixture)) { - HttpClient httpClient = new CommonsHttpClient(30); + HttpClient httpClient = new CommonsHttpClient.Builder().withTimeoutSeconds(30).build(); Request in = new Request("GET", fixtures.getUrl() + "/redirect"); in.setRedirectionBehavior( false); // If we don't set redirection behavior to false, we get 200 as it gets @@ -106,7 +106,7 @@ public void testRedirection() throws IOException { .build()); try (FixtureServer server = new FixtureServer().with(fixtures)) { - HttpClient httpClient = new CommonsHttpClient(30); + HttpClient httpClient = new CommonsHttpClient.Builder().withTimeoutSeconds(30).build(); Request in = new Request("GET", server.getUrl() + "/redirect"); Response out = httpClient.execute(in); assertEquals( diff --git a/databricks-sdk-java/src/test/java/com/databricks/sdk/core/oauth/ExternalBrowserCredentialsProviderTest.java b/databricks-sdk-java/src/test/java/com/databricks/sdk/core/oauth/ExternalBrowserCredentialsProviderTest.java index e23fab740..cec87417f 100644 --- a/databricks-sdk-java/src/test/java/com/databricks/sdk/core/oauth/ExternalBrowserCredentialsProviderTest.java +++ b/databricks-sdk-java/src/test/java/com/databricks/sdk/core/oauth/ExternalBrowserCredentialsProviderTest.java @@ -35,7 +35,7 @@ void clientAndConsentTest() throws IOException { .setAuthType("external-browser") .setHost(fixtures.getUrl()) .setClientId("test-client-id") - .setHttpClient(new CommonsHttpClient(30)); + .setHttpClient(new CommonsHttpClient.Builder().withTimeoutSeconds(30).build()); config.resolve(); assertEquals("tokenEndPointFromServer", config.getOidcEndpoints().getTokenEndpoint()); @@ -70,7 +70,7 @@ void clientAndConsentTestWithCustomRedirectUrl() throws IOException { .setAuthType("external-browser") .setHost(fixtures.getUrl()) .setClientId("test-client-id") - .setHttpClient(new CommonsHttpClient(30)) + .setHttpClient(new CommonsHttpClient.Builder().withTimeoutSeconds(30).build()) .setOAuthRedirectUrl("http://localhost:8010") .setScopes(Arrays.asList("sql")); config.resolve(); @@ -98,7 +98,7 @@ void openIDConnectEndPointsTestAccounts() throws IOException { new DatabricksConfig() .setAuthType("external-browser") .setHost("https://accounts.cloud.databricks.com") - .setHttpClient(new CommonsHttpClient(30)) + .setHttpClient(new CommonsHttpClient.Builder().withTimeoutSeconds(30).build()) .setAccountId("testAccountId"); config.resolve(); diff --git a/databricks-sdk-java/src/test/java/com/databricks/sdk/core/utils/CustomRoutePlannerTest.java b/databricks-sdk-java/src/test/java/com/databricks/sdk/core/utils/CustomRoutePlannerTest.java new file mode 100644 index 000000000..51c672360 --- /dev/null +++ b/databricks-sdk-java/src/test/java/com/databricks/sdk/core/utils/CustomRoutePlannerTest.java @@ -0,0 +1,90 @@ +package com.databricks.sdk.core.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.util.Arrays; +import org.apache.http.HttpHost; +import org.apache.http.conn.routing.HttpRoute; +import org.apache.http.conn.routing.HttpRoutePlanner; +import org.apache.http.message.BasicHttpRequest; +import org.apache.http.protocol.BasicHttpContext; +import org.apache.http.protocol.HttpContext; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class CustomRoutePlannerTest { + + private static HttpHost proxy; + private static HttpRoutePlanner customRoutePlanner; + private static HttpContext context; + + @BeforeAll + public static void setUp() { + proxy = new HttpHost("proxy.example.com", 8080); + String nonProxyHosts = + String.join("|", Arrays.asList("example.com", "localhost", "*.mydomain.com")); + customRoutePlanner = new CustomRoutePlanner(proxy, nonProxyHosts); + context = new BasicHttpContext(); + } + + @Test + public void testDirectRouteForExactNonProxyHost() throws Exception { + // Test a host that should bypass the proxy + HttpHost target = new HttpHost("example.com", 80); + HttpRoute route = + customRoutePlanner.determineRoute(target, new BasicHttpRequest("GET", "/"), context); + + // Assert that the route is direct (no proxy) + assertEquals(target, route.getTargetHost()); + assertNull(route.getProxyHost()); + } + + @Test + public void testDirectRouteForWildcardNonProxyHost() throws Exception { + // Test a host that matches the wildcard pattern (*.mydomain.com) + HttpHost target = new HttpHost("api.mydomain.com", 80); + HttpRoute route = + customRoutePlanner.determineRoute(target, new BasicHttpRequest("GET", "/"), context); + + // Assert that the route is direct (no proxy) + assertEquals(target, route.getTargetHost()); + assertNull(route.getProxyHost()); + } + + @Test + public void testDirectRouteForLocalhost() throws Exception { + // Test a localhost, which should bypass the proxy + HttpHost target = new HttpHost("localhost", 80); + HttpRoute route = + customRoutePlanner.determineRoute(target, new BasicHttpRequest("GET", "/"), context); + + // Assert that the route is direct (no proxy) + assertEquals(target, route.getTargetHost()); + assertNull(route.getProxyHost()); + } + + @Test + public void testProxyRouteForNonMatchingHost() throws Exception { + // Test a host that does not match the non-proxy patterns + HttpHost target = new HttpHost("otherdomain.com", 80); + HttpRoute route = + customRoutePlanner.determineRoute(target, new BasicHttpRequest("GET", "/"), context); + + // Assert that the route goes through the proxy + assertEquals(target, route.getTargetHost()); + assertEquals(proxy, route.getProxyHost()); + } + + @Test + public void testProxyRouteForPartialWildcardMatch() throws Exception { + // Test a host that does not fully match the wildcard pattern (*.mydomain.com) + HttpHost target = new HttpHost("mydomain.org", 80); + HttpRoute route = + customRoutePlanner.determineRoute(target, new BasicHttpRequest("GET", "/"), context); + + // Assert that the route goes through the proxy + assertEquals(target, route.getTargetHost()); + assertEquals(proxy, route.getProxyHost()); + } +} diff --git a/examples/spring-boot-oauth-u2m-demo/src/main/java/com/databricks/sdk/App.java b/examples/spring-boot-oauth-u2m-demo/src/main/java/com/databricks/sdk/App.java index e0058df0b..beb33d487 100644 --- a/examples/spring-boot-oauth-u2m-demo/src/main/java/com/databricks/sdk/App.java +++ b/examples/spring-boot-oauth-u2m-demo/src/main/java/com/databricks/sdk/App.java @@ -20,7 +20,7 @@ public static void main(String[] args) { @Bean public HttpClient getHttpClient() { - return new CommonsHttpClient(30); + return new CommonsHttpClient.Builder().withTimeoutSeconds(30).build(); } @Bean