Skip to content

Commit

Permalink
Remove sessions caches from KCB when Persistent Sessions is enabled
Browse files Browse the repository at this point in the history 
Closes #953

Signed-off-by: Pedro Ruivo <[email protected]>
  • Loading branch information
@pruivo
pruivo authored Aug 28, 2024
1 parent 2ecdea5 commit cbb2906
Show file tree
Hide file tree
Showing 7 changed files with 102 additions and 66 deletions.
25 changes: 24 additions & 1 deletion doc/kubernetes/collector/build.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,9 @@ helm template --debug ${STARTDIR}/../../../provision/infinispan/ispn-helm \
helm template --debug ${STARTDIR}/../../../provision/infinispan/ispn-helm \
--set namespace=keycloak \
--set replicas=3 \
--set cpu= \
--set memory= \
--set jvmOptions="" \
--set crossdc.enabled=true \
--set crossdc.local.name=site-a \
--set crossdc.local.gossipRouterEnabled=true \
Expand All @@ -89,10 +92,18 @@ helm template --debug ${STARTDIR}/../../../provision/infinispan/ispn-helm \
--set metrics.histograms=false \
--set hotrodPassword="strong-password" \
--set cacheDefaults.crossSiteMode=SYNC \
--set cacheDefaults.stateTransferMode=AUTO \
--set cacheDefaults.xsiteFailurePolicy=FAIL \
--set cacheDefaults.txMode=NON_XA \
--set cacheDefaults.txLockMode=PESSIMISTIC \
--set image= \
--set fd.interval=2000 \
--set fd.timeout=10000 \
--set createSessionsCaches=false \
--set acceleratorDNS=a3da6a6cbd4e27b02.awsglobalaccelerator.com \
--set alertmanager.webhook.url=https://tjqr2vgc664b6noj6vugprakoq0oausj.lambda-url.eu-west-1.on.aws/ \
--set alertmanager.webhook.username=keycloak \
--set alertmanager.webhook.password=changme \
--set alertmanager.webhook.url=https://tjqr2vgc664b6noj6vugprakoq0oausj.lambda-url.eu-west-1.on.aws/ \
> ${BUILDDIR}/helm/ispn-site-a.yaml

# Infinispan site B deployment
Expand All @@ -113,4 +124,16 @@ helm template --debug ${STARTDIR}/../../../provision/infinispan/ispn-helm \
--set metrics.histograms=false \
--set hotrodPassword="strong-password" \
--set cacheDefaults.crossSiteMode=SYNC \
--set cacheDefaults.stateTransferMode=AUTO \
--set cacheDefaults.xsiteFailurePolicy=FAIL \
--set cacheDefaults.txMode=NON_XA \
--set cacheDefaults.txLockMode=PESSIMISTIC \
--set image= \
--set fd.interval=2000 \
--set fd.timeout=10000 \
--set createSessionsCaches=false \
--set acceleratorDNS=a3da6a6cbd4e27b02.awsglobalaccelerator.com \
--set alertmanager.webhook.url=https://tjqr2vgc664b6noj6vugprakoq0oausj.lambda-url.eu-west-1.on.aws/ \
--set alertmanager.webhook.username=keycloak \
--set alertmanager.webhook.password=changme \
> ${BUILDDIR}/helm/ispn-site-b.yaml
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

<properties>
<keycloak.version>999.0.0-SNAPSHOT</keycloak.version>
<infinispan.version>15.0.7.Final</infinispan.version>
<infinispan.version>15.0.8.Final</infinispan.version>
<junit5.version>5.10.1</junit5.version>
<httpclient.version>4.5.14</httpclient.version>
<maven.enforcer.plugin.version>3.4.1</maven.enforcer.plugin.version>
Expand Down
2 changes: 2 additions & 0 deletions provision/infinispan/Utils.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ tasks:
CROSS_DC_REMOTE_GOSSIP_ROUTER: '{{.CROSS_DC_REMOTE_GOSSIP_ROUTER | default "true"}}'
CROSS_DC_FD_INTERVAL: '{{.CROSS_DC_FD_INTERVAL | default "2000"}}'
CROSS_DC_FD_TIMEOUT: '{{.CROSS_DC_FD_TIMEOUT | default "10000"}}'
CROSS_DC_VOLATILE_SESSIONS: '{{.CROSS_DC_VOLATILE_SESSIONS | default .KC_EXTERNAL_INFINISPAN | default "false"}}' # false if persistent sessions are enabled in Keycloak
cmds:
- >
KUBECONFIG=".task/kubecfg/{{.ROSA_CLUSTER_NAME}}" helm upgrade --install infinispan --namespace {{.NAMESPACE}}
Expand Down Expand Up @@ -107,6 +108,7 @@ tasks:
--set image={{.CROSS_DC_IMAGE}}
--set fd.interval={{.CROSS_DC_FD_INTERVAL}}
--set fd.timeout={{.CROSS_DC_FD_TIMEOUT}}
--set createSessionsCaches={{.CROSS_DC_VOLATILE_SESSIONS}}
--set acceleratorDNS={{ .ACCELERATOR_DNS }}
--set alertmanager.webhook.url={{ .ACCELERATOR_WEBHOOK_URL }}
--set alertmanager.webhook.username={{ .ACCELERATOR_WEBHOOK_USERNAME }}
Expand Down
24 changes: 15 additions & 9 deletions provision/infinispan/ispn-helm/templates/infinispan.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,9 @@ spec:
{{- end }}
# end::infinispan-crossdc[]
{{range $cache, $config := .Values.caches -}}
{{- if and (not $.Values.createSessionsCaches) (eq $cache "sessions" "offlineSessions" "clientSessions" "offlineClientSessions") }}
{{continue}}
{{- end}}
---
# tag::infinispan-cache-{{ $cache }}[]
apiVersion: infinispan.org/v2alpha1
Expand All @@ -146,8 +149,8 @@ spec:
locking:
acquireTimeout: {{ $config.lockTimeout | default $.Values.cacheDefaults.lockTimeout | quote }}
transaction:
mode: {{ $config.txMode | default $.Values.cacheDefaults.txMode | quote }}
locking: {{ $config.txLockMode | default $.Values.cacheDefaults.txLockMode | quote }}
mode: {{ $config.txMode | default $.Values.cacheDefaults.txMode | quote }} # <1>
locking: {{ $config.txLockMode | default $.Values.cacheDefaults.txLockMode | quote }} # <2>
{{- if and $config.memory $config.memory.maxCount }}
memory:
maxCount: {{ $config.memory.maxCount }}
Expand All @@ -157,18 +160,18 @@ spec:
{{ if $.Values.crossdc.enabled }}
{{- $_ := $.Values.crossdc.remote.name | required ".Values.crossdc.remote.name is required." -}}
backups:
{{- if $config.mergePolicy }}
mergePolicy: {{ $config.mergePolicy | quote }} # <1>
{{- if and $config.mergePolicy (eq ($config.crossSiteMode | default $.Values.cacheDefaults.crossSiteMode) "ASYNC")}}
mergePolicy: {{ $config.mergePolicy | quote }}
{{- end}}
{{$.Values.crossdc.remote.name }}: # <2>
{{$.Values.crossdc.remote.name }}: # <3>
backup:
strategy: {{ $config.crossSiteMode | default $.Values.cacheDefaults.crossSiteMode | quote }} # <3>
timeout: {{ $config.xsiteRemoteTimeout | default $.Values.cacheDefaults.xsiteRemoteTimeout | quote }}
failurePolicy: {{ $config.xsiteFailurePolicy | default $.Values.cacheDefaults.xsiteFailurePolicy | quote }}
strategy: {{ $config.crossSiteMode | default $.Values.cacheDefaults.crossSiteMode | quote }} # <4>
timeout: {{ $config.xsiteRemoteTimeout | default $.Values.cacheDefaults.xsiteRemoteTimeout | quote }} # <5>
failurePolicy: {{ $config.xsiteFailurePolicy | default $.Values.cacheDefaults.xsiteFailurePolicy | quote }} # <6>
stateTransfer:
chunkSize: "16"
{{- if eq ($config.crossSiteMode | default $.Values.cacheDefaults.crossSiteMode) "ASYNC"}}
mode: {{ $config.stateTransferMode | default $.Values.cacheDefaults.stateTransferMode | quote }} # <4>
mode: {{ $config.stateTransferMode | default $.Values.cacheDefaults.stateTransferMode | quote }}
{{- end }}
{{- end }}
# end::infinispan-cache-{{ $cache }}[]
Expand Down Expand Up @@ -244,6 +247,9 @@ metadata:
data:
batch: |+
{{range $cache, $config := .Values.caches -}}
{{- if and (not $.Values.createSessionsCaches) (eq $cache "sessions" "offlineSessions" "clientSessions" "offlineClientSessions") }}
{{continue}}
{{- end}}
clearcache {{ $cache }}
{{ end }}
# end::infinispan-crossdc-clear-caches[]
Expand Down
1 change: 1 addition & 0 deletions provision/infinispan/ispn-helm/values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,3 +65,4 @@ alertmanager:
url: ''
username: ''
password: ''
createSessionsCaches: 'false'
110 changes: 56 additions & 54 deletions ...c-tests/src/test/java/org/keycloak/benchmark/crossdc/client/ExternalInfinispanClient.java
View file
Original file line number Diff line number Diff line change
@@ -1,25 +1,20 @@
package org.keycloak.benchmark.crossdc.client;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.keycloak.benchmark.crossdc.util.HttpClientUtils.ACCEPT_ALL_HOSTNAME_VERIFIER;
import static org.keycloak.benchmark.crossdc.util.HttpClientUtils.MOCK_TRUST_MANAGER;
import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.CLIENT_SESSION_CACHE_NAME;
import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.USER_SESSION_CACHE_NAME;

import java.net.Socket;
import java.net.URI;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletionStage;
import java.util.stream.Collectors;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLEngine;
import javax.net.ssl.SSLSession;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509ExtendedTrustManager;

import org.infinispan.client.hotrod.Flag;
import org.infinispan.client.hotrod.RemoteCache;
Expand Down Expand Up @@ -63,13 +58,13 @@ private static RestClient createRestClient(String host, int port, String usernam
var builder = new RestClientConfigurationBuilder();
builder.addServer().host(host).port(port);
builder.security().authentication().username(Objects.requireNonNull(username)).password(Objects.requireNonNull(password));
builder.security().ssl().sslContext(sslContext).trustManagers(new TrustManager[]{TRUST_ALL_MANAGER}).hostnameVerifier(ACCEPT_ALL_HOSTNAME_VERIFIER);
builder.security().ssl().sslContext(sslContext).trustManagers(new TrustManager[]{MOCK_TRUST_MANAGER}).hostnameVerifier(ACCEPT_ALL_HOSTNAME_VERIFIER);
return RestClient.forConfiguration(builder.build());
}

private static SSLContext createSSLContext() {
try {
var trustManagers = new TrustManager[]{TRUST_ALL_MANAGER};
var trustManagers = new TrustManager[]{MOCK_TRUST_MANAGER};
var sslContext = SSLContext.getInstance("TLS");
sslContext.init(null, trustManagers, null);
return sslContext;
Expand All @@ -94,6 +89,55 @@ public String siteName() {
return siteName;
}

private record NonExistingCache(String cacheName) implements InfinispanClient.ExternalCache {


@Override
public void takeOffline(String backupSiteName) {
//no-op
}

@Override
public void bringOnline(String backupSiteName) {
//no-op
}

@Override
public boolean isBackupOnline(String backupSiteName) {
return false;
}

@Override
public long size() {
return 0;
}

@Override
public void clear() {

}

@Override
public boolean contains(String key) {
return false;
}

@Override
public boolean remove(String key) {
return false;
}

@Override
public Set<String> keys() {
return Set.of();
}

@Override
public String name() {
return cacheName;
}
}

public static class ExternalCache implements InfinispanClient.ExternalCache {

private final RestCacheClient cacheRestClient;
Expand Down Expand Up @@ -163,8 +207,9 @@ public boolean isBackupOnline(String backupSiteName) {
}

@Override
public ExternalCache cache(String name) {
return new ExternalCache(restClient.cache(name), hotRodClient.getCache(name));
public InfinispanClient.ExternalCache cache(String name) {
RemoteCache<Object, Object> cache = hotRodClient.getCache(name);
return cache == null ? new NonExistingCache(name) : new ExternalCache(restClient.cache(name), hotRodClient.getCache(name));
}

@Override
Expand Down Expand Up @@ -196,47 +241,4 @@ public boolean isSiteOffline(String site) {
public void bringBackupOnline(String site) {
try (var ignore = awaitAndCheckOkStatus(restClient.container().bringBackupOnline(site))) {}
}

public static final X509ExtendedTrustManager TRUST_ALL_MANAGER = new X509ExtendedTrustManager() {
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType, Socket socket) throws CertificateException {
}

@Override
public void checkServerTrusted(X509Certificate[] chain, String authType, Socket socket) throws CertificateException {

}

@Override
public void checkClientTrusted(X509Certificate[] chain, String authType, SSLEngine engine) throws CertificateException {

}

@Override
public void checkServerTrusted(X509Certificate[] chain, String authType, SSLEngine engine) throws CertificateException {

}

@Override
public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {

}

@Override
public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {

}

@Override
public X509Certificate[] getAcceptedIssuers() {
return new X509Certificate[0];
}
};

private static final HostnameVerifier ACCEPT_ALL_HOSTNAME_VERIFIER = new HostnameVerifier() {
@Override
public boolean verify(String hostname, SSLSession session) {
return true;
}
};
}
4 changes: 3 additions & 1 deletion ...mark-crossdc-tests/src/test/java/org/keycloak/benchmark/crossdc/util/HttpClientUtils.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package org.keycloak.benchmark.crossdc.util;

import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLEngine;
import javax.net.ssl.TrustManager;
Expand All @@ -15,7 +16,8 @@
public class HttpClientUtils {

public static final CookieManager MOCK_COOKIE_MANAGER = new CookieManager();
private static final TrustManager MOCK_TRUST_MANAGER = new X509ExtendedTrustManager() {
public static final HostnameVerifier ACCEPT_ALL_HOSTNAME_VERIFIER = (hostname, session) -> true;
public static final TrustManager MOCK_TRUST_MANAGER = new X509ExtendedTrustManager() {
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType, Socket socket) {

Expand Down

0 comments on commit cbb2906

Please sign in to comment.