Hello All,
We have a multi node cluster of Keycloak 26 deployed to Kubernetes in the Google Cloud with an external Infinispan cache.
Sometimes, maybe always, when an update is made vai the Admin Rest API the update is not reflected in all of the Keycloak nodes. For example, user password updates, authorization flow configuration, etc…
Any insight as to how to troubleshoot this or mistakes in the configuration would be very helpful.
Here is the config for the external infinispan cluster:
infinispan:
cacheContainer:
# Cache Container name is used for defining health status probe URL paths
name: default
statistics: true
# [USER] Specify `security: null` to disable security authorization.
security:
authorization: {}
transport:
# Configure Infinispan to use DNS PING for discovery
stack: kubernetes
cluster: ${infinispan.cluster.name}
site: ${gke.cluster.name}
# We will map the GKE Zone to the rack
rack: ${gke.zone.name}
node-name: ${gke.pod.name}
serialization:
# Configure the Keycloak classes that can be deserialized in the console
allow-list:
class: java.util.UUID
regex: org.keycloak.*
caches:
# Template for Replicated Caches
replicated-cache-cfg:
replicated-cache-configuration:
mode: SYNC
statistics: "true"
locking:
isolation: READ_COMMITTED
# Disable striping to create a new lock per entry (greater concurrent throughput, avoid potential deadlocks)
# Increases memory usage and garbage collection
striping: false
# Amount of time, in milliseconds, to wait for a contented lock
acquire-timeout: 20000
transaction:
mode: NON_XA
locking: PESSIMISTIC
encoding:
mediaType: application/x-protostream
# Provides data consistency over cache availability in case of network partition
partitionHandling:
whenSplit: DENY_READ_WRITES
mergePolicy: PREFERRED_ALWAYS
# Template for Distributed Caches
distributed-cache-cfg:
distributed-cache-configuration:
mode: SYNC
# Explicitly disable the L1 cache to prevent stale reads
l1-lifespan: -1
statistics: "true"
locking:
isolation: READ_COMMITTED
# Disable striping to create a new lock per entry (greater concurrent throughput, avoid potential deadlocks)
# Increases memory usage and garbage collection
striping: false
# Amount of time, in milliseconds, to wait for a contented lock
acquire-timeout: 20000
transaction:
mode: NON_XA
locking: PESSIMISTIC
encoding:
mediaType: application/x-protostream
# Keycloak work cache
work:
replicated-cache:
configuration: replicated-cache-cfg
# Keycloak sessions cache
sessions:
distributed-cache:
configuration: distributed-cache-cfg
owners: 1
# Keycloak authenticated sessions cache
authenticationSessions:
distributed-cache:
configuration: distributed-cache-cfg
owners: 2
# Provides data consistency over cache availability in case of network partition. Only for cache with 2 owners
partitionHandling:
whenSplit: DENY_READ_WRITES
mergePolicy: PREFERRED_ALWAYS
# Keycloak offline sessions cache
offlineSessions:
distributed-cache:
configuration: distributed-cache-cfg
owners: 1
# Keycloak client sessions cache
clientSessions:
distributed-cache:
configuration: distributed-cache-cfg
owners: 1
# Keycloak offline client sessions cache
offlineClientSessions:
distributed-cache:
configuration: distributed-cache-cfg
owners: 1
# Keycloak login failures cache
loginFailures:
distributed-cache:
configuration: distributed-cache-cfg
owners: 2
partitionHandling:
whenSplit: DENY_READ_WRITES
mergePolicy: PREFERRED_ALWAYS
# Keycloak action tokens cache
actionTokens:
distributed-cache:
configuration: distributed-cache-cfg
owners: 2
partitionHandling:
whenSplit: DENY_READ_WRITES
mergePolicy: PREFERRED_ALWAYS
server:
endpoints:
# [USER] Hot Rod and REST endpoints.
- securityRealm: default
socketBinding: default
connectors:
rest:
restConnector:
hotrod:
hotrodConnector:
# [METRICS] Metrics endpoint for cluster monitoring capabilities.
- securityRealm: metrics
socketBinding: metrics
connectors:
rest:
restConnector:
interfaces:
- name: public
anyAddress: ~
socketBindings:
defaultInterface: public
portOffset: 0
socketBinding:
# [USER] Socket binding for the Hot Rod and REST endpoints.
- name: default
port: 11222
# [METRICS] Socket binding for the metrics endpoint.
- name: metrics
port: 11223
security:
credentialStores:
- clearTextCredential:
clearText: secret
name: credentials
path: credentials.pfx
securityRealms:
# [USER] Security realm for the Hot Rod and REST endpoints.
- name: default
# [USER] Comment or remove this properties realm to disable authentication.
propertiesRealm:
groupProperties:
path: groups.properties
groupsAttribute: Roles
userProperties:
path: users.properties
# [METRICS] Security realm for the metrics endpoint.
- name: metrics
Here is the configuration for the Keycloak embedded cache:
<?xml version="1.0" encoding="UTF-8"?>
<infinispan
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="urn:infinispan:config:15.0 https://www.infinispan.org/schemas/infinispan-config-15.0.xsd"
xmlns="urn:infinispan:config:15.0">
<cache-container name="embedded-keycloak-cache" statistics="true">
<transport cluster="embedded-cluster" lock-timeout="60000" site="embedded" />
<metrics names-as-tags="true" />
<local-cache name="realms" simple-cache="true" statistics="true">
<encoding>
<key media-type="application/x-java-object"/>
<value media-type="application/x-java-object"/>
</encoding>
<memory max-count="10000"/>
</local-cache>
<local-cache name="users" simple-cache="true" statistics="true">
<encoding>
<key media-type="application/x-java-object"/>
<value media-type="application/x-java-object"/>
</encoding>
<memory max-count="10000"/>
</local-cache>
<local-cache name="authorization" simple-cache="true" statistics="true">
<encoding>
<key media-type="application/x-java-object"/>
<value media-type="application/x-java-object"/>
</encoding>
<memory max-count="10000"/>
</local-cache>
<local-cache name="keys" simple-cache="true" statistics="true">
<encoding>
<key media-type="application/x-java-object"/>
<value media-type="application/x-java-object"/>
</encoding>
<expiration max-idle="3600000"/>
<memory max-count="1000"/>
</local-cache>
<replicated-cache name="work" statistics="true">
<expiration lifespan="-1"/>
<persistence passivation="false" availability-interval="500"/>
</replicated-cache>
<distributed-cache name="sessions" owners="1" statistics="true">
<expiration lifespan="-1"/>
<persistence passivation="false" availability-interval="500"/>
<state-transfer enabled="false"/>
</distributed-cache>
<distributed-cache name="authenticationSessions" owners="2" statistics="true">
<expiration lifespan="-1"/>
<persistence passivation="false" availability-interval="500"/>
<state-transfer enabled="false"/>
</distributed-cache>
<distributed-cache name="offlineSessions" owners="1" statistics="true">
<expiration lifespan="-1"/>
<persistence passivation="false" availability-interval="500"/>
<state-transfer enabled="false"/>
</distributed-cache>
<distributed-cache name="clientSessions" owners="1" statistics="true">
<expiration lifespan="-1"/>
<persistence passivation="false" availability-interval="500"/>
<state-transfer enabled="false"/>
</distributed-cache>
<distributed-cache name="offlineClientSessions" owners="1" statistics="true">
<expiration lifespan="-1"/>
<persistence passivation="false" availability-interval="500"/>
<state-transfer enabled="false"/>
</distributed-cache>
<distributed-cache name="loginFailures" owners="2" statistics="true">
<expiration lifespan="-1"/>
<persistence passivation="false" availability-interval="500"/>
<state-transfer enabled="false"/>
</distributed-cache>
<distributed-cache name="actionTokens" owners="2" statistics="true">
<encoding>
<key media-type="application/x-java-object"/>
<value media-type="application/x-java-object"/>
</encoding>
<expiration lifespan="-1"/>
<persistence passivation="false" availability-interval="500"/>
<state-transfer enabled="false"/>
</distributed-cache>
</cache-container>
</infinispan>