We are setting up 3 instances of Keycloak v19.03 with 3 separate VM instances behind a load balancer. The Keycloak instances are wired up to an external Infinispan v14 cache running as a cluster on two separate VM’s. There is no session stickiness configured at the load balancer.
It seems to be working fine except for when we login via an external IDP. We have Azure AD setup as an keycloak-oidc IDP. When we try to login we get usually one of the two following errors:
Not found serialized context in authenticationSession.
or
Page has expired
The current URL is: https://<keycloak_host>/auth/realms//login-actions/first-broker-login?client_id=myclient&tab_id=kFTS0zbgmG4
The Keylcloak logs show up errors accross the different nodes:
ERROR [org.keycloak.services] (executor-thread-56) KC-SERVICES0068: Not found serialized context in clientSession under note 'BROKERED_CONTEXT'
WARN [org.keycloak.protocol.oidc.utils.AcrUtils] (executor-thread-53) Invalid realm configuration (ACR-LOA map)
WARN [org.keycloak.cluster.infinispan.InfinispanNotificationsManager] (kc-work-cache-event-listener-884-0) Event object wasn't available in remote cache after event was received. Event key: eeb7f8a1-b14e-4354-83ed-6b8f51689d53
The “Email as Username” option is configured in the realm as that is what we need.
The same configuration working with one Keycloak server with imbedded cache does not have any problems.
This is our Keycloak ISPN configuration:
<?xml version="1.0" encoding="UTF-8"?>
<infinispan
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="urn:infinispan:config:11.0 http://www.infinispan.org/schemas/infinispan-config-11.0.xsd"
xmlns="urn:infinispan:config:11.0">
<cache-container name="keycloak">
<transport lock-timeout="60000" site="${env.KC_ISPN_SITE_NAME}"/>
<local-cache name="realms">
<encoding>
<key media-type="application/x-java-object" />
<value media-type="application/x-java-object" />
</encoding>
<memory max-count="10000" />
</local-cache>
<local-cache name="users">
<encoding>
<key media-type="application/x-java-object" />
<value media-type="application/x-java-object" />
</encoding>
<memory max-count="10000" />
</local-cache>
<local-cache name="authorization">
<encoding>
<key media-type="application/x-java-object" />
<value media-type="application/x-java-object" />
</encoding>
<memory max-count="10000" />
</local-cache>
<local-cache name="keys">
<encoding>
<key media-type="application/x-java-object" />
<value media-type="application/x-java-object" />
</encoding>
<expiration max-idle="3600000" />
<memory max-count="1000" />
</local-cache>
<replicated-cache name="work">
<expiration lifespan="-1" />
<remote-store cache="work" xmlns="urn:infinispan:config:store:remote:11.0" fetch-state="false" purge="false" preload="false" shared="true" segmented="false" connect-timeout="${env.KC_ISPN_REMOTE_CONN_TIMEOUT:2000}">
<remote-server host="${env.KC_ISPN_REMOTE_HOST_1}" port="${env.KC_ISPN_REMOTE_PORT}" />
<remote-server host="${env.KC_ISPN_REMOTE_HOST_2}" port="${env.KC_ISPN_REMOTE_PORT}" />
<security>
<authentication>
<digest username="${env.KC_ISPN_REMOTE_USERNAME:keycloak}" password="${env.KC_ISPN_REMOTE_PASSWORD:password}" realm="default" />
</authentication>
</security>
<property name="rawValues">true</property>
<property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>
</remote-store>
</replicated-cache>
<distributed-cache name="sessions" owners="2">
<expiration lifespan="-1" />
<remote-store cache="sessions" xmlns="urn:infinispan:config:store:remote:11.0" fetch-state="false" purge="false" preload="false" shared="true" segmented="false" connect-timeout="${env.KC_ISPN_REMOTE_CONN_TIMEOUT:2000}">
<remote-server host="${env.KC_ISPN_REMOTE_HOST_1}" port="${env.KC_ISPN_REMOTE_PORT}" />
<remote-server host="${env.KC_ISPN_REMOTE_HOST_2}" port="${env.KC_ISPN_REMOTE_PORT}" />
<security>
<authentication>
<digest username="${env.KC_ISPN_REMOTE_USERNAME:keycloak}" password="${env.KC_ISPN_REMOTE_PASSWORD:password}" realm="default" />
</authentication>
</security>
<property name="rawValues">true</property>
<property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>
</remote-store>
</distributed-cache>
<distributed-cache name="authenticationSessions" owners="2">
<expiration lifespan="-1" />
<remote-store cache="authenticationSessions" xmlns="urn:infinispan:config:store:remote:11.0" fetch-state="false" purge="false" preload="false" shared="true" segmented="false" connect-timeout="${env.KC_ISPN_REMOTE_CONN_TIMEOUT:2000}">
<remote-server host="${env.KC_ISPN_REMOTE_HOST_1}" port="${env.KC_ISPN_REMOTE_PORT}" />
<remote-server host="${env.KC_ISPN_REMOTE_HOST_2}" port="${env.KC_ISPN_REMOTE_PORT}" />
<security>
<authentication>
<digest username="${env.KC_ISPN_REMOTE_USERNAME:keycloak}" password="${env.KC_ISPN_REMOTE_PASSWORD:password}" realm="default" />
</authentication>
</security>
<property name="rawValues">true</property>
<property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>
</remote-store>
</distributed-cache>
<distributed-cache name="offlineSessions" owners="2">
<expiration lifespan="-1" />
<remote-store cache="offlineSessions" xmlns="urn:infinispan:config:store:remote:11.0" fetch-state="false" purge="false" preload="false" shared="true" segmented="false" connect-timeout="${env.KC_ISPN_REMOTE_CONN_TIMEOUT:2000}">
<remote-server host="${env.KC_ISPN_REMOTE_HOST_1}" port="${env.KC_ISPN_REMOTE_PORT}" />
<remote-server host="${env.KC_ISPN_REMOTE_HOST_2}" port="${env.KC_ISPN_REMOTE_PORT}" />
<security>
<authentication>
<digest username="${env.KC_ISPN_REMOTE_USERNAME:keycloak}" password="${env.KC_ISPN_REMOTE_PASSWORD:password}" realm="default" />
</authentication>
</security>
<property name="rawValues">true</property>
<property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>
</remote-store>
</distributed-cache>
<distributed-cache name="clientSessions" owners="2">
<expiration lifespan="-1" />
<remote-store cache="clientSessions" xmlns="urn:infinispan:config:store:remote:11.0" fetch-state="false" purge="false" preload="false" shared="true" segmented="false" connect-timeout="${env.KC_ISPN_REMOTE_CONN_TIMEOUT:2000}">
<remote-server host="${env.KC_ISPN_REMOTE_HOST_1}" port="${env.KC_ISPN_REMOTE_PORT}" />
<remote-server host="${env.KC_ISPN_REMOTE_HOST_2}" port="${env.KC_ISPN_REMOTE_PORT}" />
<security>
<authentication>
<digest username="${env.KC_ISPN_REMOTE_USERNAME:keycloak}" password="${env.KC_ISPN_REMOTE_PASSWORD:password}" realm="default" />
</authentication>
</security>
<property name="rawValues">true</property>
<property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>
</remote-store>
</distributed-cache>
<distributed-cache name="offlineClientSessions" owners="2">
<expiration lifespan="-1" />
<remote-store cache="offlineClientSessions" xmlns="urn:infinispan:config:store:remote:11.0" fetch-state="false" purge="false" preload="false" shared="true" segmented="false" connect-timeout="${env.KC_ISPN_REMOTE_CONN_TIMEOUT:2000}">
<remote-server host="${env.KC_ISPN_REMOTE_HOST_1}" port="${env.KC_ISPN_REMOTE_PORT}" />
<remote-server host="${env.KC_ISPN_REMOTE_HOST_2}" port="${env.KC_ISPN_REMOTE_PORT}" />
<security>
<authentication>
<digest username="${env.KC_ISPN_REMOTE_USERNAME:keycloak}" password="${env.KC_ISPN_REMOTE_PASSWORD:password}" realm="default" />
</authentication>
</security>
<property name="rawValues">true</property>
<property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>
</remote-store>
</distributed-cache>
<distributed-cache name="loginFailures" owners="2">
<expiration lifespan="-1" />
<remote-store cache="loginFailures" xmlns="urn:infinispan:config:store:remote:11.0" fetch-state="false" purge="false" preload="false" shared="true" segmented="false" connect-timeout="${env.KC_ISPN_REMOTE_CONN_TIMEOUT:2000}">
<remote-server host="${env.KC_ISPN_REMOTE_HOST_1}" port="${env.KC_ISPN_REMOTE_PORT}" />
<remote-server host="${env.KC_ISPN_REMOTE_HOST_2}" port="${env.KC_ISPN_REMOTE_PORT}" />
<security>
<authentication>
<digest username="${env.KC_ISPN_REMOTE_USERNAME:keycloak}" password="${env.KC_ISPN_REMOTE_PASSWORD:password}" realm="default" />
</authentication>
</security>
<property name="rawValues">true</property>
<property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>
</remote-store>
</distributed-cache>
<distributed-cache name="actionTokens" owners="2">
<encoding>
<key media-type="application/x-java-object" />
<value media-type="application/x-java-object" />
</encoding>
<expiration max-idle="-1" lifespan="-1" interval="300000" />
<memory max-count="-1" />
<remote-store cache="actionTokens" xmlns="urn:infinispan:config:store:remote:11.0" fetch-state="false" purge="false" preload="false" shared="true" segmented="false" connect-timeout="${env.KC_ISPN_REMOTE_CONN_TIMEOUT:2000}">
<remote-server host="${env.KC_ISPN_REMOTE_HOST_1}" port="${env.KC_ISPN_REMOTE_PORT}" />
<remote-server host="${env.KC_ISPN_REMOTE_HOST_2}" port="${env.KC_ISPN_REMOTE_PORT}" />
<security>
<authentication>
<digest username="${env.KC_ISPN_REMOTE_USERNAME:keycloak}" password="${env.KC_ISPN_REMOTE_PASSWORD:password}" realm="default" />
</authentication>
</security>
<property name="rawValues">true</property>
<property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>
</remote-store>
</distributed-cache>
</cache-container>
</infinispan>
And our remote Infinispan configuration:
<infinispan xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:infinispan:config:14.0 https://infinispan.org/schemas/infinispan-config-14.0.xsd urn:infinispan:server:14.0 https://infinispan.org/schemas/infinispan-server-14.0.xsd" xmlns="urn:infinispan:config:14.0" xmlns:server="urn:infinispan:server:14.0">
<!--Tcpping For Infinispan cluster discovery -->
<jgroups>
<stack name="tcpping" extends="tcp">
<TCP bind_port="7800" />
<TCPPING initial_hosts="${env.JGROUPS_TCPPING_INITIAL_HOSTS:x.x.x.x[7800],x.x.x.x[7800]}" port_range="0" stack.combine="REPLACE" stack.position="MPING" />
</stack>
</jgroups>
<cache-container name="default" statistics="true">
<serialization marshaller="org.infinispan.jboss.marshalling.commons.GenericJBossMarshaller">
<allow-list>
<class>org.keycloak.cluster.infinispan.WrapperClusterEvent</class>
<regex>.*</regex>
</allow-list>
</serialization>
<transport cluster="${env.ISPN_CLUSTER_NAME:ispn-cluster}" stack="tcpping" node-name="${env.ISPN_NODE_NAME:node-1}" />
<replicated-cache-configuration name="default-replcache-cfg" mode="SYNC" start="EAGER" statistics="true" />
<replicated-cache name="work" configuration="default-replcache-cfg">
<encoding media-type="application/x-jboss-marshalling"/>
</replicated-cache>
<replicated-cache name="sessions" configuration="default-replcache-cfg">
<encoding media-type="application/x-jboss-marshalling"/>
</replicated-cache>
<replicated-cache name="clientSessions" configuration="default-replcache-cfg">
<encoding media-type="application/x-jboss-marshalling"/>
</replicated-cache>
<replicated-cache name="offlineSessions" configuration="default-replcache-cfg">
<encoding media-type="application/x-jboss-marshalling"/>
</replicated-cache>
<replicated-cache name="offlineClientSessions" configuration="default-replcache-cfg">
<encoding media-type="application/x-jboss-marshalling"/>
</replicated-cache>
<replicated-cache name="authenticationSessions" configuration="default-replcache-cfg">
<encoding media-type="application/x-jboss-marshalling"/>
</replicated-cache>
<replicated-cache name="actionTokens" configuration="default-replcache-cfg">
<encoding media-type="application/x-jboss-marshalling"/>
</replicated-cache>
<replicated-cache name="loginFailures" configuration="default-replcache-cfg">
<encoding media-type="application/x-jboss-marshalling"/>
</replicated-cache>
</cache-container>
<server xmlns="urn:infinispan:server:14.0">
<interfaces>
<interface name="public">
<inet-address value="${env.ISPN_BIND_ADDRESS}" />
</interface>
</interfaces>
<socket-bindings default-interface="public" port-offset="${env.ISPN_SOCKET_BIND_PORT_OFFSET:0}">
<socket-binding name="default" port="${env.ISPN_BIND_PORT:11222}" />
<socket-binding name="memcached" port="11221" />
</socket-bindings>
<security>
<security-realms>
<security-realm name="default">
<!-- Uncomment to enable TLS on the realm -->
<!-- server-identities>
<ssl>
<keystore path="application.keystore" password="password" alias="server" generate-self-signed-certificate-host="localhost"/>
</ssl>
</server-identities-->
<properties-realm groups-attribute="Roles">
<user-properties path="users.properties" />
<group-properties path="groups.properties" />
</properties-realm>
</security-realm>
</security-realms>
</security>
<endpoints socket-binding="default" security-realm="default" />
</server>
</infinispan>
Has anyone come across this ?
Logging in using the username and password form seems to work without any problems so far. The same setup (connecting to the same Keycloak instance) but with one Keycloak node and imbedded infinispan works without any problems.
The custom ispn configuration we are using above uses the same site-name value across all 3 Keycloak nodes. Not sure if this needs to be unique per node and is maybe contributing to the issue.