[18.0.0 Quarkus] Trouble configuring Keycloak/hotrod for remote Infinispan cluster

Configuring the server
1

Hi all, I’ve been looking at the server configuration guide and the quarkus guides.

My goal is to have 3 instances of keycloak connected to 3 instances of infinispan, using SCRAM-SHA-256 with mutual TLS. All running in docker via docker-compose locally, but will be deployed on a kubernetes cluster.

How do I configure hotrod in quarkus? Is this how I should be connecting Keycloak.X/Infinispan?

It doesn’t seem to be reading the properties from keycloak’s cache configuration.xml. For example, it’s trying to connect via DIGEST instead of SCRAM-SHA-512. Also for this current config I get this error:

 ELY05087: Client selected realm not offered by server (AllowScriptManager)

My startup command for keycloak:

  ...

  ./bin/kc.sh start \
    --cache=ispn \
    --cache-stack=kubernetes \
    --cache-config-file="$cache_config_file" \
    --auto-build \
    --http-enabled 'true' \
    --hostname "$HOSTNAME" \
    --proxy 'passthrough' \
    --db postgres \
    --db-url-host "$DATABASE_HOST" \
    --db-url-port "$DATABASE_PORT" \
    --db-url-database "$DATABASE_NAME" \
    --db-username "$DATABASE_USERNAME" \
    --db-password "$DATABASE_PASSWORD" \
    --db-url-properties "$DATABASE_PROPERTIES" \
    --log console,file \
    --spi-theme-cache-themes=false \
    --spi-theme-cache-templates=false

my keycloak cache configuration.xml:

<?xml version="1.0" encoding="UTF-8"?>
<infinispan
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xsi:schemaLocation="urn:infinispan:config:13.0 http://www.infinispan.org/schemas/infinispan-config-13.0.xsd"
        xmlns="urn:infinispan:config:13.0">

    <!-- We can define custom networking stacks here. -->
    <!--jgroups>
      <stack name="test">
        <remote-sites>
          <remote-site name="ispn" />
        </remote-sites>
      </stack>
      <outbound-socket-binding name="remote-cache"> 
        <remote-destination host="${remote.cache.host:server_hostname}" 
                            port="${remote.cache.port:11222}"/> 
      </outbound-socket-binding>
    </jgroups-->

    <cache-container name="keycloak">
        <transport lock-timeout="60000"/>

        <local-cache name="realms">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <memory max-count="10000"/>
        </local-cache>

        <local-cache name="users">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <memory max-count="10000"/>
        </local-cache>

        <local-cache name="keys">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <expiration max-idle="3600000"/>
            <memory max-count="1000"/>
        </local-cache>

        <local-cache name="authorization">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <memory max-count="10000"/>
        </local-cache>

        <replicated-cache name="work">
            <expiration lifespan="-1"/>
        </replicated-cache>

        <distributed-cache name="sessions" owners="2">
          <expiration lifespan="-1"/>
          <remote-store xmlns="urn:infinispan:config:store:remote:13.0"
                        cache="sessions"
                        fetch-state="false"
                        purge="false"
                        preload="false"
                        shared="true"
                        segmented="false"
                        connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}">

              <remote-server host="ispn" port="${infinispan.bind.port:11222}"/>

              <!--
              <property name="rawValues">true</property>
              <property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>

              <property name="infinispan.client.hotrod.auth_username">user</property>
              <property name="infinispan.client.hotrod.auth_password">user</property>
              -->

              <property name="infinispan.client.hotrod.use_ssl">true</property>
              <property name="infinispan.client.hotrod.auth_realm">trust-store-realm</property>
              <property name="infinispan.client.hotrod.auth_server_name">infinispan</property>
              <property name="infinispan.client.hotrod.sasl_mechanism">SCRAM-SHA-512</property>
              <property name="infinispan.client.hotrod.trust_store_file_name">/opt/keycloak/conf/ispn-truststore.jks</property>
              <property name="infinispan.client.hotrod.trust_store_type">PKCS12</property>
              <property name="infinispan.client.hotrod.trust_store_password">changeit</property>

              <security>
                <authentication>
                  <external/>
                </authentication>
                <encryption sni-hostname="keycloak">
                  <keystore filename="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_PATH:/opt/keycloak/conf/keycloak-keystore.pkcs12}"
                              password="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_PASSWORD:changeit}"
                              type="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_TYPE:PKCS12}"/>

                  <truststore filename="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_PATH:/opt/keycloak/conf/keycloak-keystore.pkcs12}"
                              password="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_PASSWORD:changeit}"
                              type="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_TYPE:PKCS12}"/>
                </encryption>
              </security>
          </remote-store>
        </distributed-cache>

        <distributed-cache name="authenticationSessions" owners="2">
            <expiration lifespan="-1"/>
        </distributed-cache>

        <distributed-cache name="offlineSessions" owners="2">
            <expiration lifespan="-1"/>
        </distributed-cache>

        <distributed-cache name="clientSessions" owners="2">
            <expiration lifespan="-1"/>
        </distributed-cache>

        <distributed-cache name="offlineClientSessions" owners="2">
            <expiration lifespan="-1"/>
        </distributed-cache>

        <distributed-cache name="loginFailures" owners="2">
            <expiration lifespan="-1"/>
        </distributed-cache>

        <distributed-cache name="actionTokens" owners="2">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <expiration max-idle="-1" lifespan="-1" interval="300000"/>
            <memory max-count="-1"/>
        </distributed-cache>
    </cache-container>
</infinispan>

and my infinispan configuration file:

<infinispan
      xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
      xsi:schemaLocation="urn:infinispan:config:13.0 https://infinispan.org/schemas/infinispan-config-13.0.xsd
                            urn:infinispan:server:13.0 https://infinispan.org/schemas/infinispan-server-13.0.xsd"
      xmlns="urn:infinispan:config:13.0"
      xmlns:server="urn:infinispan:server:13.0">

    <!--jgroups>
      <stack name="global-cluster" extends="tcp">
        <!- Remove MPING protocol from the stack and add TCPPING ->
        <TCPPING initial_hosts="ispn[7800]"
                 stack.combine="REPLACE" stack.position="MPING"/>
      </stack>

      <stack name="xsite" extends="udp">
        <relay.RELAY2 site="site1"
                      max_site_masters="1000"/>

          <remote-sites default-stack="global-cluster">
            <remote-site name="ispn"/>
          </remote-sites>
      </stack>
    </jgroups-->

    <cache-container name="default" statistics="true">
      <transport
        cluster="${infinispan.cluster.name:REMOTE}"
        stack="${infinispan.cluster.stack:tcp}"
        node-name="${infinispan.node.name:}"/>

      <replicated-cache-configuration name="replicated-cache-cfg">
        <encoding>
          <key media-type="application/x-protostream"/>
          <value media-type="application/x-protostream"/>
        </encoding>

        <expiration lifespan="900000000000000000"/>
      </replicated-cache-configuration>

      <distributed-cache-configuration name="distributed-cache-cfg">
        <!--encoding>
          <key media-type="application/x-protostream"/>
          <value media-type="application/x-protostream"/>
        </encoding-->

        <expiration lifespan="900000000000000000"/>
      </distributed-cache-configuration>

      <replicated-cache name="work" configuration="replicated-cache-cfg">
      </replicated-cache>

      <distributed-cache name="sessions" owners="2" configuration="distributed-cache-cfg">
      </distributed-cache>

      <distributed-cache name="authenticationSessions" owners="2" configuration="distributed-cache-cfg">
      </distributed-cache>

      <distributed-cache name="offlineSessions" owners="2" configuration="distributed-cache-cfg">
      </distributed-cache>

      <distributed-cache name="clientSessions" owners="2" configuration="distributed-cache-cfg">
      </distributed-cache>

      <distributed-cache name="offlineClientSessions" owners="2" configuration="distributed-cache-cfg">
      </distributed-cache>

      <distributed-cache name="loginFailures" owners="2" configuration="distributed-cache-cfg">
      </distributed-cache>

      <distributed-cache name="actionTokens" owners="2" configuration="distributed-cache-cfg">
        <memory max-count="-1">
        </memory>
        <expiration interval="300000" max-idle="-1"/>
        <security>
          <authorization/>
        </security>
      </distributed-cache>

      <security>
        <authorization>
          <common-name-role-mapper/>
          <role name="Andrew" permissions="ALL"/>
        </authorization>
      </security>
    </cache-container>

  <server xmlns="urn:infinispan:server:13.0">
    <interfaces>
        <interface name="public">
          <inet-address value="${infinispan.bind.address:0.0.0.0}"/>
          <!--<any-address/>-->
        </interface>
    </interfaces>

    <socket-bindings default-interface="public" port-offset="${infinispan.socket.binding.port-offset:0}">
      <socket-binding name="default" port="${infinispan.bind.port:11222}"/>
      <socket-binding name="memcached" port="11221"/>
    </socket-bindings>

    <security>
      <security-realms>
        <security-realm name="trust-store-realm">
          <server-identities>
            <ssl>
              <keystore path="ispn-truststore.pkcs12" relative-to="infinispan.server.config.path"
                password="changeit" alias="ispn" key-password="changeit" type="PKCS12"/>
              <!--generate-self-signed-certificate-host="ispn"/-->

              <truststore path="ispn-truststore.pkcs12" relative-to="infinispan.server.config.path"
                password="changeit"/>
            </ssl>
          </server-identities>
          <truststore-realm/>
          <properties-realm groups-attribute="Roles">
            <user-properties path="users.properties" relative-to="infinispan.server.config.path"
                             plain-text="true"/>
            <group-properties path="groups.properties" relative-to="infinispan.server.config.path"/>
          </properties-realm>
        </security-realm>
      </security-realms>
    </security>

    <endpoints>
      <endpoint socket-binding="default" security-realm="trust-store-realm" require-ssl-client-auth="true">
        <hotrod-connector name="hotrod">
          <authentication>
            <sasl mechanisms="EXTERNAL DIGEST-MD5 PLAIN SCRAM-SHA-512" server-name="infinispan" qop="auth" />
          </authentication>
          <!--encryption>
          </encryption-->
        </hotrod-connector>
        <rest-connector name="rest">
          <authentication mechanisms="CLIENT_CERT" />
        </rest-connector>
      </endpoint>
    </endpoints>

  <!--<endpoints socket-binding="default"/>-->
    <!--<endpoints socket-binding="default" security-realm="default" />-->
    <!--<endpoints>
      <endpoint socket-binding="default" security-realm="default">
        <hotrod-connector security-realm="default">
          <authentication>
            <sasl mechanisms="SCRAM-SHA-512 SCRAM-SHA-384 SCRAM-SHA-256
                             SCRAM-SHA-1 DIGEST-SHA-512 DIGEST-SHA-384
                             DIGEST-SHA-256 DIGEST-SHA DIGEST-MD5 PLAIN"
                  server-name="infinispan"
                  qop="auth"/>
          </authentication>
        </hotrod-connector>
        <rest-connector></rest-connector>
      </endpoint>
    </endpoints>-->
  </server>
</infinispan>


services:
  keycloak:
    image: local/keycloak:18.0.2-quarkus-centos8stream
    environment:
      KEYCLOAK_STORE_PASS: 'changeit'
      HOSTNAME: 'keycloak.local'
      KEYCLOAK_FRONTEND_URL: 'https://keycloak.local'
      COPY_ENV_VARS: 1
      PROXY_ADDRESS_FORWARDING: 'true'
      KEYCLOAK_ADMIN: 'admin'
      KEYCLOAK_ADMIN_PASSWORD: 'admin'
      #
      KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_PASSWORD: 'changeit'
      #KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_TYPE: 'PKCS12'
      # cluster
      JGROUPS_DISCOVERY_PROTOCOL: 'dns.DNS_PING'
      JGROUPS_DISCOVERY_PROPERTIES: 'dns_query=keycloak'
      JAVA_OPTS_APPEND: '-Djgroups.dns.query=keycloak'
      JGROUPS_DISCOVERY_PROPERTIES: 'keycloak1'
      # db
      DATABASE_HOST: 'postgres'
      DATABASE_NAME: 'keycloak'
      DATABASE_PORT: '5432'
      DATABASE_USERNAME: 'keycloak'
      DATABASE_PASSWORD: 'keycloak'
      DATABASE_PROPERTIES: '?sslmode=disable'
      INFINISPAN_CONFIG_FILE: 'cache-ispn-remote.xml'
    volumes:
      - "./certs/keycloak-keystore.pkcs12:/opt/keycloak/conf/keycloak-keystore.pkcs12"
      - "./cache-ispn-remote.xml:/opt/keycloak/conf/cache-ispn-remote.xml"
      #- "./infinispan/ispn-truststore.jks:/opt/keycloak/conf/truststore.jks"
      #- "./cache-custom-jgroups.xml:/opt/keycloak/conf/cache-custom-jgroups.xml"
      #- "./jgroups.xml:/opt/keycloak/conf/jgroups.xml"
      #deploy:
      #  replicas: 1
      #  endpoint_mode: dnsrr
    ports:
      - "8080:8080"
      - "8443"

  ispn:
    image: local/infinispan:13.0.10.Final-minimal
    ports:
      - "7800"
      - "11222"
    environment:
      INFINISPAN_SERVER: 'ispn'
      JGROUPS_DISCOVERY_PROTOCOL: 'dns.DNS_PING'
      JGROUPS_DISCOVERY_PROPERTIES: 'dns_query=ispn'
      JGROUPS_DISCOVERY_QUERY: 'ispn'
    volumes:
      #- "./keycloak.crt:/opt/infinispan/server/conf/keycloak.crt"
      #- "./certs/ispn-server.jks:/opt/infinispan/server/conf/ispn-server.jks"
      - "./certs/ispn-truststore.pkcs12:/opt/infinispan/server/conf/ispn-truststore.pkcs12"
      - "./infinispan/infinispan.xml:/opt/infinispan/server/conf/infinispan.xml"
    deploy:
      replicas: 3
      endpoint_mode: dnsrr
2

Anything? I can’t believe I’m the only one who’s run into this problem.

1 Like
3

Did you figure out a solution for the same?

4

The issue is your hotrod connection in keycloak is trying to access a realm that doesn’t exist. I managed to get around this by setting the security realm via the below, it’s the digest block:

            <remote-store cache="sessions"
                          fetch-state="false"
                          purge="false"
                          preload="false"
                          shared="true" segmented="false"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}"
                          xmlns="urn:infinispan:config:store:remote:13.0" >
                <remote-server host="hosthere" port="${infinispan.bind.port:11222}"/>
                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:test}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:test}"
                                realm="default"/>
                    </authentication>
                </security>

                <property name="rawValues">true</property>
                <property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>

            </remote-store>