SNI-based load balancing with HAProxy

In a bare-metal Openshift installation you need to use an external load balancer to access the API and other services. In my hone lab I also have a webserver accesible from the Internet. I also don’t want to terminate the TLS connections in the load balancer to keep using the existing certificates in my webserver and Openshift cluster.

With these requirements in mind, I chose HAProxy to be my frontend load balancer, so all the HTTPS connections to my public IP will be diverted to the appropriate server examining the SNI field in the TLS connection.

It took me a while to make the SNI filter to work correctly. I’ve left some debug logging commented out that helped me to understand better how it was filtering traffic.

If you use this example for Openshift, make sure the access to the port 22623 is restricted in the firewall to internal networks only.

#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    #
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    #
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log
    #
    log         127.0.0.1 local2

    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon

    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats

    # utilize system-wide crypto-policies
    ssl-default-bind-ciphers PROFILE=SYSTEM
    ssl-default-server-ciphers PROFILE=SYSTEM

#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000

# Frontends

frontend stats
    bind :::8080 interface br-lan
    stats enable
    stats uri /stats
    stats refresh 10s
    stats admin if LOCALHOST

frontend openshift-api-server
    bind :::6443
    default_backend openshift-api-server
    mode tcp
    option tcplog

frontend machine-config-server
    bind :::22623 interface br-dmz
    default_backend machine-config-server
    mode tcp
    option tcplog

frontend http
    bind :::80
    acl is_ocp_http hdr_end(host) -i .apps.ocp4.example.com
    use_backend openshift-http if is_ocp_http
    default_backend webserver-http

frontend https
    # DEBUG
    #log 127.0.0.1 local0 debug
    #log-format "%ci:%cp [%t] %ft %b/%s %Tw/%Tc/%Tt %B %ts %ac/%fc/%bc/%sc/%rc %sq/%bq ssl_fc_has_sni '%[ssl_fc_has_sni]' sni:'%[capture.req.hdr(0)]' ssl_fc_sni '%[ssl_fc_sni]' ssl_fc_protocol '%[ssl_fc_protocol]' ssl_bc '%[ssl_bc]' ssl_bc_alpn '%[ssl_bc_alpn]' ssl_bc_protocol '%[ssl_bc_protocol]' ssl_c_i_dn '%[ssl_c_i_dn()]' ssl_c_s_dn '%[ssl_c_s_dn()]' ssl_f_i_dn '%[ssl_f_i_dn()]' ssl_f_s_dn '%[ssl_f_s_dn]' ssl_fc_cipher '%[ssl_fc_cipher]' "
    bind :::443
    mode tcp
    option tcplog
    tcp-request inspect-delay 5s
    tcp-request content capture req.ssl_sni len 250
    tcp-request content accept if { req.ssl_hello_type 1 }
    acl is_ocp_https req.ssl_sni -m end .apps.ocp4.example.com
    use_backend openshift-https if is_ocp_https
    default_backend webserver-https

# Backends

backend openshift-api-server
    balance source
    mode tcp
    server bootstrap bootstrap.lan:6443 check
    server master0 master0.lan:6443 check
    server master1 master1.lan:6443 check
    server master2 master2.lan:6443 check

backend machine-config-server
    balance source
    mode tcp
    server bootstrap bootstrap.lan:22623 check
    server master0 master0.lan:22623 check
    server master1 master1.lan:22623 check
    server master2 master2.lan:22623 check

backend webserver-http
    server webserver webserver.lan:80

backend openshift-http
    server bootstrap bootstrap.lan:80 check
    server master0 master0.lan:80 check
    server master1 master1.lan:80 check
    server master2 master2.lan:80 check

backend webserver-https
    mode tcp
    balance source
    server webserver webserver.lan:443 check sni req.ssl_sni

backend openshift-https
    mode tcp
    balance source
    server bootstrap bootstrap.lan:443 check sni req.ssl_sni
    server master0 master0.lan:443 check sni req.ssl_sni
    server master1 master1.lan:443 check sni req.ssl_sni
    server master2 master2.lan:443 check sni req.ssl_sni

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out /  Change )

Google photo

You are commenting using your Google account. Log Out /  Change )

Twitter picture

You are commenting using your Twitter account. Log Out /  Change )

Facebook photo

You are commenting using your Facebook account. Log Out /  Change )

Connecting to %s