Jitsi SSL Offload HAProxy Not Working

[yuljk]

Hi guys - I already have a working HAProxy setup running on pfsense for various web services etc. All pointing to http backends.

I'm trying to do likewise for my Jitsi server, but I get a '503 service unavailable' and the backend is down in HAProxy stats.

Has anyone been able to get Jitsi working behind HAProxy?

Here's my config.

# Automaticaly generated, dont edit manually.
# Generated on: 2020-04-20 16:52
global
	maxconn			100
	stats socket /tmp/haproxy.socket level admin  expose-fd listeners
	uid			80
	gid			80
	nbproc			1
	nbthread			1
	hard-stop-after		15m
	chroot				/tmp/haproxy_chroot
	daemon
	tune.ssl.default-dh-param	2048
	server-state-file /tmp/haproxy_server_state
	ssl-default-bind-ciphers ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256
	ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets
	ssl-default-server-ciphers ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256
	ssl-default-server-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets

listen HAProxyLocalStats
	bind 127.0.0.1:2200 name localstats
	mode http
	stats enable
	stats admin if TRUE
	stats show-legends
	stats uri /haproxy/haproxy_stats.php?haproxystats=1
	timeout client 5000
	timeout connect 5000
	timeout server 5000

frontend HTTPS-Edge-SNI
	bind			127.0.0.1:1443 name 127.0.0.1:1443   ssl crt-list /var/etc/haproxy/HTTPS-Edge-SNI.crt_list  
	bind			someip:443 name someip:443   ssl crt-list /var/etc/haproxy/HTTPS-Edge-SNI.crt_list  
	bind /tmp/haproxy_chroot/HTTPS-Edge-SNI.socket name unixsocket uid 80 accept-proxy   ssl crt-list /var/etc/haproxy/HTTPS-Edge-SNI.crt_list 
	mode			http
	log			global
	option			http-keep-alive
	timeout client		30000
	acl			mail-acl	var(txn.txnhost) -m str -i mail.some-domain.com
	acl			monitor-acl	var(txn.txnhost) -m str -i monitor.some-domain.com
	acl			filter-acl	var(txn.txnhost) -m str -i filter.some-domain.com
	acl			autodiscover-acl	var(txn.txnhost) -m str -i autodiscover.some-domain.com
	acl			cloud-acl	var(txn.txnhost) -m str -i cloud.some-domain.com
	acl			invoice-acl	var(txn.txnhost) -m str -i invoice.domedomain.com
	acl			blog-acl	var(txn.txnhost) -m str -i blog.some-domain.com
	acl			www-acl	var(txn.txnhost) -m str -i www.somedomain.com
	acl			www-redirect	var(txn.txnhost) -m str -i somedomain.com
	acl			mail-owa	var(txn.txnpath) -m beg -i /owa
	acl			mail-ecp	var(txn.txnpath) -m beg -i /ecp
	acl			mail-mapi	var(txn.txnpath) -m beg -i /mapi
	acl			mail-ews	var(txn.txnpath) -m beg -i /EWS
	acl			mail-oab	var(txn.txnpath) -m beg -i /OAB
	acl			mail-activesync	var(txn.txnpath) -m beg -i /Microsoft-Server-ActiveSync
	acl			mail-rpc	var(txn.txnpath) -m beg -i /rpc/rpcproxy.dll
	acl			mail-autodiscover	var(txn.txnpath) -m beg -i /Autodiscover
	acl			mail-healthcheck	var(txn.txnpath) -m end -i HealthCheck.htm
	acl			source-internal	src somerange/24
	acl			mail-owa-redirect	var(txn.txnpath) -m str -i / /owa
	acl			chat-acl	var(txn.txnhost) -m str -i chat.some-domain.com
	acl			conference-acl	var(txn.txnhost) -m str -i conference.some-domain.com
	http-request set-var(txn.txnhost) hdr(host)
	http-request set-var(txn.txnpath) path
	http-request deny   if  mail-acl mail-healthcheck 
	http-request redirect prefix https://www.somedomain.com  if  www-redirect 
	http-request redirect location https://mail.some-domain.com/owa/  if  mail-acl mail-owa-redirect 
	use_backend monitor_ipvANY  if  monitor-acl 
	use_backend filter_ipvANY  if  filter-acl 
	use_backend cloud_ipvANY  if  cloud-acl 
	use_backend invoice_ipvANY  if  invoice-acl 
	use_backend blog_ipvANY  if  blog-acl 
	use_backend www_ipvANY  if  www-acl 
	use_backend mail-owa_ipvANY  if  mail-owa mail-acl 
	use_backend mail-ecp_ipvANY  if  mail-ecp mail-acl source-internal 
	use_backend mail-mapi_ipvANY  if  mail-mapi mail-acl 
	use_backend mail-ews_ipvANY  if  mail-ews mail-acl 
	use_backend mail-oab_ipvANY  if  mail-oab mail-acl 
	use_backend mail-activesync_ipvANY  if  mail-activesync mail-acl 
	use_backend mail-rpc_ipvANY  if  mail-rpc mail-acl 
	use_backend mail-autodiscover_ipvANY  if  mail-autodiscover mail-acl 
	use_backend mail-autodiscover_ipvANY  if  autodiscover-acl mail-autodiscover 
	use_backend chat_ipvANY  if  chat-acl 
	use_backend conference_ipvANY  if  conference-acl 

frontend HTTP-Edge
	bind			myip:80 name myip:80   
	mode			http
	log			global
	option			http-keep-alive
	timeout client		30000
	acl			mail-acl	var(txn.txnhost) -m str -i mail.some-domain.com
	acl			filter-acl	var(txn.txnhost) -m str -i filter.some-domain.com
	acl			cloud-acl	var(txn.txnhost) -m str -i cloud.some-domain.com
	acl			monitor-acl	var(txn.txnhost) -m str -i monitor.some-domain.com
	acl			www-acl	var(txn.txnhost) -m str -i www.somedomain.com
	acl			blog-acl	var(txn.txnhost) -m str -i blog.some-domain.con
	acl			www-redirect	var(txn.txnhost) -m str -i somedomain.com
	acl			invoice-acl	var(txn.txnhost) -m str -i invoice.somedomain.com
	http-request set-var(txn.txnhost) hdr(host)
	http-request redirect scheme https  if  mail-acl 
	http-request redirect scheme https  if  filter-acl 
	http-request redirect scheme https  if  cloud-acl 
	http-request redirect scheme https  if  monitor-acl 
	http-request redirect scheme https  if  www-acl 
	http-request redirect scheme https  if  blog-acl 
	http-request redirect prefix https://www.somedomain.com if  www-redirect 
	http-request redirect scheme https  if  invoice-acl 

frontend HTTPS-Edge-TCP
	bind			myip:443 name myip:443   
	mode			tcp
	log			global
	timeout client		14400000
	tcp-request inspect-delay	5s
	acl			netscaler-acl	req.ssl_sni -i somedomain.com
	acl			netscaler-acl	req.ssl_sni -i apps.somedomain.com:443
	acl			hasSNI	req.ssl_sni -m found
	tcp-request content accept if { req.ssl_hello_type 1 }
	use_backend netscaler_ipvANY  if  netscaler-acl || !hasSNI 
	default_backend defaultbackend_ipvANY

backend monitor_ipvANY
	mode			http
	id			113
	log			global
	timeout connect		30000
	timeout server		30000
	retries			3
	option			httpchk OPTIONS / 
	http-response add-header Content-Security-Policy upgrade-insecure-requests
	server			ICARUS ip:80 id 114 check inter 1000  

backend filter_ipvANY
	mode			http
	id			107
	log			global
	timeout connect		30000
	timeout server		30000
	retries			3
	http-response add-header Content-Security-Policy upgrade-insecure-requests
	server			STYX ip:80 id 108 check inter 1000  

backend cloud_ipvANY
	mode			http
	id			121
	log			global
	http-response set-header Strict-Transport-Security max-age=31536000;
	timeout connect		30000
	timeout server		30000
	retries			3
	option			httpchk OPTIONS / 
	http-response add-header Content-Security-Policy upgrade-insecure-requests
	server			OEDIPUS ip:80 id 122 check inter 1000  

backend invoice_ipvANY
	mode			http
	id			123
	log			global
	http-response set-header Strict-Transport-Security max-age=31536000;
	timeout connect		30000
	timeout server		30000
	retries			3
	option			httpchk OPTIONS / 
	server			CLYTIA ip:443 id 124 ssl check inter 1000  verify none 

backend blog_ipvANY
	mode			http
	id			117
	log			global
	http-response set-header Strict-Transport-Security max-age=31536000;
	timeout connect		30000
	timeout server		30000
	retries			3
	http-response add-header Content-Security-Policy upgrade-insecure-requests
	server			MINOTAUR ip:80 id 104 check inter 1000  

backend www_ipvANY
	mode			http
	id			103
	log			global
	http-response set-header Strict-Transport-Security max-age=31536000;
	timeout connect		30000
	timeout server		30000
	retries			3
	http-response add-header Content-Security-Policy upgrade-insecure-requests
	server			MINOTAUR ip:80 id 126 check inter 1000  

backend mail-owa_ipvANY
	mode			http
	id			109
	log			global
	timeout connect		30000
	timeout server		30000
	retries			3
	option			httpchk GET /owa/HealthCheck.htm 
	http-check expect string 200 OK
	server			EREBOS ip:80 id 112 check inter 1000  

backend mail-ecp_ipvANY
	mode			http
	id			102
	log			global
	timeout connect		30000
	timeout server		30000
	retries			3
	option			httpchk GET /ECP/HealthCheck.htm 
	http-check expect string 200 OK
	server			EREBOS ip:80 id 112 check inter 1000  

backend mail-mapi_ipvANY
	mode			http
	id			110
	log			global
	timeout connect		30000
	timeout server		30000
	retries			3
	option			httpchk GET /mapi/HealthCheck.htm 
	http-check expect string 200 OK
	server			EREBOS ip:80 id 112 check inter 1000  

backend mail-ews_ipvANY
	mode			http
	id			118
	log			global
	timeout connect		30000
	timeout server		30000
	retries			3
	option			httpchk GET /EWS/HealthCheck.htm 
	http-check expect string 200 OK
	server			EREBOS ip:80 id 112 check inter 1000  

backend mail-oab_ipvANY
	mode			http
	id			125
	log			global
	timeout connect		30000
	timeout server		30000
	retries			3
	option			httpchk GET /OAB/HealthCheck.htm 
	http-check expect string 200 OK
	server			EREBOS ip:80 id 112 check inter 1000  

backend mail-activesync_ipvANY
	mode			http
	id			127
	log			global
	timeout connect		30000
	timeout server		900000
	retries			3
	option			httpchk GET /Microsoft-Server-ActiveSync/HealthCheck.htm 
	http-check expect string 200 OK
	server			EREBOS ip:80 id 112 check inter 1000  

backend mail-rpc_ipvANY
	mode			http
	id			128
	log			global
	timeout connect		30000
	timeout server		30000
	retries			3
	option			httpchk GET /RPC/HealthCheck.htm 
	http-check expect string 200 OK
	server			EREBOS ip:80 id 112 check inter 1000  

backend mail-autodiscover_ipvANY
	mode			http
	id			129
	log			global
	timeout connect		30000
	timeout server		30000
	retries			3
	option			httpchk GET /Autodiscover/HealthCheck.htm 
	http-check expect string 200 OK
	server			EREBOS ip:80 id 112 check inter 1000  

backend chat_ipvANY
	mode			http
	id			100
	log			global
	http-response set-header Strict-Transport-Security max-age=31536000;
	timeout connect		30000
	timeout server		30000
	retries			3
	option			httpchk OPTIONS / 
	server			GERYON ip:3000 id 122 check inter 1000  

backend conference_ipvANY
	mode			http
	id			101
	log			global
	http-response set-header Strict-Transport-Security max-age=31536000;
	timeout connect		30000
	timeout server		30000
	retries			3
	option			httpchk OPTIONS / 
	server			GERYON ip:80 id 122 check inter 1000  

backend netscaler_ipvANY
	mode			tcp
	id			115
	log			global
	timeout connect		30000
	timeout server		14400000
	retries			10
	http-check expect status 403
	server			GANYMEDE ip:443 id 116 check inter 60000  

backend defaultbackend_ipvANY
	mode			tcp
	id			105
	log			global
	timeout connect		30000
	timeout server		30000
	retries			3
	server			defaultbackend /HTTPS-Edge-SNI.socket send-proxy-v2-ssl-cn id 106 check inter 1000

The frontend acl is 'conference-acl' and the backend is 'conference'

I have configured the following in /etc/jitsi/videobridge/sip-communicator-properties

org.ice4j.ice.harvest.NAT_HARVESTER_LOCAL_ADDRESS=localip
org.ice4j.ice.harvest.NAT_HARVESTER_PUBLIC_ADDRESS=FGDN

Kind Regards

[PiBa]

@yuljk
If the backend is 'down' according to haproxy then haproxy replies with 503 to the client so that part is 'as expected'.

The main question currently then is: 'why is the backend down' https://github.com/PiBa-NL/pfsense-haproxy-package-doc/wiki/haproxy_troubleshooting
What does the 'LastChk' column say on the stats page?

[yuljk]

Hi PiBa - Many thanks for the reply!

I've managed to fix this issue. The problem was caused by using Jitsi's embedded webserver during the installation, which didn't work atall when performing SSL offloading. This seems to be a common issue looking at their forums.

Instead I started again, this time installing Apache prior to the Jitsi installation. Jitsi then configured Apache2 accordingly.

I had to configure the backend in HAProxy for port 443 and now offloading is working correctly.

Here's my working apache2 config for reference.

<VirtualHost *:80>
    ServerName mydomain
    Redirect permanent / https://mydomain/
    RewriteEngine On
    RewriteCond %{HTTPS} off
    RewriteRule ^ https://%{HTTP_HOST}%{REQUEST_URI} [R=301,L]
</VirtualHost>

<VirtualHost *:443>

  ServerName mydomain

  SSLProtocol TLSv1 TLSv1.1 TLSv1.2
  SSLEngine on
  SSLProxyEngine on
  SSLCertificateFile /etc/jitsi/meet/mydomain.crt
  SSLCertificateKeyFile /etc/jitsi/meet/mydomain.key
  SSLCipherSuite "EECDH+ECDSA+AESGCM:EECDH+aRSA+AESGCM:EECDH+ECDSA+SHA256:EECDH+aRSA+SHA256:EECDH+ECDSA+SHA384:EECDH+ECDSA+SHA256:EECDH+aRSA+SHA384:EDH+aRSA+AESGCM:EDH+aRSA+SHA256:EDH+aRSA:EECDH:!aNULL:!eNULL:!MEDIUM:!LOW:!3DES:!MD5:!EXP:!PSK:!SRP:!DSS:!RC4:!SEED"
  SSLHonorCipherOrder on
  Header set Strict-Transport-Security "max-age=31536000"

  DocumentRoot "/usr/share/jitsi-meet"
  <Directory "/usr/share/jitsi-meet">
    Options Indexes MultiViews Includes FollowSymLinks
    AddOutputFilter Includes html
    AllowOverride All
    Order allow,deny
    Allow from all
  </Directory>

  ErrorDocument 404 /static/404.html

  Alias "/config.js" "/etc/jitsi/meet/conference.apollon-domain.co.uk-config.js"
  <Location /config.js>
    Require all granted
  </Location>

  Alias "/external_api.js" "/usr/share/jitsi-meet/libs/external_api.min.js"
  <Location /external_api.js>
    Require all granted
  </Location>

  ProxyPreserveHost on
  ProxyPass /http-bind http://localhost:5280/http-bind/
  ProxyPassReverse /http-bind http://localhost:5280/http-bind/

  RewriteEngine on
  RewriteRule ^/([a-zA-Z0-9]+)$ /index.html
</VirtualHost>

Kind Regards