Varnish help on setting up for multiple internal webservers

[marcelloc]

Yes.  :)

Just don't forget to check backend status on Dashboard.

[normtodd]

im having a strange issue that im sure is something easy that im not doing - im pretty sure everthing is working vis a vis varnish, but i am not getting any data on the varnish widget except that the backend is up and running - everything else (cache hits, cache hits pass, cache missed, conn accepted, req. received, and uptime) are empty.  ive attached a picture of the widget from the dashboard to illustrate.

right now i only have one machine (an atmail mail server) behind pfsense/varnish and want to confirm that everything is working before i deploy more machines behind pfsense.

here is my varnish config -

# Varnish configuration file
# Automatically generated by the pfSense package system

# This file is located in /var/etc/default.vcl

sub vcl_error {
		if (obj.status == 503 && req.restarts < 1) {
			return(restart);
		}

	set obj.http.Content-Type = "text/html; charset=utf-8";
	synthetic {"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

<title>"} + obj.status + " " + obj.response + {"</title>

# Error "} + obj.status + " " + obj.response + {"

"} + obj.response + {"

### Guru Meditation:

XID: "} + req.xid + {"

* * *

Varnish cache server

138	"};
	return(deliver);

}

backend atmailBACKEND {
	# used in catch_all 
	.host = "192.168.20.5";
	.port = "80";
	.first_byte_timeout = 300s;
	.connect_timeout = 25s;
	.probe = {
		.url = "/";
		.interval = 5s;
		.timeout = 1s;
		.window = 5;
		.threshold = 3;
	}
}

sub vcl_recv {

	#BASIC VCL RULES SETTING
	#Fix gzip compression
	if (req.http.Accept-Encoding) {
	if (req.url ~ "\.(gif|jpg|jpeg|bmp|png|ico|img|tga|wmf|gz|tgz|bz2|tbz|mp3|ogg)$") {
		unset req.http.Accept-Encoding;
		}
	else if (req.http.Accept-Encoding ~ "gzip") {
		set req.http.Accept-Encoding = "gzip";
		}
	else if (req.http.Accept-Encoding ~ "deflate") {
		set req.http.Accept-Encoding = "deflate";
		}
	else	{
		unset req.http.Accept-Encoding;
		}
	}
	#set client balance identity
	set client.identity = client.ip;

	#set X-forward
	set req.http.X-Forwarded-Varnish = client.ip;

	if (req.http.host == "www.coldhardmath.com") {
		set req.backend = atmailBACKEND;
	} 

	#respect client wish to refresh the page
	if (req.http.Pragma ~ "no-cache")
		{
		return(pass);
		}

	#BASIC VCL RULES ACTIONS
	#Disable post cache
	if (req.request == "POST") {
		return(pass);
		}
	#Disable session cache
	if (req.http.Cookie && req.http.Cookie ~ "(PHPSESSID|phpsessid)") {
		return(pass);
		}
	if (req.http.Cookie && req.http.Cookie ~ "(JSESSION|jsession)") {
		return(pass);
		}
	if (req.http.X-Requested-With == "XMLHttpRequest" || req.url ~ "nocache") {
		return(pass);
		}
	#Enable static cache
	if (req.request=="GET" && req.url ~ "\.(css|js|txt|zip|pdf|rtf|flv|swf|html|htm)$") {
		unset req.http.cookie;
		return(lookup);
		}
	if (req.request=="GET" && req.url ~ "\.(gif|jpg|jpeg|bmp|png|ico|img|tga|wmf|mp3|ogg)$") {
		unset req.http.cookie;
		return(lookup);
		}
	#Be rfc2616 compliant
	if (req.request ~ "^(GET|HEAD|PUT|POST|TRACE|OPTIONS|DELETE)$") {
		return(lookup);
		}
	else	{
		return(pipe);
		}

	return(lookup);
}

sub vcl_pipe {

	# If we don't set the Connection: close header, any following
	# requests from the client will also be piped through and
	# left untouched by varnish. We don't want that.
	set req.http.connection = "close";
	# Note: no "pipe" action here - we'll fall back to the default
	# pipe method so that when any changes are made there, we
	# still inherit them.

}
sub vcl_hit {
	return (deliver);
}

sub vcl_miss {
	return (fetch);
}

sub vcl_fetch {
	set beresp.do_stream = true;

	#Disable cache when backend is starting a session
	if (beresp.http.Set-Cookie && beresp.http.Set-Cookie ~ "(PHPSESSID|phpsessid)") {
		return(hit_for_pass);
		}
	if (beresp.http.Set-Cookie && beresp.http.Set-Cookie ~ "(JSESSION|jsession)") {
		return(hit_for_pass);
		}
	if (beresp.http.X-Requested-With == "XMLHttpRequest" || req.url ~ "nocache") {
		return(hit_for_pass);
		}

	# Varnish respects the wishes of the backend application.
	if (beresp.http.Pragma ~ "no-cache" || beresp.http.Cache-Control ~ "(no-cache|no-store|private)") {
	return(hit_for_pass);
	}
    ## If the request to the backend returns a code other than 200, restart the loop
    ## If the number of restarts reaches the value of the parameter max_restarts,
    ## the request will be error'ed.  max_restarts defaults to 4.  This prevents
    ## an eternal loop in the event that, e.g., the object does not exist at all.
    if (beresp.status != 200 && beresp.status != 403 && beresp.status != 404 && 
    	beresp.status != 303 && beresp.status != 302 && beresp.status != 301 && beresp.status != 401 ) {
		return(restart);
        }

	set beresp.grace = 60s;
		return(deliver);
}

sub vcl_deliver {
	##set resp.http.X-Served-By = server.hostname;
  	if (obj.hits > 0) {
    	set resp.http.X-Cache = "HIT";	
    	set resp.http.X-Cache-Hits = obj.hits;
  	} else {
    	set resp.http.X-Cache = "MISS";	
  	}
	return(deliver);
}

sub vcl_init {
	return (ok);
}

sub vcl_fini {
	return (ok);
}	

im sure there is something very simple that im missing and ive been staring at this so long im missing the forest staring at the trees.

anyone have any ideas?

all the best and thanks in advance
NT


[marcelloc]

Are you sure you do not have any nat or pfsense gui on port 80?

[normtodd]

thanks marcelloc for responding and for all of your wonderful work on pfsense

pfsense gui is on :81 and i removed the nat forwarding to port 80, just adding a rule on the wan opening port 80 (toward varnish)

[marcelloc]

@normtodd:

just adding a rule on the wan opening port 80 (toward varnish)

Without this rule, varnish will get no traffic.  :)

[normtodd]

i apologize - i think im getting mixed up ! :)

per this thread, i removed the nat rule (which i previously had configured prior to installing varnish) for port 80

in order to have port 80 open (in order for traffic to reach varnish), i added a firewall rule on the wan for port 80 to be open

isnt this the correct config? :)

all the best and thanks for all

[marcelloc]

Yes, the setup is this way.

Are you sure it's working through varnish?

[normtodd]

i assume that varnish listens on port 80 and the wan firewall rule would allow access?

how would i confirm that its working through varnish?

[marcelloc]

Well, widget should show it to you but you can try to stop varnish and see if you can access internal server or if publishing has stopped.

[normtodd]

per your suggestion i stopped varnish which made the website unreachable/not accessible

is this the expected result?

original problem of nonupdating widget remained when i restarted varnish (in case you were curious)

all the best and thanks in advance

[marcelloc]

@normtodd:

per your suggestion i stopped varnish which made the website unreachable/not accessible

is this the expected result?

Yes.

@normtodd:

original problem of nonupdating widget remained when i restarted varnish (in case you were curious)

I have no idea how to help you right now.

[normtodd]

as a last ditch i uninstalled and resinstalled varnish.

this is the part where i hang my head in shame for not doing proper troubleshooting before posting.

[shame]

i had installed and was working from the varnish3 package.  that was the package that was not populating the widget.  i uninstalled varnish3, installed varnish and it works great, which the widget updating as it should.

[/shame]

so that was the issue.

thank you marcelloc for all of your work

best
BT

[marcelloc]

@normtodd:

I uninstalled varnish3, installed varnish and it works great, which the widget updating as it should.

Thanks for the feedback. I'll check if varnish3 changed something that widget can't read.

[rajbps]

Hiya,

I had install varnish on pfsense and everything was working fine.

I had to rebuild my firewall and now my websites are down.

I followed this:

First be sure your pfsense gui is not listening on port 80 (system-> advanced)

mine is on https and port 441

Varnish config begins with backend configuration

Each backend means a webserver

inside each configuration, you must provide:

Backend name = test
    Ip Address = 192.168.2.10
    Port= 80
    Probe URL= http://www.mywebsiteaddress.com
    Probe Interval (5)
    Probe Timeout (1)
    Probe window (5)
    Probe Threshold (3)

on backend mapping, you could choose

map=host
match=equal
expression=www.mydomain.com (for first web server)

save backend config

on settings

Listen port=80
    Managment interface=127.0.0.1:85
    storage type=memory
    cache storage size=1024

on General VCL settings, check all options  # checked all options

all other options on settings could be in default value

When I go to the website, I get a :

It works!
This is the default web page for this server.

The web server software is running but no content has been added, yet.

I think this is coming from pfsense and not getting redirected.

I have varnish 3 installed.

Hope someone could show me my mistake.

On the wan I have this rule:

TCP * * WAN address 80 (HTTP) * none

I have rebooted the firewall a few times

Cheers,

Raj

[marcelloc]

try to replace http://www.mywebsiteaddress.com probe url to a full url like http://www.mywebsiteaddress.com/index.html

[rajbps]

did and no joy

the only way I got one of the website to work was this :

host equal www.mywebsite.com

and

host equal mywebsite.com

previously i do not remember having it like that as I have screenshot of varnish setting from before.

[rajbps]

is there any logs to see why even though varnish is working pfsense is not redirecting properly pls?

Cheers,

Raj

[marcelloc]

the widget will show backend status and to see varnish logs, you can use varnishlog from console

[serangku]

Thanks @marcelloc for this topic

after a week to re read also re read to other post …
it's configure with success to serve multiple server behind 1 ip public

great package

[skipmed]

@marcelloc:

the widget will show backend status and to see varnish logs, you can use varnishlog from console

Hi Marcello, I have the same problem with my pfsense:

2.0.2-RELEASE (amd64)
built on Fri Dec 7 22:39:43 EST 2012
FreeBSD 8.1-RELEASE-p13

and Varnish 3.0.2 pkg v.0.1

I tried all suggestion you gave but nothing solve my problem: when I call the Varnish on litening port (8082 in my case), only the firt defined backend is returned, no change if I call www.call1.com or www.call2.com (both domains point to public ip of pfsense of course).
this is the configuration:

backend crmBACKEND {

used in catch_all

.host = "10.8.0.1";
.port = "51231";
.first_byte_timeout = 100s;
.connect_timeout = 100s;
.probe = {
.request =
"GET /diagnostico.asp HTTP/1.1"
"Accept: text/*"
"User-Agent: Varnish"
"Host: www.site1.com"
"Connection: Close";
.interval = 10s;
.timeout = 100s;
.window = 1;
.threshold = 1;
}
}

backend wwwBACKEND {

used in catch_all

.host = "192.168.1.2";
.port = "80";
.first_byte_timeout = 100s;
.connect_timeout = 100s;
.probe = {
.url = "www.site2.com";
.interval = 10s;
.timeout = 100s;
.window = 1;
.threshold = 1;
}
}

sub vcl_recv {

#BASIC VCL RULES SETTING
#set client balance identity
set client.identity = req.http.user-agent;

#set X-forward
set req.http.X-Forwarded-For = client.ip;

if (req.http.host == "www.call1.com") {
set req.backend = crmBACKEND;
set req.grace=1s;
}
else if (req.http.host == "www.call2.com") {
set req.backend = wwwBACKEND;
set req.grace=1s;
}

#respect client wish to refresh the page
if (req.http.Pragma ~ "no-cache")
{
return(pass);
}

#BASIC VCL RULES ACTIONS
#Disable session cache
if (req.http.Cookie && req.http.Cookie ~ "(PHPSESSID|phpsessid)") {
return(pass);
}
if (req.http.Cookie && req.http.Cookie ~ "(JSESSION|jsession)") {
return(pass);
}
if (req.http.X-Requested-With == "XMLHttpRequest" || req.url ~ "nocache") {
return(pass);
}
#Enable static cache
if (req.request=="GET" && req.url ~ ".(css|js|txt|zip|pdf|rtf|flv|swf|html|htm)$") {
unset req.http.cookie;
return(lookup);
}
if (req.request=="GET" && req.url ~ ".(gif|jpg|jpeg|bmp|png|ico|img|tga|wmf|mp3|ogg)$") {
unset req.http.cookie;
return(lookup);
}
if (req.request != "GET" && req.request != "HEAD") {return(pipe);}

return(lookup);
}

sub vcl_pipe {

If we don't set the Connection: close header, any following

requests from the client will also be piped through and

left untouched by varnish. We don't want that.

set req.http.connection = "close";

Note: no "pipe" action here - we'll fall back to the default

pipe method so that when any changes are made there, we

still inherit them.

}
sub vcl_hit {
return (deliver);
}

sub vcl_miss {
return (fetch);
}

sub vcl_fetch {

#Disable cache when backend is starting a session
if (beresp.http.Set-Cookie && beresp.http.Set-Cookie ~ "(PHPSESSID|phpsessid)") {
return(hit_for_pass);
}
if (beresp.http.Set-Cookie && beresp.http.Set-Cookie ~ "(JSESSION|jsession)") {
return(hit_for_pass);
}
if (beresp.http.X-Requested-With == "XMLHttpRequest" || req.url ~ "nocache") {
return(hit_for_pass);
}

Varnish respects the wishes of the backend application.

if (beresp.http.Pragma ~ "no-cache" || beresp.http.Cache-Control ~ "(no-cache|no-store|private)") {
return(hit_for_pass);
}
    ## If the request to the backend returns a code other than 200, restart the loop
    ## If the number of restarts reaches the value of the parameter max_restarts,
    ## the request will be error'ed.  max_restarts defaults to 4.  This prevents
    ## an eternal loop in the event that, e.g., the object does not exist at all.
    if (beresp.status != 200 && beresp.status != 403 && beresp.status != 404 &&
    beresp.status != 303 && beresp.status != 302 && beresp.status != 301 && beresp.status != 401 ) {
set beresp.saintmode = 60s;
return(restart);
        }

set beresp.grace = 30m;
return(deliver);
}

sub vcl_deliver {
##set resp.http.X-Served-By = server.hostname;
  if (obj.hits > 0) {
    set resp.http.X-Cache = "HIT";
    set resp.http.X-Cache-Hits = obj.hits;
  } else {
    set resp.http.X-Cache = "MISS";
  }
return(deliver);
}

sub vcl_init {
return (ok);
}

sub vcl_fini {
return (ok);
}

what is wrong? thanks in advance