Varnish keeps missing cache ,cookies?

问题:

I have a bit of trouble getting varnish to cache correctly.

When I go to a page and hit refresh , varnish will return the cached page. But if I use another computer and hit that same page again ( within the cache time-frame) it will report a MISS.

I used the cpanel varnish plugin from http://www.unixy.net/varnish
I removed session_start() from my (php) pages and I do not need userbased cookies.
I am also using google analytics.

This is my default.vcl

###################################################
# Copyright (c) UNIXY  -  http://www.unixy.net    #
# The leading truly fully managed server provider #
###################################################

include "/etc/varnish/cpanel.backend.vcl";

include "/etc/varnish/backends.vcl";

sub vcl_recv {

# Use the default backend for all other requests
set req.backend = default;

# Setup the different backends logic
include "/etc/varnish/acllogic.vcl";

# Allow a grace period for offering "stale" data in case backend lags
set req.grace = 5m;

remove req.http.X-Forwarded-For;
set req.http.X-Forwarded-For = client.ip;

# cPanel URLs
include "/etc/varnish/cpanel.url.vcl";

# Properly handle different encoding types
if (req.http.Accept-Encoding) {
    if (req.url ~ "\.(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|ico)$") {
        # No point in compressing these
        remove req.http.Accept-Encoding;
    } elsif (req.http.Accept-Encoding ~ "gzip") {
        set req.http.Accept-Encoding = "gzip";
    } elsif (req.http.Accept-Encoding ~ "deflate") {
        set req.http.Accept-Encoding = "deflate";
    } else {
        # unkown algorithm
        remove req.http.Accept-Encoding;
    }
}

# Set up disabled
include "/etc/varnish/disabled.vcl";

# Exclude upgrade, install, server-status, etc
include "/etc/varnish/known.exclude.vcl";

# Set up exceptions
include "/etc/varnish/url.exclude.vcl";

# Set up exceptions
include "/etc/varnish/vhost.exclude.vcl";

# Set up vhost+url exceptions
include "/etc/varnish/vhosturl.exclude.vcl";

# Set up cPanel reseller exceptions
include "/etc/varnish/reseller.exclude.vcl";

# Restart rule for bfile recv
include "/etc/varnish/bigfile.recv.vcl";


if (req.request == "PURGE") {
        if (!client.ip ~ acl127_0_0_1) {error 405 "Not permitted";}
        return (lookup);
}

## Default request checks
if (req.request != "GET" &&
req.request != "HEAD" &&
req.request != "PUT" &&
req.request != "POST" &&
req.request != "TRACE" &&
req.request != "OPTIONS" &&
req.request != "DELETE") {
    return (pipe);
}

if (req.request != "GET" && req.request != "HEAD") {
    return (pass);
}

## Modified from default to allow caching if cookies are set, but not http auth
if (req.http.Authorization) {
    return (pass);
}

include "/etc/varnish/versioning.static.vcl";

## Remove has_js and Google Analytics cookies.
set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(__[a-z]+|has_js)=[^;]*", "");

set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", "");



if (req.http.Cookie ~ "^\s*$") {
    unset req.http.Cookie;
}

include "/etc/varnish/slashdot.recv.vcl";

# Cache things with these extensions
if (req.url ~ "\.(js|css|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|pdf)$" && ! (req.url ~ "\.(php)") ) {
    unset req.http.Cookie;
    return (lookup);
}

return (lookup);
}

sub vcl_fetch {

set beresp.ttl = 40s;
set beresp.http.Server = " - Web acceleration by http://www.unixy.net/varnish ";

# Turn off Varnish gzip processing
include "/etc/varnish/gzip.off.vcl";

# Grace to allow varnish to serve content if backend is lagged
set beresp.grace = 5m;

# Restart rule bfile for fetch
include "/etc/varnish/bigfile.fetch.vcl";

# These status codes should always pass through and never cache.
if (beresp.status == 503 || beresp.status == 500) {
    set beresp.http.X-Cacheable = "NO: beresp.status";
    set beresp.http.X-Cacheable-status = beresp.status;
    return (hit_for_pass);
}

if (beresp.status == 404) {
    set beresp.http.magicmarker = "1";
    set beresp.http.X-Cacheable = "YES";
    set beresp.ttl = 20s;
    return (deliver);
}

/* Remove Expires from backend, it's not long enough */    
unset beresp.http.expires;

if (req.url ~ "\.(js|css|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|pdf|ico)$" && ! (req.url ~ "\.(php)") ) {
    unset beresp.http.set-cookie;
    include "/etc/varnish/static.ttl.vcl";
}
include "/etc/varnish/slashdot.fetch.vcl"; 
else {
    include "/etc/varnish/dynamic.ttl.vcl";
}

/* marker for vcl_deliver to reset Age: */
set beresp.http.magicmarker = "1";

# All tests passed, therefore item is cacheable
set beresp.http.X-Cacheable = "YES";

return (deliver);
}

sub vcl_deliver {

  # From http://varnish-cache.org/wiki/VCLExampleLongerCaching
  if (resp.http.magicmarker) {
     /* Remove the magic marker */
     unset resp.http.magicmarker;

     /* By definition we have a fresh object */
     set resp.http.age = "0";
   }

   #add cache hit data
   if (obj.hits > 0) {
     #if hit add hit count
     set resp.http.X-Cache = "HIT";
     set resp.http.X-Cache-Hits = obj.hits;
   }
else {
     set resp.http.X-Cache = "MISS";
   }

}

sub vcl_error {

if (obj.status == 503 && req.restarts < 5) {
set obj.http.X-Restarts = req.restarts;
return (restart);
}

}

# Added to let users force refresh
sub vcl_hit {

if (obj.ttl < 1s) {
    return (pass);
}

if (req.http.Cache-Control ~ "no-cache") {
# Ignore requests via proxy caches,  IE users and badly behaved crawlers
# like msnbot that send no-cache with every request.
if (! (req.http.Via || req.http.User-Agent ~ "bot|MSIE|HostTracker")) {
    set obj.ttl = 0s;
    return (restart);
} 
}

return (deliver);

}

sub vcl_hash {

    hash_data(req.http.cookie);
}

My headers ( on the second computer ):

http://www.subtitleseeker.com/

GET / HTTP/1.1
Host: www.subtitleseeker.com
User-Agent: Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko/20100101 Firefox/11.0
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
Accept-Language: en-us,en;q=0.5
Accept-Encoding: gzip, deflate
Connection: keep-alive
Referer: http://www.subtitleseeker.com/search/
Cookie: __utma=107027200.157820436.1334670333.1334670333.1334670333.1; __utmb=107027200.51.10.1334670333; __utmz=107027200.1334670333.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); __utmc=107027200; __unam=4ca9c7b-136c095f22c-1396bc94-1

HTTP/1.1 200 OK
X-Powered-By: PHP/5.3.8
Vary: User-Agent,Accept-Encoding
Content-Encoding: gzip
Content-Type: text/html
Server: - Web acceleration by http://www.unixy.net/varnish
X-Cacheable: YES
Content-Length: 8169
Accept-Ranges: bytes
Date: Tue, 17 Apr 2012 14:10:12 GMT
X-Varnish: 916645159
Via: 1.1 varnish
Connection: keep-alive
Age: 0
X-Cache: MISS

Can someone explain to me why this is happening and what I should change ?

Thanks !

回答1:

This is because of Google Analytics. Even if your Web application does not use cookies your analytics JavaScript does. The result is the same, Varnish will pass the request to the back-end and avoid using it's cache.

To fix this define URLs of your Web application where you have to use cookies (e.g. administration panel) and where you don't (Here you can ignore the requirements of Google Analytics. Most Web analytics tools need cookies only between the browser and the JavaScript.).

Below you can see an example of a Varnish configuration file for this. The interesting part is that for non-administrative parts of your Web site: Sent cookies will be deleted as well as some browser request headers for fresh content.

sub vcl_recv {
  # regex to find all URLs where cookies are required
  if (req.url ~ "^/admin/") {
    # administration panel
    set req.http.admin = 1;
  } else {
    # public web site, ignore client request for fresh content, remove cookies
    unset req.http.Cache-Control;
    unset req.http.Max-Age;
    unset req.http.Pragma;
    unset req.http.Cookie;
  }
  ...
}

sub vcl_fetch {
  if (req.http.admin == 1) {
    # administration panel
    return (hit_for_pass);
  } else {
    # public web site, not allowed to set cookies
    unset beresp.http.Set-Cookie;
    ...
  }
  ...
}

回答2:

You can use the varnishlog command line tool to watch what Varnish is doing as it handles each request. You can see if any cookies are splitting through for example.

I have a feeling it may be the user-agent that is slightly different on the other computer. Varnish includes this value in the key if it is know. We have a rule in our vcl_recv which is:

set req.http.User-Agent = "";

Which fixed a similar problem for us.