2

On a setup with : - Magento 1.7.0.2 - Memcache - apache 2 with APC - Varnish - Turpentine

It appears that the 1st time you load a page (homepage for ex.), it's being loaded from Apache (takes time ...)

Then, Pressing F5 gives an instant load, with Varnish tag in the HTTP header, illustrating that varnish is here, doing his job.

Thing is : If you load the homepage again from another IP address, it's taking a while again.

It appears that Varnish makes a cache copy of every page, AND for every different visitor...

I was wondering what was the parameter I could alter to change this behaviour. Except for my ESI blocks (that are well loaded thanks to Turpentine), the 50 websites on my Magento setup is pretty static and I thought it would be part of the same cache ?

I thought someone visiting a page, will make it cached in Varnish for everybody else ?!

Looks like it's not, and I'd be very glad if you could indicate me where to look.

Thanks,

Sorcy

The VCL source

C{
#include <stdlib.h>
#include <stdio.h>
#include <time.h>
#include <pthread.h>
static pthread_mutex_t lrand_mutex = PTHREAD_MUTEX_INITIALIZER;
void generate_uuid(char* buf) {
pthread_mutex_lock(&lrand_mutex);
long a = lrand48();
long b = lrand48();
long c = lrand48();
long d = lrand48();
pthread_mutex_unlock(&lrand_mutex);
sprintf(buf, "frontend=%08lx-%04lx-%04lx-%04lx-%04lx%08lx",
a,
b & 0xffff,
(b & ((long)0x0fff0000) >> 16) | 0x4000,
(c & 0x0fff) | 0x8000,
(c & (long)0xffff0000) >> 16,
d
);
return;
}
}C
import std;
backend default {
.host = "127.0.0.1";
.port = "8080";
.first_byte_timeout = 300s;
.between_bytes_timeout = 300s;
}
backend admin {
.host = "127.0.0.1";
.port = "8080";
.first_byte_timeout = 21600s;
.between_bytes_timeout = 21600s;
}
acl crawler_acl {
"127.0.0.1";
}
acl debug_acl {
"XXXXXX";
"XXXXXX";
}
sub remove_cache_headers {
unset beresp.http.Cache-Control;
unset beresp.http.Expires;
unset beresp.http.Pragma;
unset beresp.http.Cache;
unset beresp.http.Age;
}
sub remove_double_slashes {
set req.url = regsub(req.url, "(.*)//+(.*)", "\1/\2");
}
sub generate_session {
if (req.url ~ ".*[&?]SID=([^&]+).*") {
set req.http.X-Varnish-Faked-Session = regsub(
req.url, ".*[&?]SID=([^&]+).*", "frontend=\1");
} else {
C{
char uuid_buf [50];
generate_uuid(uuid_buf);
VRT_SetHdr(sp, HDR_REQ,
"\030X-Varnish-Faked-Session:",
uuid_buf,
vrt_magic_string_end
);
}C
}
if (req.http.Cookie) {
std.collect(req.http.Cookie);
set req.http.Cookie = req.http.X-Varnish-Faked-Session +
"; " + req.http.Cookie;
} else {
set req.http.Cookie = req.http.X-Varnish-Faked-Session;
}
}

sub generate_session_expires {
C{
time_t now = time(NULL);
struct tm now_tm = *gmtime(&now);
now_tm.tm_sec += 3600;
mktime(&now_tm);
char date_buf [50];
strftime(date_buf, sizeof(date_buf)-1, "%a, %d-%b-%Y %H:%M:%S %Z", &now_tm);
VRT_SetHdr(sp, HDR_RESP,
"\031X-Varnish-Cookie-Expires:",
date_buf,
vrt_magic_string_end
);
}C
}

sub vcl_recv {

//redirects if www is missing
if( !req.http.host ~ "^www" ) {
    error 750 regsub(req.http.host, "^(w+\.)?", "http://www.") + req.url;
}

if (req.restarts == 0) {
if (req.http.X-Forwarded-For) {
set req.http.X-Forwarded-For =
req.http.X-Forwarded-For + ", " + client.ip;
} else {
set req.http.X-Forwarded-For = client.ip;
}
}
if (!true || req.http.Authorization ||
req.request !~ "^(GET|HEAD)$" ||
req.http.Cookie ~ "varnish_bypass=1") {
return (pipe);
}
call remove_double_slashes;
if (req.http.Accept-Encoding) {
if (req.http.Accept-Encoding ~ "gzip") {
set req.http.Accept-Encoding = "gzip";
} else if (req.http.Accept-Encoding ~ "deflate") {
set req.http.Accept-Encoding = "deflate";
} else {
unset req.http.Accept-Encoding;
}
}
if (req.http.User-Agent ~ "iP(?:hone|ad|od)|BlackBerry|Palm|Googlebot-Mobile|Mobile|mobile|mobi|Windows Mobile|Safari Mobile|Android|Opera (?:Mini|Mobi)") {
set req.http.X-Normalized-User-Agent = "mobile";
} else if (req.http.User-Agent ~ "MSIE") {
set req.http.X-Normalized-User-Agent = "msie";
} else if (req.http.User-Agent ~ "Firefox") {
set req.http.X-Normalized-User-Agent = "firefox";
} else if (req.http.User-Agent ~ "Safari") {
set req.http.X-Normalized-User-Agent = "safari";
} else if (req.http.User-Agent ~ "Chrome") {
set req.http.X-Normalized-User-Agent = "chrome";
} else if (req.http.User-Agent ~ "Opera") {
set req.http.X-Normalized-User-Agent = "opera";
} else {
set req.http.X-Normalized-User-Agent = "other";
}
if (req.url ~ "^(/)(?:(?:index|litespeed)\.php/)?") {
set req.http.X-Turpentine-Secret-Handshake = "1";
if (req.url ~ "^(/)(?:(?:index|litespeed)\.php/)?admin") {
set req.backend = admin;
return (pipe);
}
if (req.http.Cookie ~ "\bcurrency=") {
set req.http.X-Varnish-Currency = regsub(
req.http.Cookie, ".*\bcurrency=([^;]*).*", "\1");
}
if (req.http.Cookie ~ "\bstore=") {
set req.http.X-Varnish-Store = regsub(
req.http.Cookie, ".*\bstore=([^;]*).*", "\1");
}
if (req.url ~ "/turpentine/esi/getBlock/") {
set req.http.X-Varnish-Esi-Method = regsub(
req.url, ".*/method/(\w+)/.*", "\1");
set req.http.X-Varnish-Esi-Access = regsub(
req.url, ".*/access/(\w+)/.*", "\1");
if (req.http.X-Varnish-Esi-Method == "esi" && req.esi_level == 0 &&
!(false || client.ip ~ debug_acl)) {
error 403 "External ESI requests are not allowed";
}
}
if (req.http.Cookie !~ "frontend=") {
if (client.ip ~ crawler_acl ||
req.http.User-Agent ~ "^(?:ApacheBench/.*|.*Googlebot.*|JoeDog/.*Siege.*|magespeedtest\.com|Nexcessnet_Turpentine/.*)$") {
set req.http.Cookie = "frontend=crawler-session";
} else {
call generate_session;
}
}
if (true &&
req.url ~ ".*\.(?:css|js|jpe?g|png|gif|ico|swf)(?=\?|&|$)") {
unset req.http.Cookie;
unset req.http.X-Varnish-Faked-Session;
return (lookup);
}
if (req.url ~ "^(/)(?:(?:index|litespeed)\.php/)?(?:admin|api|cron\.php|checkout/cart|onepagecheckout|customer|paybox|contacts|paypal|amshopby)" ||
req.url ~ "\?.*__from_store=") {
return (pipe);
}

if (true &&
req.url ~ "(?:[?&](?:__SID|XDEBUG_PROFILE)(?=[&=]|$))") {
return (pass);
}
if (req.url ~ "[?&](utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=") {
set req.url = regsuball(req.url, "(?:(\?)?|&)(?:utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=[^&]+", "\1");
set req.url = regsuball(req.url, "(?:(\?)&|\?$)", "\1");
}
return (lookup);
}
}

sub vcl_pipe {
unset bereq.http.X-Turpentine-Secret-Handshake;
set bereq.http.Connection = "close";
}

sub vcl_hash {
hash_data(req.url);
if (req.http.Host) {
hash_data(req.http.Host);
} else {
hash_data(server.ip);
}
hash_data(req.http.Ssl-Offloaded);
if (req.http.X-Normalized-User-Agent) {
hash_data(req.http.X-Normalized-User-Agent);
}
if (req.http.Accept-Encoding) {
hash_data(req.http.Accept-Encoding);
}
if (req.http.X-Varnish-Store || req.http.X-Varnish-Currency) {
hash_data("s=" + req.http.X-Varnish-Store + "&c=" + req.http.X-Varnish-Currency);
}
if (req.http.X-Varnish-Esi-Access == "private" &&
req.http.Cookie ~ "frontend=") {
hash_data(regsub(req.http.Cookie, "^.*?frontend=([^;]*);*.*$", "\1"));
}
return (hash);
}

sub vcl_hit {
}
sub vcl_fetch {
set req.grace = 15s;
if (req.url ~ "^(/)(?:(?:index|litespeed)\.php/)?") {
unset beresp.http.Vary;
set beresp.do_gzip = true;
if (beresp.status != 200 && beresp.status != 404) {
set beresp.ttl = 15s;
return (hit_for_pass);
} else {
if (beresp.http.Set-Cookie) {
set beresp.http.X-Varnish-Set-Cookie = beresp.http.Set-Cookie;
unset beresp.http.Set-Cookie;
}
call remove_cache_headers;
if (beresp.http.X-Turpentine-Esi == "1") {
set beresp.do_esi = true;
}
if (beresp.http.X-Turpentine-Cache == "0") {
set beresp.ttl = 15s;
return (hit_for_pass);
} else {
if (true &&
bereq.url ~ ".*\.(?:css|js|jpe?g|png|gif|ico|swf)(?=\?|&|$)") {
set beresp.ttl = 2592000s;
set beresp.http.Cache-Control = "max-age=2592000";
} elseif (req.http.X-Varnish-Esi-Method) {
if (req.http.X-Varnish-Esi-Access == "private" &&
req.http.Cookie ~ "frontend=") {
set beresp.http.X-Varnish-Session = regsub(req.http.Cookie,
"^.*?frontend=([^;]*);*.*$", "\1");
}
if (req.http.X-Varnish-Esi-Method == "ajax" &&
req.http.X-Varnish-Esi-Access == "public") {
set beresp.http.Cache-Control = "max-age=" + regsub(
req.url, ".*/ttl/(\d+)/.*", "\1");
}
set beresp.ttl = std.duration(
regsub(
req.url, ".*/ttl/(\d+)/.*", "\1s"),
300s);
if (beresp.ttl == 0s) {
set beresp.ttl = 15s;
return (hit_for_pass);
}
} else {
set beresp.ttl = 99999s;
}
}
}
return (deliver);
}
}

sub vcl_deliver {
if (req.http.X-Varnish-Faked-Session) {
call generate_session_expires;
set resp.http.Set-Cookie = req.http.X-Varnish-Faked-Session +
"; expires=" + resp.http.X-Varnish-Cookie-Expires + "; path=/";
if (req.http.Host) {
set resp.http.Set-Cookie = resp.http.Set-Cookie +
"; domain=" + regsub(req.http.Host, ":\d+$", "");
}
set resp.http.Set-Cookie = resp.http.Set-Cookie + "; httponly";
unset resp.http.X-Varnish-Cookie-Expires;
}
if (false || client.ip ~ debug_acl) {
set resp.http.X-Varnish-Hits = obj.hits;
set resp.http.X-Varnish-Esi-Method = req.http.X-Varnish-Esi-Method;
set resp.http.X-Varnish-Esi-Access = req.http.X-Varnish-Esi-Access;
set resp.http.X-Varnish-Currency = req.http.X-Varnish-Currency;
set resp.http.X-Varnish-Store = req.http.X-Varnish-Store;
} else {
unset resp.http.X-Varnish;
unset resp.http.Via;
unset resp.http.X-Powered-By;
unset resp.http.Server;
unset resp.http.X-Turpentine-Cache;
unset resp.http.X-Turpentine-Esi;
unset resp.http.X-Turpentine-Flush-Events;
unset resp.http.X-Turpentine-Block;
unset resp.http.X-Varnish-Session;
unset resp.http.X-Varnish-Set-Cookie;
}
}

sub vcl_error {

// catch missing www error
 if (obj.status == 750) {
       set obj.http.Location = obj.response;
       set obj.status = 301;
       return (deliver);
   }
}
4
  • 1
    What Normalization options are you using? github.com/nexcess/magento-turpentine/wiki/…
    – B00MER
    May 12, 2014 at 23:26
  • Encoding and user-agent are Enabled. Normalize HOst is disabled as it's recommended not to enable it on multistore servers.
    – Sorcy
    May 13, 2014 at 15:17
  • your varnish.vcl file may offer some insight, github.com/fbrnc/magneto-varnish/blob/master/varnish.vcl may help also.
    – B00MER
    May 13, 2014 at 19:41
  • I edited my post with my .vcl If you have any clue on what could 'cause Varnish to cache a page for EACH visitor, I'd be very glad to hear it ! Thank you!
    – Sorcy
    May 15, 2014 at 10:37

2 Answers 2

2

disable agent normalization, and make sure there is no other cookies set, like "currency" or "store"

all the answers are here in hash section:

sub vcl_hash {
hash_data(req.url);
if (req.http.Host) {
hash_data(req.http.Host);
} else {
hash_data(server.ip);
}
hash_data(req.http.Ssl-Offloaded);
if (req.http.X-Normalized-User-Agent) {
hash_data(req.http.X-Normalized-User-Agent);
}
if (req.http.Accept-Encoding) {
hash_data(req.http.Accept-Encoding);
}
if (req.http.X-Varnish-Store || req.http.X-Varnish-Currency) {
hash_data("s=" + req.http.X-Varnish-Store + "&c=" + req.http.X-Varnish-Currency);
}
if (req.http.X-Varnish-Esi-Access == "private" &&
req.http.Cookie ~ "frontend=") {
hash_data(regsub(req.http.Cookie, "^.*?frontend=([^;]*);*.*$", "\1"));
}
return (hash);
}

and this line was extended:
if (req.http.Cookie !~ "frontend=" && !req.http.X-Varnish-Esi-Method)

2
  • Thanks for your answer. Unfortunately, I've dropped this solution for another alternative, because I wasn't able to obtain the result I wanted in the time I had :(
    – Sorcy
    Jan 5, 2015 at 17:46
  • too bad, Varnish is the best cache to date...
    – user2857
    Jan 26, 2015 at 21:26
0

All varnish configurations entry point for a request is sub vcl_recv After this conditions are checked, at this point:

 if (req.http.Cookie!~"frontend=") {
            if (client.ip~ crawler_acl ||
                req.http.User - Agent~ "^(?:ApacheBench/.*|.*Googlebot.*|JoeDog/.*Siege.*|magespeedtest\.com|Nexcessnet_Turpentine/.*)$") {
                set req.http.Cookie = "frontend=crawler-session";
            } else {
                call generate_session;
            }
        }

If no Magento standard frontend cookie is found, a check for crawlers/bots is made and will skip the generation of a dummy session ID. You may be able to simply set a new cookie that is NOT unique ID here similarly.

Instead of call generate_session; you could try

set req.http.Cookie = "frontend=homepage-static-session";

However, since most crawler/bots do not save/send cookies you'll need to find a place to invalidate the newly created cookie, either by a TTYL and/or within the flush/purge of Varnishes cache. This would occur in sub vcl_fetch where a condition is met to purge the static cookie.

With the above in mind, Turpentine has extended Varnish itself to generate a random Session ID or User ID to create the fake request (no cookies, fresh load).

generate_uuid

This is explained in further detail here:

You can find the C file under misc/uuid.c

You may be able to a) Test with a BOT mimicked header browser extension and b) verify you are seeing the temporary session show for new fresh requests.

Also, bear in mind since Turpentine handles its .vcl configuration files via the Admin, making changes directly they would be overwritten.

3
  • Hi BOOMER, Thx for your help on this. About the C function generate_uuid, as you can see in the .vcl, that's exactly what generate_session is doing.
    – Sorcy
    May 19, 2014 at 14:28
  • I guess I only have to set the frontend cookie as you suggested, and then find a way to give it a TTL
    – Sorcy
    May 19, 2014 at 14:37
  • hmmm looks like the set frontend fix does not change anything. Inspecting the HTTP request show a XVarnish-Set-cookie = some unique number. I guess the problem comes from there, everyone has a different generated ID there, and that's what makes Varnish to duplicate the caching...
    – Sorcy
    May 19, 2014 at 15:03

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge that you have read and understand our privacy policy and code of conduct.

Not the answer you're looking for? Browse other questions tagged or ask your own question.