0

I have multiple website on the server and I am trying to configure Varnish cache for them. I have bypassed other sites as for now I am focusing only on the magento site i.e. magento.site. However, if I open https://www.magento.site then it doesn't respond but upon opening it with the file name the website then responds i.e. https://www.magento.site/home.

Even if I open it with the homepage name Varnish is still not caching it and I see no MAIN.cache_hit etc in the varnishstat.

I am using below varnish configuration file.

vcl 4.1;

import std;
import proxy;

backend default {
.host = "127.0.0.1";
.port = "8080";
.first_byte_timeout = 600s;
}
backend career {
.host = "127.0.0.1";
.port = "8081";
.first_byte_timeout = 600s;
}

backend othersite {
.host = "127.0.0.1";
.port = "8082";
.first_byte_timeout = 600s;
.probe = {
    .url = "/health_check.php";
.timeout = 2s;
    .interval = 5s;
    .window = 10;
    .threshold = 5;
}
}

backend magento {
.host = "localhost";
.port = "8085";
.first_byte_timeout = 600s;
.probe = {
.url = "/health_check.php";
    .timeout = 2s;
    .interval = 5s;
    .window = 10;
    .threshold = 5;
}
}
# Add hostnames, IP addresses and subnets that are allowed to purge content
acl purge {
"localhost";
"127.0.0.1";
"::1";
}

sub vcl_recv {

if (req.http.host == "www.site.com" || req.http.host == "site.com") {
set req.backend_hint = othersite;
} else if (req.http.host == "www.magento.site" || req.http.host == "magento.site") {
set req.backend_hint = magento;
} else if (req.http.host == "career.site.com") {
set req.backend_hint = career;
} else {
set req.backend_hint = default;
}

# Forward client's IP to the backend
if (req.restarts == 0) {
    if (req.http.X-Real-IP) {
        set req.http.X-Forwarded-For = req.http.X-Real-IP;
    } else if (req.http.X-Forwarded-For) {
        set req.http.X-Forwarded-For = req.http.X-Forwarded-For + ", " + client.ip;
    } else {
        set req.http.X-Forwarded-For = client.ip;
    }
}


# Remove empty query string parameters
# e.g.: www.example.com/index.html?    
if (req.url ~ "\?$") {
    set req.url = regsub(req.url, "\?$", "");
}

# Remove port number from host header
set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");

# Sorts query string parameters alphabetically for cache normalization purposes    
set req.url = std.querysort(req.url);

# Remove the proxy header to mitigate the httpoxy vulnerability
# See https://httpoxy.org/    
unset req.http.proxy;

# Add X-Forwarded-Proto header when using https
if (!req.http.X-Forwarded-Proto) {
    if(std.port(server.ip) == 443 || std.port(server.ip) == 8443) {
        set req.http.X-Forwarded-Proto = "https";
    } else {
        set req.http.X-Forwarded-Proto = "http";
    }
}


if (req.restarts > 0) {
    set req.hash_always_miss = true;
}

if (req.method == "PURGE") {
    if (client.ip !~ purge) {
        return (synth(405, "Method not allowed"));
    }
    # To use the X-Pool header for purging varnish during automated deployments, make sure the X-Pool header
    # has been added to the response in your backend server config. This is used, for example, by the
    # capistrano-magento2 gem for purging old content from varnish during it's deploy routine.
    if (!req.http.X-Magento-Tags-Pattern && !req.http.X-Pool) {
        return (synth(400, "X-Magento-Tags-Pattern or X-Pool header required"));
    }
    if (req.http.X-Magento-Tags-Pattern) {
      ban("obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern);
    }
    if (req.http.X-Pool) {
      ban("obj.http.X-Pool ~ " + req.http.X-Pool);
    }
    return (synth(200, "Purged"));
}

if (req.method != "GET" &&
    req.method != "HEAD" &&
    req.method != "PUT" &&
    req.method != "POST" &&
    req.method != "TRACE" &&
    req.method != "OPTIONS" &&
    req.method != "DELETE") {
      /* Non-RFC2616 or CONNECT which is weird. */
      return (pipe);
}

# We only deal with GET and HEAD by default
if (req.method != "GET" && req.method != "HEAD") {
    return (pass);
}

# Bypass customer, shopping cart, checkout
if (req.url ~ "/customer" || req.url ~ "/checkout" || req.url ~ "/klaviyo" || req.url ~ "/payfast" || req.url ~ "/payment" || req.url ~ "/reclaim" || req.url ~ "/order-confirmation-summary") {
    return (pass);
}

# Bypass health check requests
if (req.url ~ "^/(pub/)?(health_check.php)$") {
    return (pass);
}

# Set initial grace period usage status
set req.http.grace = "none";

# normalize url in case of leading HTTP scheme and domain
set req.url = regsub(req.url, "^http[s]?://", "");

# collect all cookies
std.collect(req.http.Cookie);

# Compression filter. See https://www.varnish-cache.org/trac/wiki/FAQ/Compression
if (req.http.Accept-Encoding) {
    if (req.url ~ "\.(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|flv|webp)$") {
        # No point in compressing these
        unset req.http.Accept-Encoding;
    } elsif (req.http.Accept-Encoding ~ "gzip") {
        set req.http.Accept-Encoding = "gzip";
    } elsif (req.http.Accept-Encoding ~ "deflate" && req.http.user-agent !~ "MSIE") {
        set req.http.Accept-Encoding = "deflate";
    } else {
        # unknown algorithm
        unset req.http.Accept-Encoding;
    }
}

# Remove tracking query string parameters used by analytics tools
if (req.url ~ "(\?|&)(_branch_match_id|_bta_[a-z]+|campid|customid|_ga|gclid|gclsrc|gdf[a-z]+|cx|dm_i|ef_id|epik|ie|igshid|cof|hsa_[a-z]+|_ke|mk[a-z]{3}|msclkid|(mtm|matomo)_[a-z]+|pcrid|p(iwi)?k_[a-z]+|redirect(_log)?_mongo_id|siteurl|s_kwcid|sb_referer_host|si|trk_[a-z]+|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=") {
    set req.url = regsuball(req.url, "(_branch_match_id|_bta_[a-z]+|campid|customid|_ga|gclid|gclsrc|cx|dm_i|ef_id|epik|ie|igshid|cof|hsa_[a-z]+|_ke|mk[a-z]{3}|msclkid|(mtm|matomo)_[a-z]+|pcrid|p(iwi)?k_[a-z]+|redirect(_log)?_mongo_id|siteurl|s_kwcid|sb_referer_host|si|trk_[a-z]+|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=[-_A-z0-9+()%.]+&?", "");
    set req.url = regsub(req.url, "[?|&]+$", "");
}


# Remove all marketing get parameters to minimize the cache objects
if (req.url ~ "(\?|&)(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=") {
    set req.url = regsuball(req.url, "(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=[-_A-z0-9+()%.]+&?", "");
    set req.url = regsub(req.url, "[?|&]+$", "");
}

# Static files caching
if (req.url ~ "^/(pub/)?(media|static)/") {
    # Static files should not be cached by default
    return (pass);

    # But if you use a few locales and don't use CDN you can enable caching static files by commenting previous line (#return (pass);) and uncommenting next 3 lines
    #unset req.http.Https;
    #unset req.http.X-Forwarded-Proto;
    #unset req.http.Cookie;
}

# Authenticated GraphQL requests should not be cached by default
if (req.url ~ "/graphql" && req.http.Authorization ~ "^Bearer") {
    return (pass);
}

return (hash);
}

sub vcl_hash {
if (req.http.cookie ~ "X-Magento-Vary=") {
    hash_data(regsub(req.http.cookie, "^.*?X-Magento-Vary=([^;]+);*.*$", "\1"));
}

# To make sure http users don't see ssl warning
if (req.http.X-Forwarded-Proto) {
    hash_data(req.http.X-Forwarded-Proto);
}


if (req.url ~ "/graphql") {
    call process_graphql_headers;
}
}

sub process_graphql_headers {
if (req.http.Store) {
    hash_data(req.http.Store);
}
if (req.http.Content-Currency) {
    hash_data(req.http.Content-Currency);
}
}

sub vcl_backend_response {

set beresp.grace = 3d;

if (beresp.http.content-type ~ "text") {
    set beresp.do_esi = true;
}

if (bereq.url ~ "\.js$" || beresp.http.content-type ~ "text") {
    set beresp.do_gzip = true;
}

if (beresp.http.X-Magento-Debug) {
    set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control;
}

# cache only successfully responses and 404s
if (beresp.status != 200 && beresp.status != 404) {
    set beresp.ttl = 0s;
    set beresp.uncacheable = true;
    return (deliver);
} elsif (beresp.http.Cache-Control ~ "private") {
    set beresp.uncacheable = true;
    set beresp.ttl = 86400s;
    return (deliver);
}

# validate if we need to cache it and prevent from setting cookie
if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) {
    unset beresp.http.set-cookie;
}

# If page is not cacheable then bypass varnish for 2 minutes as Hit-For-Pass
if (beresp.ttl <= 0s ||
   beresp.http.Surrogate-control ~ "no-store" ||
   (!beresp.http.Surrogate-Control &&
   beresp.http.Cache-Control ~ "no-cache|no-store") ||
   beresp.http.Vary == "*") {
    # Mark as Hit-For-Pass for the next 2 minutes
    set beresp.ttl = 120s;
    set beresp.uncacheable = true;
}

return (deliver);
}

sub vcl_deliver {
if (resp.http.X-Magento-Debug) {
    if (obj.uncacheable) {
    set resp.http.X-Magento-Cache-Debug = "UNCACHEABLE";
} else if (resp.http.x-varnish ~ " ") {
        set resp.http.X-Magento-Cache-Debug = "HIT";
        set resp.http.Grace = req.http.grace;
    } else {
        set resp.http.X-Magento-Cache-Debug = "MISS";
    }
} else {
    unset resp.http.Age;
}

# Not letting browser to cache non-static files.
if (resp.http.Cache-Control !~ "private" && req.url !~ "^/(pub/)?(media|static)/") {
    set resp.http.Pragma = "no-cache";
    set resp.http.Expires = "-1";
    set resp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0";
}

unset resp.http.X-Magento-Debug;
unset resp.http.X-Magento-Tags;
unset resp.http.X-Powered-By;
unset resp.http.Server;
unset resp.http.X-Varnish;
unset resp.http.Via;
unset resp.http.Link;
}

sub vcl_hit {
if (obj.ttl >= 0s) {
    # Hit within TTL period
    return (deliver);
}
if (std.healthy(req.backend_hint)) {
    if (obj.ttl + 300s > 0s) {
        # Hit after TTL expiration, but within grace period
        set req.http.grace = "normal (healthy server)";
        return (deliver);
    } else {
        # Hit after TTL and grace expiration
        return (restart);
    }
} else {
    # server is not healthy, retrieve from cache
    set req.http.grace = "unlimited (unhealthy server)";
    return (deliver);
}
}

I am using following proxy related block for Nginx in Magento's nginx configuration file for the location / { block.

        proxy_pass http://127.0.0.1/;
proxy_set_header   Host $host;
    proxy_set_header   X-Forwarded-Host $http_host;
    proxy_set_header   X-Real-IP $remote_addr;
    proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header   Ssl-Offloaded "1";
    proxy_set_header   X-Forwarded-Port 443;
    proxy_set_header   X-Forwarded-Proto https;
    proxy_set_header   X-Forwarded-Ssl on;
    proxy_buffer_size   128k;
    proxy_buffers      4 256k;
    proxy_busy_buffers_size 256k;
    fastcgi_buffer_size 32k;
    fastcgi_buffers    4 32k;

Please let me know if anything else is required. But its been days and I am still stuck on this, and couldn't find any solution to my problem.

UPDATE:

Below is my varnish.service output and Varnish is running on port 80.

    [Unit]
Description=Varnish Cache, a high-performance HTTP accelerator
After=network-online.target nss-lookup.target

[Service]
Type=forking
KillMode=process

# Maximum number of open files (for ulimit -n)
LimitNOFILE=131072

# Locked shared memory - should suffice to lock the shared memory log
# (varnishd -l argument)
# Default log size is 80MB vsl + 1M vsm + header -> 82MB
# unit is bytes
LimitMEMLOCK=85983232

# Enable this to avoid "fork failed" on reload.
TasksMax=infinity

# Maximum size of the corefile.
LimitCORE=infinity

ExecStart=/usr/sbin/varnishd \
          -a :80 \
          -a localhost:8443,PROXY \
          -p feature=+http2 \
          -f /etc/varnish/default.vcl \
          -s malloc,3g
ExecReload=/usr/sbin/varnishreload

[Install]
WantedBy=multi-user.target
Improve this question

1 Answer 1

Reset to default
0

Is Varnish running on port 80?

I see that everything is hosted on the same server, so please run the following command to see what ports are in use:

sudo netstat -tulpn

This will give you an of ports and associated processes. The goal is to check whether the varnishd process is listening on port 80.

If varnishd runs as a systemd service, you can also run the following command to check the varnishd runtime parameters:

sudo systemctl cat varnish.service

This will display the systemd unit file for Varnish. Have a look at the -a runtime parameters to see what port is in use.

Running the following command also helps to see what the runtime parameters are of a running varnishd instance:

ps aux | grep varnishd | grep -v grep

If your conclusion is that Varnish is not running on port 80, alter your systemd config by running the following commands:

sudo systemctl edit --full varnish.service

And when you're done editing:

sudo systemctl restart varnish.service

Also check if other services already run on port 80 and may clash with Varnish.

Run varnishlog to inspect traffic

Once you're confident that Varnish is running on port 80 and that Nginx forwards HTTPS traffic to Varnish, you can run the following command to track incoming requests for the homepage:

sudo varnishlog -g request -q "ReqUrl eq '/'"

You'll find the relevant information in the log output. If you're still confused, don't hesitate to add the log output to your question and I'll help you inspect.

Improve this answer
9
  • Thanks for the reply. Yes varnish is running on port 80, and I have updated my question and added the content of varnish.service.
    – nix
    Commented Apr 5 at 21:11
  • I tested this command sudo varnishlog -g request -q "ReqUrl eq '/'" and also the varnishlog but, the output is nothing while opening the website.
    – nix
    Commented Apr 5 at 21:41
  • Can you try ps aux | grep varnishd | grep -v grep to see if Varnish is really running on that port? If that's the case and there's no traffic, I wonder if the DNS is actually pointing to that node.
    – Thijs Feryn
    Commented Apr 6 at 15:45
  • 1
    @nix Have a look at this "debugging Varnish" article I wrote about to check why Varnish is not caching: varnish-software.com/developers/tutorials/…
    – Thijs Feryn
    Commented Apr 8 at 7:01
  • 1
    @nix Please also ensure that Nginx is properly proxying HTTPS traffic to Varnish. Please also run a test over plain HTTP (maybe via curl avoid the browser's TLS enforcement) and see if Varnish is properly handling it.
    – Thijs Feryn
    Commented Apr 8 at 7:02

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.