Varnish always shows miss on my website

Question

magento2.4.6 varnish6 nginx

Varnish always shows miss on my website I have checked all the cacheable=false and isScopePrivate = true and setNoCacheHeaders in my website but it still shows miss on the homepage of the website

This is the head of my website：

But my js and css successfully displayed the hit:

and this is my varnish.vcl:

# VCL version 5.0 is not supported so it should be 4.0 even though actually used Varnish version is 6
vcl 4.0;

import std;
# The minimal Varnish version is 6.0
# For SSL offloading, pass the following header in your proxy server or load balancer: 'SSL-OFFLOADED: https'

backend default {
    .host = "localhost";
    .port = "8080";
    .first_byte_timeout = 6000s;


    #.probe = {
    #    .url = "/health_check.php";
    #    .timeout = 2s;
    #    .interval = 5s;
    #    .window = 10;
    #    .threshold = 5;
   #}
}

acl purge {
"127.0.0.1";
    "localhost";
}

## Bypass for IP /not working
#acl passem { "211.22.147.226"; }
#sub vcl_recv {
#  if (!(client.ip ~ passem)) {
#    return (pass);
#  }
#}

sub vcl_recv {

if (req.restarts > 0) { set req.hash_always_miss = true; }

set req.http.X-Varnish-Esi-Level = req.esi_level;

# Set X-Forwarded-For to client IP instead of localhost
unset req.http.X-Forwarded-For;
set req.http.X-Forwarded-For = client.ip;


if (req.url ~ "^/404") {
        return (synth(999, "make 404 error explicitly"));
    }

    if (req.method == "PURGE") {
        if (client.ip !~ purge) {
            return (synth(405, "Method not allowed"));
        }
        # To use the X-Pool header for purging varnish during automated deployments, make sure the X-Pool header
        # has been added to the response in your backend server config. This is used, for example, by the
        # capistrano-magento2 gem for purging old content from varnish during it's deploy routine.
        if (!req.http.X-Magento-Tags-Pattern && !req.http.X-Pool) {
            return (synth(400, "X-Magento-Tags-Pattern or X-Pool header required"));
        }
        if (req.http.X-Magento-Tags-Pattern) {
          ban("obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern);
        }
        if (req.http.X-Pool) {
          ban("obj.http.X-Pool ~ " + req.http.X-Pool);
        }
        return (synth(200, "Purged"));
    }

    if (req.method != "GET" &&
        req.method != "HEAD" &&
        req.method != "PUT" &&
        req.method != "POST" &&
        req.method != "TRACE" &&
        req.method != "OPTIONS" &&
        req.method != "DELETE") {
          /* Non-RFC2616 or CONNECT which is weird. */
          return (pipe);
    }

    # We only deal with GET and HEAD by default
    if (req.method != "GET" && req.method != "HEAD") {
        return (pass);
    }
# Bypass customer, shopping cart, checkout
    if (req.url ~ "/customer" || req.url ~ "/checkout") {
        return (pass);
    }
    # Set initial grace period usage status
    set req.http.grace = "none";

    # normalize url in case of leading HTTP scheme and domain
    set req.url = regsub(req.url, "^http[s]?://", "");

    # collect all cookies
    std.collect(req.http.Cookie);

    # Compression filter. See https://www.varnish-cache.org/trac/wiki/FAQ/Compression
    if (req.http.Accept-Encoding) {
        if (req.url ~ "\.(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|flv)$") {
            # No point in compressing these
            unset req.http.Accept-Encoding;
        } elsif (req.http.Accept-Encoding ~ "gzip") {
            set req.http.Accept-Encoding = "gzip";
        } elsif (req.http.Accept-Encoding ~ "deflate" && req.http.user-agent !~ "MSIE") {
            set req.http.Accept-Encoding = "deflate";
        } else {
            # unknown algorithm
            unset req.http.Accept-Encoding;
        }
    }

    # Remove all marketing get parameters to minimize the cache objects
    if (req.url ~ "(\?|&)(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=") {
        set req.url = regsuball(req.url, "(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=[-_A-z0-9+()%.]+&?", "");
        set req.url = regsub(req.url, "[?|&]+$", "");
    }

   # Bypass shopping cart, checkout and search requests
   #    if (req.url ~ "/checkout" || req.url ~ "/catalogsearch") {
   #     return (pass);
   # }
    #add 20210628
   # if (req.url ~ "/adyen") {
   #     return (pass);
   # }
    #add 20210628
    # Bypass health check requests
   # if (req.url ~ "/health_check.php") {
   #     return (pass);
   # }

    # Static files caching
   if (req.url ~ "/static/") {
        # Static files should not be cached by default
        # return (pass);

        # But if you use a few locales and don't use CDN you can enable caching static files by commenting previous line (#return (pass);) and uncommenting next 3 lines
        unset req.http.Https;
        unset req.http.SSL-OFFLOADED;
        unset req.http.Cookie;
   }

  if (req.url ~ "/graphql" && !req.http.X-Magento-Cache-Id && req.http.Authorization ~ "^Bearer") {
        return (pass);
    }

  return (hash);
}

sub vcl_hash {
    if (req.http.cookie ~ "X-Magento-Vary=") {
        hash_data(regsub(req.http.cookie, "^.*?X-Magento-Vary=([^;]+);*.*$", "\1"));
    }

    # For multi site configurations to not cache each other's content
    if (req.http.host) {
        hash_data(req.http.host);
    } else {
        hash_data(server.ip);
    }

    # To make sure http users don't see ssl warning
    if (req.http.SSL-OFFLOADED) {
        hash_data(req.http.SSL-OFFLOADED);
    }
    

    if (req.url ~ "/graphql") {
        call process_graphql_headers;
    }
}

sub process_graphql_headers {
    if (req.http.Store) {
        hash_data(req.http.Store);
    }
    if (req.http.Content-Currency) {
        hash_data(req.http.Content-Currency);
    }
}

sub vcl_backend_response {

    set beresp.grace = 3d;

    if (beresp.http.content-type ~ "text") {
        set beresp.do_esi = true;
    }

    if (bereq.url ~ "\.js$" || beresp.http.content-type ~ "text") {
        set beresp.do_gzip = true;
    }

    if (beresp.http.X-Magento-Debug) {
        set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control;
    }

    # cache only successfully responses and 404s
    if (beresp.status != 200 && beresp.status != 404) {
        set beresp.ttl = 0s;
        set beresp.uncacheable = true;
        return (deliver);
    } elsif (beresp.http.Cache-Control ~ "private") {
        set beresp.uncacheable = true;
        set beresp.ttl = 86400s;
        return (deliver);
    }

    # validate if we need to cache it and prevent from setting cookie
    if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) {
        unset beresp.http.set-cookie;
    }

   # If page is not cacheable then bypass varnish for 2 minutes as Hit-For-Pass
   #if (beresp.ttl <= 0s ||
   #    beresp.http.Surrogate-control ~ "no-store" ||
   #    (!beresp.http.Surrogate-Control &&
   #    beresp.http.Cache-Control ~ "no-cache|no-store") ||
   #    beresp.http.Vary == "*") {
   #     # Mark as Hit-For-Pass for the next 2 minutes
   #     set beresp.ttl = 120s;
   #     set beresp.uncacheable = true;
   # }

    # Bypass shopping cart, checkout and search requests
    if (bereq.url ~ "/checkout" || bereq.url ~ "/catalogsearch") {
        set beresp.uncacheable = true;
        set beresp.ttl = 120s;
        return(deliver);
    }
    #add 20210628
    if (bereq.url ~ "/adyen") {
        set beresp.uncacheable = true;
        set beresp.ttl = 120s;
        return(deliver);
    }
    #add 20210628
    # Bypass health check requests
    if (bereq.url ~ "/health_check.php") {
        set beresp.uncacheable = true;
        set beresp.ttl = 120s;
        return(deliver);
    }

   

    #if (bereq.url ~ "^/(pub/)?(media|static)/") {
    #    set beresp.uncacheable = true;
    #    set beresp.ttl = 120s;
    #    return(deliver);
    #}


    return (deliver);
}


sub vcl_deliver {
    # Add debug headers
    
    if (resp.http.x-varnish ~ " ") {
        set resp.http.X-Magento-Cache-Debug = "HIT";
        set resp.http.Grace = req.http.grace;
    } else {
        set resp.http.X-Magento-Cache-Debug = "MISS";
    }

 if (!resp.http.X-Magento-Debug) {
        unset resp.http.Age;
    }

    unset resp.http.X-Magento-Debug;
    unset resp.http.X-Magento-Tags;
    unset resp.http.X-Powered-By;
    unset resp.http.Server;
    unset resp.http.X-Varnish;
    unset resp.http.Via;
    unset resp.http.Link;
}

sub vcl_hit {
    if (obj.ttl >= 0s) {
        # Hit within TTL period
        return (deliver);
    }
    if (std.healthy(req.backend_hint)) {
        if (obj.ttl + 300s > 0s) {
            # Hit after TTL expiration, but within grace period
            set req.http.grace = "normal (healthy server)";
            return (deliver);
        } else {
            # Hit after TTL and grace expiration
            return (restart);
        }
    } else {
        # server is not healthy, retrieve from cache
        set req.http.grace = "unlimited (unhealthy server)";
        return (deliver);
    }
}

Answer 1

Usually I would ask you to send me the output of sudo varnishlog -g request -q "ReqUrl eq '/'". But the output you present already has some indicators.

I see a bunch of Set-Cookie headers in your output. The built-in VCL doesn't store objects in the cache when Set-Cookie headers are set.

However, the Hit-For-Miss behavior in Varnish will store the object in the cache when the next request doesn't contain the Set-Cookie headers. And that's entirely realistic, because typically you would only set the cookies once and then return output without these headers.

Please ensure that Set-Cookie headers are only used once and don't appear in the output on subsequent responses. If they do appear every time, you need to fix this.

Please also ask yourself the question why cookies need to be set when a user is not logged in.

UPDATE

After having checked the VCL, I still think your problem is related to Set-Cookie headers being set.

The following VCL code appears in your sub vcl_backend_response subroutine:

if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) {
    unset beresp.http.set-cookie;
}

Its role is to remove the Set-Cookie headers for HTTP GET & HEAD if the TTL is greater than zero.

However, this is the value of the corresponding Cache-Control headers:

Cache-Control: public, must-revalidate, proxy-revalidate, max-age=0

The max-age=0 value sets beresp.ttl to zero, which should bypass the cache.

In your case I'm not 100% sure whether the cache is actually bypassed or if the object is stored in the cache for zero seconds. Eitherway, a miss takes place.

The fact that this header is set, is usually related to cookies being set. A Set-Cookie response cannot be cached, that's why the web server or application server usually uses the Cache-Control header to force cache bypassing.

If you solve the Set-Cookie issue, you will probably solve the cache misses. However, the VCL file contains all sorts of weird constructs. Zero TTLs are a bad idea because they bypass the so-called waiting list which is used for request collapsing.

If you have a lot of traffic on paths with a zero TTL, Varnish will probably not perform as expected and your Magento shop may slow down quite bit.

If things aren't cacheable, I would recommend using set beresp.uncacheable=true with the default TTL of 120s. This is very waitlist friendly and for some paths this already happens in your VCL.

The fact that the following code block is commented out is slightly worrying, because it's part of the built-in VCL and protects you:

   # If page is not cacheable then bypass varnish for 2 minutes as Hit-For-Pass
   #if (beresp.ttl <= 0s ||
   #    beresp.http.Surrogate-control ~ "no-store" ||
   #    (!beresp.http.Surrogate-Control &&
   #    beresp.http.Cache-Control ~ "no-cache|no-store") ||
   #    beresp.http.Vary == "*") {
   #     # Mark as Hit-For-Pass for the next 2 minutes
   #     set beresp.ttl = 120s;
   #     set beresp.uncacheable = true;
   # }

I would also recommand that you look at https://www.varnish-software.com/developers/tutorials/configuring-varnish-magento/ for the VCL file we recommend. It might require some tuning for your use case, because your VCL file contains extra cache bypasses for specific URL patterns.