Mageno 2.4.6 with Apache - Varnish - Nginx Proxy on HTTPS

Question

System:
Ubuntu 22.04 LTS
Varnish Cache 7.0.3
Magento 2.4.6-p2
Ngnix 1.8
Apache 2.4

Magento (Setting Web) set Base URLs with HTTP http://magento.com
Magento (Setting Web) set Base URLs (Secure) with HTTPS https://magento.com

I'm trying to set up a site with Apache (Webserver) - Varnish (Cache) - Nginx (Proxy):

Apache port 7080 and 7081
Varnish port 6081
Nginx Proxy port 80 and 443

The purpose is this:
(if not cache varnish)
Request Client -> Nginx -> Varnish -> Apache Response => Varnish ->Nginx (HTTPS)

(if cache is in varnish)
Request Client -> Nginx -> Varnish Response => Nginx (HTTPS)

Apache config

/etc/apache2/sites-available/magento-ssl.conf

<IfModule mod_ssl.c>
<VirtualHost _default_:7081>
    ServerAdmin webmaster@localhost
    ServerName magento.com
    ServerAlias magento.com
    DocumentRoot /var/www/html/magento
    
    <Directory /var/www/html/magento>
        Options -Indexes +FollowSymLinks +MultiViews
        AllowOverride All
        Require all granted
    </Directory>

    <FilesMatch \.php$>
        SetHandler "proxy:unix:/run/php/php8.2-fpm.sock|fcgi://localhost/"
    </FilesMatch>

    ErrorLog ${APACHE_LOG_DIR}/error.log
    CustomLog ${APACHE_LOG_DIR}/access.log combined

    SSLEngine on

    SSLCertificateFile  /etc/ssl/certs/ssl-cert-snakeoil.pem
    SSLCertificateKeyFile /etc/ssl/private/ssl-cert-snakeoil.key

    <FilesMatch "\.(cgi|shtml|phtml|php)$">
            SSLOptions +StdEnvVars
    </FilesMatch>
    <Directory /usr/lib/cgi-bin>
            SSLOptions +StdEnvVars
    </Directory>

</VirtualHost>

Nginx config

/etc/nginx/sites-available/magento

    server {   
        listen 80;
        server_name magento.com www.magento.com;
        
        location / {
            proxy_buffer_size 128k;
            proxy_buffers 4 256k;
            proxy_busy_buffers_size 256k;
            
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_pass http://magento.com:6081;
    
        }
    }
    
    server {   
        listen 443 ssl;
        server_name magento.com www.magento.com;
    
        include snippets/snakeoil.conf;
        include snippets/ssl-params.conf;
    
        location / {
            
            proxy_buffer_size 128k;
            proxy_buffers 4 256k;
            proxy_busy_buffers_size 256k;
        
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            
            proxy_pass http://magento.com:6081;
        }
    }

Varnish config

/etc/systemd/system/varnish.service

[Unit]
Description=Varnish Cache, a high-performance HTTP accelerator
After=network-online.target nss-lookup.target

[Service]
Type=forking
KillMode=process

LimitNOFILE=131072

LimitMEMLOCK=85983232

LimitCORE=infinity

ExecStart=/usr/sbin/varnishd \
    -a :6081 \
    -a localhost:6082,PROXY \
    -p feature=+http2 \
    -f /etc/varnish/default.vcl \
    -s malloc,256m \
    -p http_resp_hdr_len=65536 \
    -p http_resp_size=98304

ExecReload=/usr/sbin/varnishreload

[Install]
WantedBy=multi-user.target

Varnish config

/etc/systemd/system/varnish.service.d/customexec.conf

     Service]
    ExecStart=
    ExecStart=/usr/sbin/varnishd -a :6061 -T localhost:6082 -f /etc/varnish/default.vcl -S /etc/varnish/secret -s default,256m

Varnish config

/etc/varnish/default.vcl

# VCL version 5.0 is not supported so it should be 4.0 even though actually used Varnish version is 6
    vcl 4.0;
    
    import std;
    
    import proxy;
    
    # The minimal Varnish version is 6.0
    # For SSL offloading, pass the following header in your proxy server or load balancer: 'SSL-OFFLOADED: https'
    
    backend default {
        .host = "magento.com";
        .port = "7080";
        .first_byte_timeout = 600s;
        .probe = {
            .url = "/pub/health_check.php";
            .timeout = 2s;
            .interval = 5s;
            .window = 10;
            .threshold = 5;
        }
    }
    
    acl purge {
        "magento.com";
    }
    
    sub vcl_recv {
        if (req.restarts > 0) {
            set req.hash_always_miss = true;
        }
        
        if (req.method == "PURGE") {
            if (client.ip !~ purge) {
                return (synth(405, "Method not allowed"));
            }

            if (!req.http.X-Magento-Tags-Pattern && !req.http.X-Pool) {
                return (synth(400, "X-Magento-Tags-Pattern or X-Pool header required"));
            }
            if (req.http.X-Magento-Tags-Pattern) {
              ban("obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern);
            }
            if (req.http.X-Pool) {
              ban("obj.http.X-Pool ~ " + req.http.X-Pool);
            }
            return (synth(200, "Purged"));
        }
        
        if (req.method != "GET" &&
            req.method != "HEAD" &&
            req.method != "PUT" &&
            req.method != "POST" &&
            req.method != "TRACE" &&
            req.method != "OPTIONS" &&
            req.method != "DELETE") {
              /* Non-RFC2616 or CONNECT which is weird. */
              return (pipe);
        }
        
        
        # We only deal with GET and HEAD by default
        if (req.method != "GET" && req.method != "HEAD") {
            return (pass);
        }
    
        # Bypass customer, shopping cart, checkout
        if (req.url ~ "/customer" || req.url ~ "/checkout") {
            return (pass);
        }
    
        # Bypass health check requests
        if (req.url ~ "^/(pub/)?(health_check.php)$") {
            return (pass);
        }
        
        # Set initial grace period usage status
        set req.http.grace = "none";
    
        # normalize url in case of leading HTTP scheme and domain
        set req.url = regsub(req.url, "^http[s]?://", "");
    
        # collect all cookies
        std.collect(req.http.Cookie);
        
        # Compression filter. See https://www.varnish-cache.org/trac/wiki/FAQ/Compression
        if (req.http.Accept-Encoding) {
            if (req.url ~ "\.(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|flv)$") {
                # No point in compressing these
                unset req.http.Accept-Encoding;
            } elsif (req.http.Accept-Encoding ~ "gzip") {
                set req.http.Accept-Encoding = "gzip";
            } elsif (req.http.Accept-Encoding ~ "deflate" && req.http.user-agent !~ "MSIE") {
                set req.http.Accept-Encoding = "deflate";
            } else {
                # unknown algorithm
                unset req.http.Accept-Encoding;
            }
        }    
        
        # Remove all marketing get parameters to minimize the cache objects
        if (req.url ~ "(\?|&)(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=") {
            set req.url = regsuball(req.url, "(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=[-_A-z0-9+()%.]+&?", "");
            set req.url = regsub(req.url, "[?|&]+$", "");
        }
    
        # Static files caching
        if (req.url ~ "^/(pub/)?(media|static)/") {
            # Static files should not be cached by default
            return (pass);
    
            # But if you use a few locales and don't use CDN you can enable caching static files by commenting previous line (#return (pass);) and uncommenting next 3 lines
            #unset req.http.Https;
            #unset req.http.SSL-OFFLOADED;
            #unset req.http.Cookie;
        }
    
        # Bypass authenticated GraphQL requests without a X-Magento-Cache-Id
        if (req.url ~ "/graphql" && !req.http.X-Magento-Cache-Id && req.http.Authorization ~ "^Bearer") {
            return (pass);
        }
    
        return (hash);    
    }
    
    sub vcl_hash {
        if ((req.url !~ "/graphql" || !req.http.X-Magento-Cache-Id) && req.http.cookie ~ "X-Magento-Vary=") {
            hash_data(regsub(req.http.cookie, "^.*?X-Magento-Vary=([^;]+);*.*$", "\1"));
        }
    
        # To make sure http users don't see ssl warning
        if (req.http.SSL-OFFLOADED) {
            hash_data(req.http.SSL-OFFLOADED);
        }
        
    
        if (req.url ~ "/graphql") {
            call process_graphql_headers;
        }
    }
    
    
    sub process_graphql_headers {
        if (req.http.X-Magento-Cache-Id) {
            hash_data(req.http.X-Magento-Cache-Id);
    
            # When the frontend stops sending the auth token, make sure users stop getting results cached for logged-in users
            if (req.http.Authorization ~ "^Bearer") {
                hash_data("Authorized");
            }
        }
    
        if (req.http.Store) {
            hash_data(req.http.Store);
        }
    
        if (req.http.Content-Currency) {
            hash_data(req.http.Content-Currency);
        }
    }
    
    
    sub vcl_backend_response {
    
        set beresp.grace = 3d;
    
        if (beresp.http.content-type ~ "text") {
            set beresp.do_esi = true;
        }
    
        if (bereq.url ~ "\.js$" || beresp.http.content-type ~ "text") {
            set beresp.do_gzip = true;
        }
    
        if (beresp.http.X-Magento-Debug) {
            set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control;
        }
    
        # cache only successfully responses and 404s that are not marked as private
        if (beresp.status != 200 &&
                beresp.status != 404 &&
                beresp.http.Cache-Control ~ "private") {
            set beresp.uncacheable = true;
            set beresp.ttl = 86400s;
            return (deliver);
        }
    
        # validate if we need to cache it and prevent from setting cookie
        if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) {
            unset beresp.http.set-cookie;
        }
    
       # If page is not cacheable then bypass varnish for 2 minutes as Hit-For-Pass
       if (beresp.ttl <= 0s ||
           beresp.http.Surrogate-control ~ "no-store" ||
           (!beresp.http.Surrogate-Control &&
           beresp.http.Cache-Control ~ "no-cache|no-store") ||
           beresp.http.Vary == "*") {
            # Mark as Hit-For-Pass for the next 2 minutes
            set beresp.ttl = 120s;
            set beresp.uncacheable = true;
       }
    
       # If the cache key in the Magento response doesn't match the one that was sent in the request, don't cache under the request's key
       if (bereq.url ~ "/graphql" && bereq.http.X-Magento-Cache-Id && bereq.http.X-Magento-Cache-Id != beresp.http.X-Magento-Cache-Id) {
          set beresp.ttl = 0s;
          set beresp.uncacheable = true;
       }
    
        return (deliver);
    }
    
    sub vcl_deliver {
        if (resp.http.x-varnish ~ " ") {
            set resp.http.X-Magento-Cache-Debug = "HIT";
            set resp.http.Grace = req.http.grace;
        } else {
            set resp.http.X-Magento-Cache-Debug = "MISS";
        }
    
        # Not letting browser to cache non-static files.
        if (resp.http.Cache-Control !~ "private" && req.url !~ "^/(pub/)?(media|static)/") {
            set resp.http.Pragma = "no-cache";
            set resp.http.Expires = "-1";
            set resp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0";
        }
    
        if (!resp.http.X-Magento-Debug) {
            unset resp.http.Age;
        }
        #unset resp.http.X-Magento-Debug;
        #unset resp.http.X-Magento-Tags;
        #unset resp.http.X-Powered-By;
        #unset resp.http.Server;
        #unset resp.http.X-Varnish;
        #unset resp.http.Via;
        #unset resp.http.Link;
    }
    
    sub vcl_hit {
        if (obj.ttl >= 0s) {
            # Hit within TTL period
            return (deliver);
        }
        if (std.healthy(req.backend_hint)) {
            if (obj.ttl + 300s > 0s) {
                # Hit after TTL expiration, but within grace period
                set req.http.grace = "normal (healthy server)";
                return (deliver);
            } else {
                # Hit after TTL and grace expiration
                return (restart);
            }
        } else {
            # server is not healthy, retrieve from cache
            set req.http.grace = "unlimited (unhealthy server)";
            return (deliver);
        }   
    }

on http://magento.com everything works correctly.

if I go to https://magento.com it loads the static resources in http for example

<link rel="stylesheet" type="text/css" media="all" href="http://magento.com/static/version1698239072/frontend/Mypackage/mytheme/en_US/css/styles-m.min.css">

All static file css, js in <header> section have http with prefix

Any help would be greatly appreciated.

Answer 1

Replace the SSL-OFFLOADED header in your VCL file with X-Forwarded-Proto to ensure that there is protocol awareness in the cache hash.

The following line is part of your vcl_hash subroutine:

hash_data(req.http.SSL-OFFLOADED);

This means that the value of the SSL-OFFLOADED header is used in the creation of the hash. This ads protocol awareness to the URL & Host header that are used to create the hash.

The problem is that this SSL-OFFLOADED header is nowhere to be found in your Nginx proxy config. Instead I notice the conventional X-Forwarded-Proto header.

So please replace the occurences req.http.SSL-OFFLOADED in your VCL with req.http.X-Forwarded-Proto.

That should do the trick.