Continuing my issue from this thread.
I think it’s not caching due to Cache-Control header. Firstly, let me share my updated VCL configuration below maybe my VCL has an issue somewhere.
vcl 4.0;
import std;
include "./vars.vcl";
include "./pagespeed-requirement.vcl";
sub vcl_recv {
if (req.restarts > 0) {
set req.hash_always_miss = true;
}
if (req.http.host ~ "www.domain1.com" || req.http.host ~ "domain1.com") {
set req.backend_hint = domain1;
return (pass);
} else if (req.http.host ~ "www.domain2.com" || req.http.host ~ "domain2.com") {
set req.backend_hint = domain2;
} else if (req.http.host ~ "career.domain3.com") {
set req.backend_hint = career;
return (pass);
} else {
set req.backend_hint = default;
return (pass);
}
if (req.url ~ "^/.well-known/") {
return (pass);
}
# Remove the proxy header to mitigate the httpoxy vulnerability
# See https://httpoxy.org/
unset req.http.proxy;
if (req.method == "PURGE") {
if (client.ip !~ purge) {
return (synth(405, "Method not allowed"));
}
# capistrano-magento2 gem for purging old content from varnish during it's deploy routine.
if (!req.http.X-Magento-Tags-Pattern && !req.http.X-Pool) {
return (synth(400, "X-Magento-Tags-Pattern or X-Pool header required"));
}
if (req.http.X-Magento-Tags-Pattern) {
ban("obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern);
}
# ban everything to catch assets as well
if (req.http.X-Magento-Tags-Pattern == ".*") {
ban("req.url ~ .*");
}
if (req.http.X-Pool) {
ban("obj.http.X-Pool ~ " + req.http.X-Pool);
}
return (synth(200, "Purged"));
}
if (req.http.X-Forwarded-Proto !~ "https") {
set req.http.location = "https://" + req.http.host + req.url;
return (synth(750, "Permanently moved"));
}
if (req.method != "GET" &&
req.method != "HEAD" &&
req.method != "PUT" &&
req.method != "POST" &&
req.method != "TRACE" &&
req.method != "OPTIONS" &&
req.method != "DELETE") {
/* Non-RFC2616 or CONNECT which is weird. */
return (pipe);
}
# We only deal with GET and HEAD by default
if (req.method != "GET" && req.method != "HEAD") {
return (pass);
}
# Bypass customer, shopping cart, checkout
if (req.url ~ "/customer" || req.url ~ "/checkout" || req.url ~ "/payment" || req.url ~ "/reclaim" || req.url ~ "/search" || req.url ~ "/order-confirmation-summary") {
return (pass);
}
# Bypass health check requests
if (req.url ~ "/health_check.php") {
return (pass);
}
# Set initial grace period usage status
set req.http.grace = "none";
# normalize url in case of leading HTTP scheme and domain
set req.url = regsub(req.url, "^http[s]?://", "");
# collect all cookies
std.collect(req.http.Cookie);
# Compression filter. See https://www.varnish-cache.org/trac/wiki/FAQ/Compression
if (req.http.Accept-Encoding) {
if (req.url ~ ".(jpg|jpeg|webp|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|flv|webp)$") {
# No point in compressing these
unset req.http.Accept-Encoding;
} elsif (req.http.Accept-Encoding ~ "gzip") {
set req.http.Accept-Encoding = "gzip";
} elsif (req.http.Accept-Encoding ~ "deflate" && req.http.user-agent !~ "MSIE") {
set req.http.Accept-Encoding = "deflate";
} else {
# unknown algorithm
unset req.http.Accept-Encoding;
}
}
# Remove all marketing get parameters to minimize the cache objects
if (req.url ~ "(?|&)(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=") {
set req.url = regsuball(req.url, "(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=[-_A-z0-9+()%.]+&?", "");
set req.url = regsub(req.url, "[?|&]+$", "");
}
# Static files caching
if (req.url ~ "^/(pub/)?(media|static)/") {
# Static files should not be cached by default
return (pass);
# But if you use a few locales and don't use CDN you can enable caching static files by commenting previous line (#return (pass);) and uncommenting next 3 lines
#unset req.http.Https;
#unset req.http.X-Forwarded-Proto;
#unset req.http.Cookie;
}
# Authenticated GraphQL requests should not be cached by default
if (req.url ~ "/graphql" && req.http.Authorization ~ "^Bearer") {
return (pass);
}
return (hash);
}
sub vcl_hash {
if ((req.url !~ "/graphql" || !req.http.X-Magento-Cache-Id) && req.http.cookie ~ "X-Magento-Vary=") {
hash_data(regsub(req.http.cookie, "^.*?X-Magento-Vary=([^;]+);*.*$", "1"));
}
# For multi site configurations to not cache each other's content
if (req.http.host) {
hash_data(req.http.host);
} else {
hash_data(server.ip);
}
# To make sure http users don't see ssl warning
if (req.http.X-Forwarded-Proto) {
hash_data(req.http.X-Forwarded-Proto);
}
if (req.http.cookie ~ "currency=") {
set req.http.X-TMP = regsub(req.http.cookie, ".*currency=([^;]+);.*","1");
hash_data(req.http.X-TMP);
unset req.http.X-TMP;
}
if (req.url ~ "/graphql") {
call process_graphql_headers;
}
}
sub process_graphql_headers {
if (req.http.X-Magento-Cache-Id) {
hash_data(req.http.X-Magento-Cache-Id);
# When the frontend stops sending the auth token, make sure users stop getting results cached for logged-in users
if (req.http.Authorization ~ "^Bearer") {
hash_data("Authorized");
}
}
if (req.http.Store) {
hash_data(req.http.Store);
}
if (req.http.Content-Currency) {
hash_data(req.http.Content-Currency);
}
}
sub vcl_backend_response {
set beresp.grace = 3d;
if (beresp.http.content-type ~ "text") {
set beresp.do_esi = true;
}
if (bereq.url ~ ".js$" || beresp.http.content-type ~ "text") {
set beresp.do_gzip = true;
}
if (beresp.http.X-Magento-Debug) {
set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control;
}
# cache only successfully responses and 404s
if (beresp.status != 200 && beresp.status != 404) {
set beresp.ttl = 0s;
set beresp.uncacheable = true;
return (deliver);
} elsif (beresp.http.Cache-Control ~ "private") {
set beresp.uncacheable = true;
set beresp.ttl = 86400s;
return (deliver);
}
# validate if we need to cache it and prevent from setting cookie
if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) {
unset beresp.http.set-cookie;
}
# If page is not cacheable then bypass varnish for 2 minutes as Hit-For-Pass
if (beresp.ttl <= 0s ||
beresp.http.Surrogate-control ~ "no-store" ||
(!beresp.http.Surrogate-Control &&
beresp.http.Cache-Control ~ "no-cache|no-store") ||
beresp.http.Vary == "*") {
# Mark as Hit-For-Pass for the next 2 minutes
set beresp.ttl = 120s;
set beresp.uncacheable = true;
}
return (deliver);
}
sub vcl_deliver {
if (resp.http.X-Magento-Debug) {
if (resp.http.x-varnish ~ " ") {
set resp.http.X-Magento-Cache-Debug = "HIT";
set resp.http.Grace = req.http.grace;
} else {
set resp.http.X-Magento-Cache-Debug = "MISS";
}
} else {
unset resp.http.Age;
}
if (obj.hits > 0) {
set resp.http.X-Cache = "HIT";
set resp.http.X-Cache-Hits = obj.hits;
} else {
set resp.http.X-Cache = "MISS";
}
# Not letting browser to cache non-static files.
if (resp.http.Cache-Control !~ "private" && req.url !~ "^/(pub/)?(media|static)/") {
set resp.http.Pragma = "no-cache";
set resp.http.Expires = "-1";
set resp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0";
}
unset resp.http.X-Magento-Debug;
unset resp.http.X-Magento-Tags;
unset resp.http.X-Powered-By;
unset resp.http.Server;
unset resp.http.X-Varnish;
unset resp.http.Via;
unset resp.http.Link;
}
sub vcl_synth {
if (resp.status == 750) {
set resp.http.location = req.http.location;
set resp.status = 301;
return (deliver);
}
}
sub vcl_hit {
if (obj.ttl >= 0s) {
# Hit within TTL period
return (deliver);
}
if (std.healthy(req.backend_hint)) {
if (obj.ttl + 300s > 0s) {
# Hit after TTL expiration, but within grace period
set req.http.grace = "normal (healthy server)";
return (deliver);
} else {
# Hit after TTL and grace expiration
return (restart);
}
} else {
# server is not healthy, retrieve from cache
set req.http.grace = "unlimited (unhealthy server)";
return (deliver);
}
}
We need to focus on domain2, the Cache-Control headers in response headers are max-age=0, no-cache, no-store. No custom code has cacheable=false in Magento’s app/code directory. Hence, I don’t know how is this being added.
I commented the following lines in vcl_backend_response and was able to see X-Cache: HIT on the browser only for the / home page. Other page’s don’t have this parameter at all.
if (beresp.ttl <= 0s ||
beresp.http.Surrogate-control ~ "no-store" ||
(!beresp.http.Surrogate-Control &&
beresp.http.Cache-Control ~ "no-cache|no-store") ||
beresp.http.Vary == "*") {
# Mark as Hit-For-Pass for the next 2 minutes
set beresp.ttl = 120s;
set beresp.uncacheable = true;
}
Upon checking the logs for command:
varnishlog -g request -b -i BeReqUrl -I BerespHeader:Cache-Control
-I BerespHeader:Expires -I BerespHeader:Vary -I BerespHeader:Set-Cookie
-I BerespHeader:Surrogate-Control -i TTL -q "TTL[6] eq 'uncacheable'"
I get following result for the main domain/home page. (https://www.domain2.com).
** << BeReq >> 65569
-- BereqURL /
-- BerespHeader Cache-Control: max-age=0, no-cache, s-maxage=60
-- TTL RFC 60 10 0 1724061342 1724061342 1724061342 0 60 cacheable
-- TTL VCL 60 259200 0 1724061342 cacheable
-- TTL VCL 120 259200 0 1724061342 cacheable
-- TTL VCL 120 259200 0 1724061342 uncacheable
-- BerespHeader Vary: Accept-Encoding
Also, there are no logs if I click on another category page or product, it’s as there was never made a request and x-cache is also missing in response headers. Probably all other pages/paths are being bypassed.
Update:
Below are my transaction logs for homepage on refresh.
* << Request >> 5
- Begin req 4 rxreq
- Timestamp Start: 1724150405.424135 0.000000 0.000000
- Timestamp Req: 1724150405.424135 0.000000 0.000000
- VCL_use boot
- ReqStart 127.0.0.1 35242 a0
- ReqMethod GET
- ReqURL /
- ReqProtocol HTTP/1.0
- ReqHeader Host: www.domain2.com
- ReqHeader X-Forwarded-Host: www.domain2.com
- ReqHeader X-Real-IP: 1.1.1.1
- ReqHeader X-Forwarded-For: 1.1.1.1, 1.1.1.1
- ReqHeader Ssl-Offloaded: 1
- ReqHeader X-Forwarded-Port: 443
- ReqHeader X-Forwarded-Proto: https
- ReqHeader X-Forwarded-Ssl: on
- ReqHeader Connection: close
- ReqHeader cdn-loop: cloudflare
- ReqHeader cf-ipcountry: PK
- ReqHeader accept-encoding: gzip, br
- ReqHeader cf-ray: 8b61c9c7cf07be84-SIN
- ReqHeader cf-visitor: {"scheme":"https"}
- ReqHeader cache-control: max-age=0
- ReqHeader sec-ch-ua: "Not)A;Brand";v="99", "Google Chrome";v="127", "Chromium";v="127"
- ReqHeader accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7
- ReqHeader sec-ch-ua-mobile: ?0
- ReqHeader upgrade-insecure-requests: 1
- ReqHeader user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36
- ReqHeader sec-ch-ua-platform: "Linux"
- ReqHeader sec-fetch-site: same-origin
- ReqHeader sec-fetch-mode: navigate
- ReqHeader sec-fetch-dest: empty
- ReqHeader accept-language: en-US,en;q=0.9
- ReqHeader priority: u=0, i
- ReqHeader cf-connecting-ip: 1.1.1.1
- ReqHeader cookie: ajs_anonymous_id=0ef0-485a-ae6a-d03d649b79e6; _gcl_au=1.1.1724011616; _fbp=fb.1.1724011616211; _hjSessionUser_3574595=56yfdvdvdvgfrvvd
- ReqUnset X-Forwarded-For: 1.1.1.1, 1.1.1.1
- ReqHeader X-Forwarded-For: 1.1.1.1, 1.1.1.1, 127.0.0.1
- VCL_call RECV
- ReqHeader PS-CapabilityList: ll,ii,dj,jw,ws:
- ReqHeader grace: none
- ReqURL /
- ReqUnset accept-encoding: gzip, br
- ReqHeader Accept-Encoding: gzip
- VCL_return hash
- VCL_call HASH
- VCL_return lookup
- HitMiss 3 115.365105
- VCL_call MISS
- VCL_return fetch
- Link bereq 6 fetch
- Timestamp Fetch: 1724150405.585118 0.160983 0.160983
- RespProtocol HTTP/1.1
- RespStatus 200
- RespReason OK
- RespHeader Server: nginx
- RespHeader Content-Type: text/html;charset=UTF-8
- RespHeader X-Content-Type-Options: nosniff
- RespHeader Date: Tue, 20 Aug 2024 10:40:05 GMT
- RespHeader X-Page-Speed: ngx_pagespeed
- RespHeader Link: </RootCmp_CMS_PAGE__default.9b247fcd07254236f444.js>; rel=preload; as=script; nopush
- RespHeader Link: </RootCmp_CMS_PAGE__default.9b247fcd07254236f444.js.gz>; rel=preload; as=script; nopush
- RespHeader Link: </runtime.8f084be222019779ec78.js>; rel=preload; as=script; nopush
- RespHeader Link: </vendors.2510f870ac43c344ee54.js>; rel=preload; as=script; nopush
- RespHeader Link: </client.7a54ea122b5c72f16096.js>; rel=preload; as=script; nopush
- RespHeader Cache-Control: max-age=0, no-cache
- RespHeader Content-Encoding: gzip
- RespHeader Vary: Accept-Encoding
- RespHeader X-Varnish: 5
- RespHeader Age: 0
- RespHeader Via: 1.1 varnish (Varnish/6.5)
- VCL_call DELIVER
- RespUnset Age: 0
- RespHeader X-Cache: MISS
- RespHeader Pragma: no-cache
- RespHeader Expires: -1
- RespUnset Cache-Control: max-age=0, no-cache
- RespHeader Cache-Control: no-store, no-cache, must-revalidate, max-age=0
- RespUnset Server: nginx
- RespUnset X-Varnish: 5
- RespUnset Via: 1.1 varnish (Varnish/6.5)
- RespUnset Link: </RootCmp_CMS_PAGE__default.9b247fcd07254236f444.js>; rel=preload; as=script; nopush
- RespUnset Link: </RootCmp_CMS_PAGE__default.9b247fcd07254236f444.js.gz>; rel=preload; as=script; nopush
- RespUnset Link: </runtime.8f084be222019779ec78.js>; rel=preload; as=script; nopush
- RespUnset Link: </vendors.2510f870ac43c344ee54.js>; rel=preload; as=script; nopush
- RespUnset Link: </client.7a54ea122b5c72f16096.js>; rel=preload; as=script; nopush
- VCL_return deliver
- Timestamp Process: 1724150405.585169 0.161034 0.000051
- Filters
- RespHeader Accept-Ranges: bytes
- RespHeader Content-Length: 72801
- RespHeader Connection: close
- Timestamp Resp: 1724150405.585305 0.161170 0.000135
- ReqAcct 2066 0 2066 377 72801 73178
- End
** << BeReq >> 6
-- Begin bereq 5 fetch
-- VCL_use boot
-- Timestamp Start: 1724150405.424291 0.000000 0.000000
-- BereqMethod GET
-- BereqURL /
-- BereqProtocol HTTP/1.0
-- BereqHeader Host: www.domain2.com
-- BereqHeader X-Forwarded-Host: www.domain2.com
-- BereqHeader X-Real-IP: 1.1.1.1
-- BereqHeader Ssl-Offloaded: 1
-- BereqHeader X-Forwarded-Port: 443
-- BereqHeader X-Forwarded-Proto: https
-- BereqHeader X-Forwarded-Ssl: on
-- BereqHeader cdn-loop: cloudflare
-- BereqHeader cf-ipcountry: PK
-- BereqHeader cf-ray: 8b61c9c7cf07be84-SIN
-- BereqHeader cf-visitor: {"scheme":"https"}
-- BereqHeader sec-ch-ua: "Not)A;Brand";v="99", "Google Chrome";v="127", "Chromium";v="127"
-- BereqHeader accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7
-- BereqHeader sec-ch-ua-mobile: ?0
-- BereqHeader upgrade-insecure-requests: 1
-- BereqHeader user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36
-- BereqHeader sec-ch-ua-platform: "Linux"
-- BereqHeader sec-fetch-site: same-origin
-- BereqHeader sec-fetch-mode: navigate
-- BereqHeader sec-fetch-dest: empty
-- BereqHeader accept-language: en-US,en;q=0.9
-- BereqHeader priority: u=0, i
-- BereqHeader cf-connecting-ip: 1.1.1.1
-- BereqHeader cookie: ajs_anonymous_id=0ef0-485a-ae6a-d03d649b79e6; _gcl_au=1.1.1777705759; _fbp=fb.1.1724011616211; _hjSessionUser_3574595=242424v24vhvgv
-- BereqHeader X-Forwarded-For: 1.1.1.1, 1.1.1.1, 127.0.0.1
-- BereqHeader PS-CapabilityList: ll,ii,dj,jw,ws:
-- BereqHeader grace: none
-- BereqHeader Accept-Encoding: gzip
-- BereqProtocol HTTP/1.1
-- BereqHeader X-Varnish: 6
-- VCL_call BACKEND_FETCH
-- VCL_return fetch
-- BackendOpen 29 domain2 127.0.0.1 8084 127.0.0.1 33002 reuse
-- Timestamp Bereq: 1724150405.424392 0.000100 0.000100
-- Timestamp Beresp: 1724150405.580200 0.155909 0.155808
-- BerespProtocol HTTP/1.1
-- BerespStatus 200
-- BerespReason OK
-- BerespHeader Server: nginx
-- BerespHeader Content-Type: text/html;charset=UTF-8
-- BerespHeader Transfer-Encoding: chunked
-- BerespHeader Connection: keep-alive
-- BerespHeader X-Content-Type-Options: nosniff
-- BerespHeader Date: Tue, 20 Aug 2024 10:40:05 GMT
-- BerespHeader X-Page-Speed: ngx_pagespeed
-- BerespHeader Link: </RootCmp_CMS_PAGE__default.9b247fcd07254236f444.js>; rel=preload; as=script; nopush
-- BerespHeader Link: </RootCmp_CMS_PAGE__default.9b247fcd07254236f444.js.gz>; rel=preload; as=script; nopush
-- BerespHeader Link: </runtime.8f084be222019779ec78.js>; rel=preload; as=script; nopush
-- BerespHeader Link: </vendors.2510f870ac43c344ee54.js>; rel=preload; as=script; nopush
-- BerespHeader Link: </client.7a54ea122b5c72f16096.js>; rel=preload; as=script; nopush
-- BerespHeader Cache-Control: max-age=0, no-cache
-- TTL RFC 0 10 0 1724150406 1724150406 1724150405 0 0 cacheable
-- VCL_call BACKEND_RESPONSE
-- TTL VCL 0 259200 0 1724150406 cacheable
-- TTL VCL 120 259200 0 1724150406 cacheable
-- TTL VCL 120 259200 0 1724150406 uncacheable
-- VCL_return deliver
-- Filters esi_gzip
-- BerespHeader Content-Encoding: gzip
-- BerespHeader Vary: Accept-Encoding
-- Storage malloc Transient
-- Fetch_Body 2 chunked -
-- Gzip G F E 98588 72801 80 582328 582338
-- BackendClose 29 domain2 recycle
-- Timestamp BerespBody: 1724150405.585110 0.160819 0.004910
-- Length 72801
-- BereqAcct 2091 0 2091 668 0 668
-- End