When I change any objects in the Magento backend, a PURGE request is sent to Varnish which results in a slow response time once a customer requests the changed product. The site is then loaded into cache, so that the following customers will have a faster page loading.
So far so good.
I want to prevent that this “unlucky” customer is experiencing the slow reponse.
I want that the first request after purging/invalidating the cache still serves the old value, while fetching the new value from the backend, putting it into the Varnish cache and delivering it upon the next request as a HIT on the cache.
I already discovered the “soft purge” function, but did not manage to make it work. It either did not work or resulted in the product not being updated at all.
The following Varnish config was generated using the Magento backend. Except for the hostnames, everything is on default.
# VCL version 5.0 is not supported so it should be 4.0 even though actually used Varnish version is 6
vcl 4.0;
import std;
# The minimal Varnish version is 6.0
# For SSL offloading, pass the following header in your proxy server or load balancer: 'X-Forwarded-Proto: https'
backend default {
.host = "web-host01.example.com;
.port = "80";
.first_byte_timeout = 600s;
.probe = {
.url = "/pub/health_check.php";
.timeout = 2s;
.interval = 5s;
.window = 10;
.threshold = 5;
}
}
acl purge {
"localhost";
"web-host01.example.com";
}
sub vcl_recv {
if (req.restarts > 0) {
set req.hash_always_miss = true;
}
if (req.method == "PURGE") {
if (client.ip !~ purge) {
return (synth(405, "Method not allowed"));
}
# To use the X-Pool header for purging varnish during automated deployments, make sure the X-Pool header
# has been added to the response in your backend server config. This is used, for example, by the
# capistrano-magento2 gem for purging old content from varnish during it's deploy routine.
if (!req.http.X-Magento-Tags-Pattern && !req.http.X-Pool) {
return (synth(400, "X-Magento-Tags-Pattern or X-Pool header required"));
}
if (req.http.X-Magento-Tags-Pattern) {
ban("obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern);
}
if (req.http.X-Pool) {
ban("obj.http.X-Pool ~ " + req.http.X-Pool);
}
return (synth(200, "Purged"));
}
if (req.method != "GET" &&
req.method != "HEAD" &&
req.method != "PUT" &&
req.method != "POST" &&
req.method != "TRACE" &&
req.method != "OPTIONS" &&
req.method != "DELETE") {
/* Non-RFC2616 or CONNECT which is weird. */
return (pipe);
}
# We only deal with GET and HEAD by default
if (req.method != "GET" && req.method != "HEAD") {
return (pass);
}
# Bypass customer, shopping cart, checkout
if (req.url ~ "/customer" || req.url ~ "/checkout") {
return (pass);
}
# Bypass health check requests
if (req.url ~ "^/(pub/)?(health_check.php)$") {
return (pass);
}
# Set initial grace period usage status
set req.http.grace = "none";
# normalize url in case of leading HTTP scheme and domain
set req.url = regsub(req.url, "^http[s]?://", "");
# collect all cookies
std.collect(req.http.Cookie);
# Compression filter. See https://www.varnish-cache.org/trac/wiki/FAQ/Compression
if (req.http.Accept-Encoding) {
if (req.url ~ ".(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|flv)$") {
# No point in compressing these
unset req.http.Accept-Encoding;
} elsif (req.http.Accept-Encoding ~ "gzip") {
set req.http.Accept-Encoding = "gzip";
} elsif (req.http.Accept-Encoding ~ "deflate" && req.http.user-agent !~ "MSIE") {
set req.http.Accept-Encoding = "deflate";
} else {
# unknown algorithm
unset req.http.Accept-Encoding;
}
}
# Remove all marketing get parameters to minimize the cache objects
if (req.url ~ "(?|&)(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=") {
set req.url = regsuball(req.url, "(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=[-_A-z0-9+()%.]+&?", "");
set req.url = regsub(req.url, "[?|&]+$", "");
}
# Static files caching
if (req.url ~ "^/(pub/)?(media|static)/") {
# Static files should not be cached by default
return (pass);
# But if you use a few locales and don't use CDN you can enable caching static files by commenting previous line (#return (pass);) and uncommenting next 3 lines
#unset req.http.Https;
#unset req.http.X-Forwarded-Proto;
#unset req.http.Cookie;
}
# Bypass authenticated GraphQL requests without a X-Magento-Cache-Id
if (req.url ~ "/graphql" && !req.http.X-Magento-Cache-Id && req.http.Authorization ~ "^Bearer") {
return (pass);
}
return (hash);
}
sub vcl_hash {
if ((req.url !~ "/graphql" || !req.http.X-Magento-Cache-Id) && req.http.cookie ~ "X-Magento-Vary=") {
hash_data(regsub(req.http.cookie, "^.*?X-Magento-Vary=([^;]+);*.*$", "1"));
}
# To make sure http users don't see ssl warning
if (req.http.X-Forwarded-Proto) {
hash_data(req.http.X-Forwarded-Proto);
}
if (req.url ~ "/graphql") {
call process_graphql_headers;
}
}
sub process_graphql_headers {
if (req.http.X-Magento-Cache-Id) {
hash_data(req.http.X-Magento-Cache-Id);
# When the frontend stops sending the auth token, make sure users stop getting results cached for logged-in users
if (req.http.Authorization ~ "^Bearer") {
hash_data("Authorized");
}
}
if (req.http.Store) {
hash_data(req.http.Store);
}
if (req.http.Content-Currency) {
hash_data(req.http.Content-Currency);
}
}
sub vcl_backend_response {
set beresp.grace = 3d;
if (beresp.http.content-type ~ "text") {
set beresp.do_esi = true;
}
if (bereq.url ~ ".js$" || beresp.http.content-type ~ "text") {
set beresp.do_gzip = true;
}
if (beresp.http.X-Magento-Debug) {
set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control;
}
# cache only successfully responses and 404s that are not marked as private
if (beresp.status != 200 &&
beresp.status != 404 &&
beresp.http.Cache-Control ~ "private") {
set beresp.uncacheable = true;
set beresp.ttl = 86400s;
return (deliver);
}
# validate if we need to cache it and prevent from setting cookie
if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) {
unset beresp.http.set-cookie;
}
# If page is not cacheable then bypass varnish for 2 minutes as Hit-For-Pass
if (beresp.ttl <= 0s ||
beresp.http.Surrogate-control ~ "no-store" ||
(!beresp.http.Surrogate-Control &&
beresp.http.Cache-Control ~ "no-cache|no-store") ||
beresp.http.Vary == "*") {
# Mark as Hit-For-Pass for the next 2 minutes
set beresp.ttl = 120s;
set beresp.uncacheable = true;
}
# If the cache key in the Magento response doesn't match the one that was sent in the request, don't cache under the request's key
if (bereq.url ~ "/graphql" && bereq.http.X-Magento-Cache-Id && bereq.http.X-Magento-Cache-Id != beresp.http.X-Magento-Cache-Id) {
set beresp.ttl = 0s;
set beresp.uncacheable = true;
}
return (deliver);
}
sub vcl_deliver {
if (resp.http.x-varnish ~ " ") {
set resp.http.X-Magento-Cache-Debug = "HIT";
set resp.http.Grace = req.http.grace;
} else {
set resp.http.X-Magento-Cache-Debug = "MISS";
}
# Not letting browser to cache non-static files.
if (resp.http.Cache-Control !~ "private" && req.url !~ "^/(pub/)?(media|static)/") {
set resp.http.Pragma = "no-cache";
set resp.http.Expires = "-1";
set resp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0";
}
if (!resp.http.X-Magento-Debug) {
unset resp.http.Age;
}
unset resp.http.X-Magento-Debug;
unset resp.http.X-Magento-Tags;
unset resp.http.X-Powered-By;
unset resp.http.Server;
unset resp.http.X-Varnish;
unset resp.http.Via;
unset resp.http.Link;
}
sub vcl_hit {
if (obj.ttl >= 0s) {
# Hit within TTL period
return (deliver);
}
if (std.healthy(req.backend_hint)) {
if (obj.ttl + 300s > 0s) {
# Hit after TTL expiration, but within grace period
set req.http.grace = "normal (healthy server)";
return (deliver);
} else {
# Hit after TTL and grace expiration
return (restart);
}
} else {
# server is not healthy, retrieve from cache
set req.http.grace = "unlimited (unhealthy server)";
return (deliver);
}
}
varnishlog after purging
* << BeReq >> 65548
- Begin bereq 65547 fetch
- VCL_use boot
- Timestamp Start: 1666813603.131250 0.000000 0.000000
- BereqMethod GET
- BereqURL /example-req-url
- BereqProtocol HTTP/1.0
- BereqHeader Host: staging.example.com
- BereqHeader X-Forwarded-Proto: https
- BereqHeader sec-ch-ua: "Google Chrome";v="107", "Chromium";v="107", "Not=A?Brand";v="24"
- BereqHeader sec-ch-ua-mobile: ?0
- BereqHeader sec-ch-ua-platform: "Linux"
- BereqHeader Upgrade-Insecure-Requests: 1
- BereqHeader User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36
- BereqHeader Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
- BereqHeader Sec-Fetch-Site: same-origin
- BereqHeader Sec-Fetch-Mode: navigate
- BereqHeader Sec-Fetch-User: ?1
- BereqHeader Sec-Fetch-Dest: document
- BereqHeader Referer: https://staging.example.com/path/to/something
- BereqHeader Accept-Language: en-US,en;q=0.9
- BereqHeader Cookie: COOKIES
- BereqHeader X-Forwarded-For: 192.168.10.10
- BereqHeader Via: 1.1 varnish-host01 (Varnish/7.2)
- BereqHeader grace: none
- BereqHeader Accept-Encoding: gzip
- BereqProtocol HTTP/1.1
- BereqHeader X-Varnish: 65548
- VCL_call BACKEND_FETCH
- VCL_return fetch
- Timestamp Fetch: 1666813603.131283 0.000032 0.000032
- Timestamp Connected: 1666813603.131767 0.000517 0.000484
- BackendOpen 27 web-host01 172.19.25.32 80 172.19.25.34 36108 connect
- Timestamp Bereq: 1666813603.131833 0.000583 0.000065
- Timestamp Beresp: 1666813604.043211 0.911961 0.911378
- BerespProtocol HTTP/1.1
- BerespStatus 200
- BerespReason OK
- BerespHeader Date: Wed, 26 Oct 2022 19:46:43 GMT
- BerespHeader Server: Apache/2.4.52 (Ubuntu)
- BerespHeader Expires: Thu, 27 Oct 2022 19:46:43 GMT
- BerespHeader Cache-Control: max-age=86400, public, s-maxage=86400
- BerespHeader Pragma: cache
- BerespHeader Set-Cookie: PHPSESSID=3drps5kqelveq4nl3cf41pdggd; expires=Wed, 26-Oct-2022 20:46:43 GMT; Max-Age=3600; path=/; domain=staging.example.com; secure; HttpOnly; SameSite=Lax
- BerespHeader Set-Cookie: form_key=3UR6dQy6ODHqy3w3; expires=Wed, 26-Oct-2022 20:46:43 GMT; Max-Age=3600; path=/; domain=staging.example.com; secure; SameSite=Lax
- BerespHeader Set-Cookie: PHPSESSID=3drps5kqelveq4nl3cf41pdggd; expires=Wed, 26-Oct-2022 20:46:43 GMT; Max-Age=3600; path=/; domain=staging.example.com; secure; HttpOnly; SameSite=Lax
- BerespHeader Set-Cookie: PHPSESSID=3drps5kqelveq4nl3cf41pdggd; expires=Wed, 26-Oct-2022 20:46:43 GMT; Max-Age=3600; path=/; domain=staging.example.com; secure; HttpOnly; SameSite=Lax
- BerespHeader Set-Cookie: X-Magento-Vary=2627569dcdbeca193d32905c799896fce3eeb169; expires=Wed, 26-Oct-2022 20:46:44 GMT; Max-Age=3600; path=/; secure; HttpOnly; SameSite=Lax
- BerespHeader X-Magento-Tags: store,cms_b,cms_b_custom_navigation_links,cms_b_header_top_message,cms_b_header_top_quicklink,cms_b_page_bottom_steps,cms_b_page_bottom_faq_outer,cms_b_page_bottom_faq,cms_b_page_bottom_faq_background,cms_b_prefooter_block,cms_b_footer_bo
- BerespHeader Content-Security-Policy-Report-Only: font-src fonts.gstatic.com https://widgets.trustedshops.com https://integrations.etrusted.com data: 'self' 'unsafe-inline'; form-action geostag.cardinalcommerce.com geo.cardinalcommerce.com 1eafstag.cardinalcommerce.c
- BerespHeader X-Content-Type-Options: nosniff
- BerespHeader X-XSS-Protection: 1; mode=block
- BerespHeader X-Frame-Options: SAMEORIGIN
- BerespHeader Vary: Accept-Encoding
- BerespHeader Content-Encoding: gzip
- BerespHeader Content-Length: 19070
- BerespHeader Content-Type: text/html; charset=UTF-8
- TTL RFC 86400 10 0 1666813604 1666813604 1666813603 1666900003 86400 cacheable
- VCL_call BACKEND_RESPONSE
- TTL VCL 86400 259200 0 1666813604 cacheable
- BerespUnset Set-Cookie: PHPSESSID=3drps5kqelveq4nl3cf41pdggd; expires=Wed, 26-Oct-2022 20:46:43 GMT; Max-Age=3600; path=/; domain=staging.example.com; secure; HttpOnly; SameSite=Lax
- BerespUnset Set-Cookie: form_key=3UR6dQy6ODHqy3w3; expires=Wed, 26-Oct-2022 20:46:43 GMT; Max-Age=3600; path=/; domain=staging.example.com; secure; SameSite=Lax
- BerespUnset Set-Cookie: PHPSESSID=3drps5kqelveq4nl3cf41pdggd; expires=Wed, 26-Oct-2022 20:46:43 GMT; Max-Age=3600; path=/; domain=staging.example.com; secure; HttpOnly; SameSite=Lax
- BerespUnset Set-Cookie: PHPSESSID=3drps5kqelveq4nl3cf41pdggd; expires=Wed, 26-Oct-2022 20:46:43 GMT; Max-Age=3600; path=/; domain=staging.example.com; secure; HttpOnly; SameSite=Lax
- BerespUnset Set-Cookie: X-Magento-Vary=2627569dcdbeca193d32905c799896fce3eeb169; expires=Wed, 26-Oct-2022 20:46:44 GMT; Max-Age=3600; path=/; secure; HttpOnly; SameSite=Lax
- VCL_return deliver
- Timestamp Process: 1666813604.043276 0.912026 0.000065
- Filters gunzip esi_gzip
- BerespUnset Content-Encoding: gzip
- BerespUnset Content-Length: 19070
- BerespHeader Content-Encoding: gzip
- Storage malloc s0
- Fetch_Body 3 length -
- Gzip G F E 87119 19971 80 159688 159698
- Gzip U F - 19070 87119 80 80 152490
- BackendClose 27 web-host01 recycle
- Timestamp BerespBody: 1666813604.050210 0.918960 0.006933
- Length 19971
- BereqAcct 1377 0 1377 6858 19070 25928
- End
* << Request >> 65547
- Begin req 65546 rxreq
- Timestamp Start: 1666813603.130872 0.000000 0.000000
- Timestamp Req: 1666813603.130872 0.000000 0.000000
- VCL_use boot
- ReqStart 192.168.10.10 3330 a0
- ReqMethod GET
- ReqURL /example-req-url
- ReqProtocol HTTP/1.0
- ReqHeader Host: staging.example.com
- ReqHeader X-Forwarded-Proto: https
- ReqHeader Connection: close
- ReqHeader Cache-Control: max-age=0
- ReqHeader sec-ch-ua: "Google Chrome";v="107", "Chromium";v="107", "Not=A?Brand";v="24"
- ReqHeader sec-ch-ua-mobile: ?0
- ReqHeader sec-ch-ua-platform: "Linux"
- ReqHeader Upgrade-Insecure-Requests: 1
- ReqHeader User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36
- ReqHeader Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
- ReqHeader Sec-Fetch-Site: same-origin
- ReqHeader Sec-Fetch-Mode: navigate
- ReqHeader Sec-Fetch-User: ?1
- ReqHeader Sec-Fetch-Dest: document
- ReqHeader Referer: https://staging.example.com/path/to/something
- ReqHeader Accept-Encoding: gzip, deflate
- ReqHeader Accept-Language: en-US,en;q=0.9
- ReqHeader Cookie: __delivery_area_postcode=00000; __delivery_area_storeId=3; store=beelitz; form_key=3UR6dQy6ODHqy3w3; mage-messages=; mage-cache-storage={}; mage-cache-storage-section-invalidation={}; recently_viewed_product={}; recently_viewed_product_previous={
- ReqHeader X-Forwarded-For: 192.168.10.10
- ReqHeader Via: 1.1 varnish-host01 (Varnish/7.2)
- VCL_call RECV
- ReqHeader grace: none
- ReqURL /example-req-url
- ReqUnset Accept-Encoding: gzip, deflate
- ReqHeader Accept-Encoding: gzip
- VCL_return hash
- VCL_call HASH
- VCL_return lookup
- ExpBan 3 banned lookup
- VCL_call MISS
- VCL_return fetch
- Link bereq 65548 fetch
- Timestamp Fetch: 1666813604.050238 0.919365 0.919365
- RespProtocol HTTP/1.1
- RespStatus 200
- RespReason OK
- RespHeader Date: Wed, 26 Oct 2022 19:46:43 GMT
- RespHeader Server: Apache/2.4.52 (Ubuntu)
- RespHeader Expires: Thu, 27 Oct 2022 19:46:43 GMT
- RespHeader Cache-Control: max-age=86400, public, s-maxage=86400
- RespHeader Pragma: cache
- RespHeader X-Magento-Tags: store,cms_b,cms_b_custom_navigation_links,cms_b_header_top_message,cms_b_header_top_quicklink,cms_b_page_bottom_steps,cms_b_page_bottom_faq_outer,cms_b_page_bottom_faq,cms_b_page_bottom_faq_background,cms_b_prefooter_block,cms_b_footer_bo
- RespHeader Content-Security-Policy-Report-Only: font-src fonts.gstatic.com https://widgets.trustedshops.com https://integrations.etrusted.com data: 'self' 'unsafe-inline'; form-action geostag.cardinalcommerce.com geo.cardinalcommerce.com 1eafstag.cardinalcommerce.c
- RespHeader X-Content-Type-Options: nosniff
- RespHeader X-XSS-Protection: 1; mode=block
- RespHeader X-Frame-Options: SAMEORIGIN
- RespHeader Vary: Accept-Encoding
- RespHeader Content-Type: text/html; charset=UTF-8
- RespHeader Content-Encoding: gzip
- RespHeader X-Varnish: 65547
- RespHeader Age: 0
- RespHeader Via: 1.1 varnish-host01 (Varnish/7.2)
- RespHeader Accept-Ranges: bytes
- VCL_call DELIVER
- RespHeader X-Magento-Cache-Debug: HIT
- RespHeader Grace: none
- RespUnset Pragma: cache
- RespHeader Pragma: no-cache
- RespUnset Expires: Thu, 27 Oct 2022 19:46:43 GMT
- RespHeader Expires: -1
- RespUnset Cache-Control: max-age=86400, public, s-maxage=86400
- RespHeader Cache-Control: no-store, no-cache, must-revalidate, max-age=0
- RespUnset Age: 0
- RespUnset X-Magento-Tags: store,cms_b,cms_b_custom_navigation_links,cms_b_header_top_message,cms_b_header_top_quicklink,cms_b_page_bottom_steps,cms_b_page_bottom_faq_outer,cms_b_page_bottom_faq,cms_b_page_bottom_faq_background,cms_b_prefooter_block,cms_b_footer_bo
- RespUnset Server: Apache/2.4.52 (Ubuntu)
- RespUnset X-Varnish: 65547
- RespUnset Via: 1.1 varnish-host01 (Varnish/7.2)
- VCL_return deliver
- Timestamp Process: 1666813604.050321 0.919449 0.000083
- Filters esi
- RespHeader Connection: close
- Link req 65549 esi 1
- Timestamp Resp: 1666813604.050855 0.919982 0.000533
- ReqAcct 1328 0 1328 5467 23108 28575
- End
* << Session >> 65546
- Begin sess 0 HTTP/1
- SessOpen 192.168.10.10 3330 a0 172.19.25.34 80 1666813603.129864 24
- Link req 65547 rxreq
- SessClose TX_EOF 0.921
- End
I’m currently running Varnish 7.2 in front of Magento 2.4.5. I also tested Varnish 6. So that’s not an issue.