From 835579b338deca98751fbcf3444928f26e293543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sun, 2 Feb 2025 11:48:37 +0100 Subject: [PATCH] Re-introduce the LRU-evicted identities in change set calculation This is a follow up to db28695ff505f84aee69c72dcc9e192f674c86a1 -- that commit dropped the cache items evicted in the LRU process. This was done as performance optimization for large Hugo sites. That made much sense, but now there's a slight chance that we miss out on a change when rebuilding. This commit fixes this by applying the same logic to the evicted items as if they were still in the cache. This should preserve the performance gains in db28695ff505f84aee69c72dcc9e192f674c86a1 and close the hole for the possible false negatives. --- hugolib/content_map_page.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go index 9485f2e13..fcc650c39 100644 --- a/hugolib/content_map_page.go +++ b/hugolib/content_map_page.go @@ -1124,7 +1124,7 @@ func (h *HugoSites) resolveAndClearStateForIdentities( cachebuster func(s string) bool, changes []identity.Identity, ) error { // Drain the cache eviction stack to start fresh. - h.Deps.MemCache.DrainEvictedIdentities() + evictedStart := h.Deps.MemCache.DrainEvictedIdentities() h.Log.Debug().Log(logg.StringFunc( func() string { @@ -1200,6 +1200,27 @@ func (h *HugoSites) resolveAndClearStateForIdentities( for _, c := range evicted { changes = append(changes, c.Identity) } + + if len(evictedStart) > 0 { + // In low memory situations and/or very big sites, there can be a lot of unrelated evicted items, + // but there's a chance that some of them are related to the changes we are about to process, + // so check. + depsFinder := identity.NewFinder(identity.FinderConfig{}) + var addends []identity.Identity + for _, ev := range evictedStart { + for _, id := range changes { + if cachebuster != nil && cachebuster(ev.Key.(string)) { + addends = append(addends, ev.Identity) + break + } + if r := depsFinder.Contains(id, ev.Identity, -1); r > 0 { + addends = append(addends, ev.Identity) + break + } + } + } + changes = append(changes, addends...) + } } else { // Mass eviction, we might as well invalidate everything. changes = []identity.Identity{identity.GenghisKhan}