quentin/hugo-wasm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Fix some server rebuild issues for non-HTML custom output formats

Browse source 
The failing test case here is

* A custom search output format defined on the home page, marked as `noAlternative` and not `permalinkable`
* In fast render mode, when making a change to a data source for that search output format, the JSON file was not refreshed.

There are variants of the above, but the gist of it is:

* The change set was correctly determined, but since the search JSON file was not in the recently visited browser stack, we skipped rendering it.

Running with `hugo server --disableFastRender` would be a workaround for the above.

This commit fixes this by:

* Adding a check for the HTTP request header `Sec-Fetch-Mode = navigation` to the condition for if we should track server request as a user navigation (and not e.g. a HTTP request for a linked CSS stylesheet).
* Making sure that we compare against the real relative URL for non-permalinkable output formats.

Fixes #13014
This commit is contained in:
Bjørn Erik Pedersen 2025-01-23 12:46:08 +01:00
parent c939c33fd3
commit cd7dc7a372
11 changed files with 109 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
commands/hugobuilder.go
View file

@ -62,7 +62,7 @@ type hugoBuilder struct {
// Currently only set when in "fast render mode".
changeDetector *fileChangeDetector
visitedURLs *types.EvictingStringQueue
visitedURLs *types.EvictingQueue[string]
fullRebuildSem *semaphore.Weighted
debounce func(f func())
@ -1103,7 +1103,7 @@ func (c *hugoBuilder) rebuildSites(events []fsnotify.Event) (err error) {
if err != nil {
return
}
err = h.Build(hugolib.BuildCfg{NoBuildLock: true, RecentlyVisited: c.visitedURLs, ErrRecovery: c.errState.wasErr()}, events...)
err = h.Build(hugolib.BuildCfg{NoBuildLock: true, RecentlyTouched: c.visitedURLs, ErrRecovery: c.errState.wasErr()}, events...)
return
}
@ -1119,7 +1119,7 @@ func (c *hugoBuilder) rebuildSitesForChanges(ids []identity.Identity) (err error
}
whatChanged := &hugolib.WhatChanged{}
whatChanged.Add(ids...)
err = h.Build(hugolib.BuildCfg{NoBuildLock: true, WhatChanged: whatChanged, RecentlyVisited: c.visitedURLs, ErrRecovery: c.errState.wasErr()})
err = h.Build(hugolib.BuildCfg{NoBuildLock: true, WhatChanged: whatChanged, RecentlyTouched: c.visitedURLs, ErrRecovery: c.errState.wasErr()})
return
}

13
commands/server.go
View file

@ -85,9 +85,9 @@ const (
)
func newHugoBuilder(r *rootCommand, s *serverCommand, onConfigLoaded ...func(reloaded bool) error) *hugoBuilder {
var visitedURLs *types.EvictingStringQueue
var visitedURLs *types.EvictingQueue[string]
if s != nil && !s.disableFastRender {
visitedURLs = types.NewEvictingStringQueue(20)
visitedURLs = types.NewEvictingQueue[string](20)
}
return &hugoBuilder{
r: r,
@ -364,7 +364,10 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, net.Listener, string
}
if f.c.fastRenderMode && f.c.errState.buildErr() == nil {
if strings.HasSuffix(requestURI, "/") || strings.HasSuffix(requestURI, "html") || strings.HasSuffix(requestURI, "htm") {
// Sec-Fetch-Mode should be sent by all recent browser versions, see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Sec-Fetch-Mode#navigate
// Fall back to the file extension if not set.
// The main take here is that we don't want to have CSS/JS files etc. partake in this logic.
if r.Header.Get("Sec-Fetch-Mode") == "navigate" || strings.HasSuffix(requestURI, "/") || strings.HasSuffix(requestURI, "html") || strings.HasSuffix(requestURI, "htm") {
if !f.c.visitedURLs.Contains(requestURI) {
// If not already on stack, re-render that single page.
if err := f.c.partialReRender(requestURI); err != nil {
@ -838,7 +841,7 @@ func (c *serverCommand) partialReRender(urls ...string) (err error) {
defer func() {
c.errState.setWasErr(false)
}()
visited := types.NewEvictingStringQueue(len(urls))
visited := types.NewEvictingQueue[string](len(urls))
for _, url := range urls {
visited.Add(url)
}
@ -850,7 +853,7 @@ func (c *serverCommand) partialReRender(urls ...string) (err error) {
}
// Note: We do not set NoBuildLock as the file lock is not acquired at this stage.
err = h.Build(hugolib.BuildCfg{NoBuildLock: false, RecentlyVisited: visited, PartialReRender: true, ErrRecovery: c.errState.wasErr()})
err = h.Build(hugolib.BuildCfg{NoBuildLock: false, RecentlyTouched: visited, PartialReRender: true, ErrRecovery: c.errState.wasErr()})
return
}

37
common/types/evictingqueue.go
View file

@ -18,24 +18,24 @@ import (
"sync"
)
// EvictingStringQueue is a queue which automatically evicts elements from the head of
// EvictingQueue is a queue which automatically evicts elements from the head of
// the queue when attempting to add new elements onto the queue and it is full.
// This queue orders elements LIFO (last-in-first-out). It throws away duplicates.
// Note: This queue currently does not contain any remove (poll etc.) methods.
type EvictingStringQueue struct {
type EvictingQueue[T comparable] struct {
size int
vals []string
set map[string]bool
vals []T
set map[T]bool
mu sync.Mutex
zero T
}
// NewEvictingStringQueue creates a new queue with the given size.
func NewEvictingStringQueue(size int) *EvictingStringQueue {
return &EvictingStringQueue{size: size, set: make(map[string]bool)}
// NewEvictingQueue creates a new queue with the given size.
func NewEvictingQueue[T comparable](size int) *EvictingQueue[T] {
return &EvictingQueue[T]{size: size, set: make(map[T]bool)}
}
// Add adds a new string to the tail of the queue if it's not already there.
func (q *EvictingStringQueue) Add(v string) *EvictingStringQueue {
func (q *EvictingQueue[T]) Add(v T) *EvictingQueue[T] {
q.mu.Lock()
if q.set[v] {
q.mu.Unlock()
@ -54,7 +54,7 @@ func (q *EvictingStringQueue) Add(v string) *EvictingStringQueue {
return q
}
func (q *EvictingStringQueue) Len() int {
func (q *EvictingQueue[T]) Len() int {
if q == nil {
return 0
}
@ -64,7 +64,7 @@ func (q *EvictingStringQueue) Len() int {
}
// Contains returns whether the queue contains v.
func (q *EvictingStringQueue) Contains(v string) bool {
func (q *EvictingQueue[T]) Contains(v T) bool {
if q == nil {
return false
}
@ -74,12 +74,12 @@ func (q *EvictingStringQueue) Contains(v string) bool {
}
// Peek looks at the last element added to the queue.
func (q *EvictingStringQueue) Peek() string {
func (q *EvictingQueue[T]) Peek() T {
q.mu.Lock()
l := len(q.vals)
if l == 0 {
q.mu.Unlock()
return ""
return q.zero
}
elem := q.vals[l-1]
q.mu.Unlock()
@ -87,9 +87,12 @@ func (q *EvictingStringQueue) Peek() string {
}
// PeekAll looks at all the elements in the queue, with the newest first.
func (q *EvictingStringQueue) PeekAll() []string {
func (q *EvictingQueue[T]) PeekAll() []T {
if q == nil {
return nil
}
q.mu.Lock()
vals := make([]string, len(q.vals))
vals := make([]T, len(q.vals))
copy(vals, q.vals)
q.mu.Unlock()
for i, j := 0, len(vals)-1; i < j; i, j = i+1, j-1 {
@ -99,9 +102,9 @@ func (q *EvictingStringQueue) PeekAll() []string {
}
// PeekAllSet returns PeekAll as a set.
func (q *EvictingStringQueue) PeekAllSet() map[string]bool {
func (q *EvictingQueue[T]) PeekAllSet() map[T]bool {
all := q.PeekAll()
set := make(map[string]bool)
set := make(map[T]bool)
for _, v := range all {
set[v] = true
}

4
common/types/evictingqueue_test.go
View file

@ -23,7 +23,7 @@ import (
func TestEvictingStringQueue(t *testing.T) {
c := qt.New(t)
queue := NewEvictingStringQueue(3)
queue := NewEvictingQueue[string](3)
c.Assert(queue.Peek(), qt.Equals, "")
queue.Add("a")
@ -53,7 +53,7 @@ func TestEvictingStringQueueConcurrent(t *testing.T) {
var wg sync.WaitGroup
val := "someval"
queue := NewEvictingStringQueue(3)
queue := NewEvictingQueue[string](3)
for j := 0; j < 100; j++ {
wg.Add(1)

20
hugolib/hugo_sites.go
View file

@ -416,8 +416,8 @@ type BuildCfg struct {
// Set in server mode when the last build failed for some reason.
ErrRecovery bool
// Recently visited URLs. This is used for partial re-rendering.
RecentlyVisited *types.EvictingStringQueue
// Recently visited or touched URLs. This is used for partial re-rendering.
RecentlyTouched *types.EvictingQueue[string]
// Can be set to build only with a sub set of the content source.
ContentInclusionFilter *glob.FilenameFilter
@ -429,7 +429,7 @@ type BuildCfg struct {
}
// shouldRender returns whether this output format should be rendered or not.
func (cfg *BuildCfg) shouldRender(p *pageState) bool {
func (cfg *BuildCfg) shouldRender(infol logg.LevelLogger, p *pageState) bool {
if p.skipRender() {
return false
}
@ -457,18 +457,20 @@ func (cfg *BuildCfg) shouldRender(p *pageState) bool {
return false
}
if p.outputFormat().IsHTML {
// This is fast render mode and the output format is HTML,
// rerender if this page is one of the recently visited.
return cfg.RecentlyVisited.Contains(p.RelPermalink())
if relURL := p.getRelURL(); relURL != "" {
if cfg.RecentlyTouched.Contains(relURL) {
infol.Logf("render recently touched URL %q (%s)", relURL, p.outputFormat().Name)
return true
}
}
// In fast render mode, we want to avoid re-rendering the sitemaps etc. and
// other big listings whenever we e.g. change a content file,
// but we want partial renders of the recently visited pages to also include
// but we want partial renders of the recently touched pages to also include
// alternative formats of the same HTML page (e.g. RSS, JSON).
for _, po := range p.pageOutputs {
if po.render && po.f.IsHTML && cfg.RecentlyVisited.Contains(po.RelPermalink()) {
if po.render && po.f.IsHTML && cfg.RecentlyTouched.Contains(po.getRelURL()) {
infol.Logf("render recently touched URL %q, %s version of %s", po.getRelURL(), po.f.Name, p.outputFormat().Name)
return true
}
}

6
hugolib/hugo_sites_build.go
View file

@ -341,7 +341,7 @@ func (h *HugoSites) render(l logg.LevelLogger, config *BuildCfg) error {
loggers.TimeTrackf(l, start, h.buildCounters.loggFields(), "")
}()
siteRenderContext := &siteRenderContext{cfg: config, multihost: h.Configs.IsMultihost}
siteRenderContext := &siteRenderContext{cfg: config, infol: l, multihost: h.Configs.IsMultihost}
renderErr := func(err error) error {
if err == nil {
@ -902,12 +902,12 @@ func (h *HugoSites) processPartialFileEvents(ctx context.Context, l logg.LevelLo
needsPagesAssemble = true
if config.RecentlyVisited != nil {
if config.RecentlyTouched != nil {
// Fast render mode. Adding them to the visited queue
// avoids rerendering them on navigation.
for _, id := range changes {
if p, ok := id.(page.Page); ok {
config.RecentlyVisited.Add(p.RelPermalink())
config.RecentlyTouched.Add(p.RelPermalink())
}
}
}

4
hugolib/integrationtest_builder.go
View file

@ -487,11 +487,11 @@ func (s *IntegrationTestBuilder) BuildPartialE(urls ...string) (*IntegrationTest
if !s.Cfg.Running {
panic("BuildPartial can only be used in server mode")
}
visited := types.NewEvictingStringQueue(len(urls))
visited := types.NewEvictingQueue[string](len(urls))
for _, url := range urls {
visited.Add(url)
}
buildCfg := BuildCfg{RecentlyVisited: visited, PartialReRender: true}
buildCfg := BuildCfg{RecentlyTouched: visited, PartialReRender: true}
return s, s.build(buildCfg)
}

3
hugolib/page__paths.go
View file

@ -71,11 +71,12 @@ func newPagePaths(ps *pageState) (pagePaths, error) {
// Use the main format for permalinks, usually HTML.
permalinksIndex := 0
if f.Permalinkable {
// Unless it's permalinkable
// Unless it's permalinkable.
permalinksIndex = i
}
targets[f.Name] = targetPathsHolder{
relURL: relPermalink,
paths: paths,
OutputFormat: pageOutputFormats[permalinksIndex],
}

10
hugolib/page__per_output.go
View file

@ -469,13 +469,21 @@ type pagePerOutputProviders interface {
type targetPather interface {
targetPaths() page.TargetPaths
getRelURL() string
}
type targetPathsHolder struct {
paths page.TargetPaths
// relURL is usually the same as OutputFormat.RelPermalink, but can be different
// for non-permalinkable output formats. These shares RelPermalink with the main (first) output format.
relURL string
paths page.TargetPaths
page.OutputFormat
}
func (t targetPathsHolder) getRelURL() string {
return t.relURL
}
func (t targetPathsHolder) targetPaths() page.TargetPaths {
return t.paths
}

47
hugolib/rebuild_test.go
View file

@ -357,8 +357,8 @@ RegularPages: {{ range .Site.RegularPages }}{{ .RelPermalink }}|{{ end }}$
}
func TestRebuildRenameDirectoryWithBranchBundleFastRender(t *testing.T) {
recentlyVisited := types.NewEvictingStringQueue(10).Add("/a/b/c/")
b := TestRunning(t, rebuildFilesSimple, func(cfg *IntegrationTestConfig) { cfg.BuildCfg = BuildCfg{RecentlyVisited: recentlyVisited} })
recentlyVisited := types.NewEvictingQueue[string](10).Add("/a/b/c/")
b := TestRunning(t, rebuildFilesSimple, func(cfg *IntegrationTestConfig) { cfg.BuildCfg = BuildCfg{RecentlyTouched: recentlyVisited} })
b.RenameDir("content/mysection", "content/mysectionrenamed").Build()
b.AssertFileContent("public/mysectionrenamed/index.html", "My Section")
b.AssertFileContent("public/mysectionrenamed/mysectionbundle/index.html", "My Section Bundle")
@ -1181,6 +1181,49 @@ Content: {{ .Content }}
b.AssertFileContent("public/index.html", "Content: <p>Home</p>")
}
// Issue #13014.
func TestRebuildEditNotPermalinkableCustomOutputFormatTemplateInFastRenderMode(t *testing.T) {
t.Parallel()
files := `
-- hugo.toml --
baseURL = "https://example.com/docs/"
disableLiveReload = true
[internal]
fastRenderMode = true
disableKinds = ["taxonomy", "term", "sitemap", "robotsTXT", "404"]
[outputFormats]
[outputFormats.SearchIndex]
baseName = 'Search'
isPlainText = true
mediaType = 'text/plain'
noAlternative = true
permalinkable = false
[outputs]
home = ['HTML', 'SearchIndex']
-- content/_index.md --
---
title: "Home"
---
Home.
-- layouts/index.html --
Home.
-- layouts/_default/index.searchindex.txt --
Text. {{ .Title }}|{{ .RelPermalink }}|
`
b := TestRunning(t, files, TestOptInfo())
b.AssertFileContent("public/search.txt", "Text.")
b.EditFileReplaceAll("layouts/_default/index.searchindex.txt", "Text.", "Text Edited.").Build()
b.BuildPartial("/docs/search.txt")
b.AssertFileContent("public/search.txt", "Text Edited.")
}
func TestRebuildVariationsAssetsJSImport(t *testing.T) {
t.Parallel()
files := `

5
hugolib/site_render.go
View file

@ -20,6 +20,7 @@ import (
"strings"
"sync"
"github.com/bep/logg"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/hugolib/doctree"
@ -33,6 +34,8 @@ import (
type siteRenderContext struct {
cfg *BuildCfg
infol logg.LevelLogger
// languageIdx is the zero based index of the site.
languageIdx int
@ -86,7 +89,7 @@ func (s *Site) renderPages(ctx *siteRenderContext) error {
Tree: s.pageMap.treePages,
Handle: func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) {
if p, ok := n.(*pageState); ok {
if cfg.shouldRender(p) {
if cfg.shouldRender(ctx.infol, p) {
select {
case <-s.h.Done():
return true, nil