From 17af79a03e249a731cf5634ffea23ca00774333d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Wed, 22 Jan 2020 11:57:23 +0100 Subject: [PATCH] Fix 0.62.1 server rebuild slowdown regression Fixes #6784 --- hugofs/fileinfo.go | 28 ++----- hugofs/rootmapping_fs.go | 6 +- hugofs/rootmapping_fs_test.go | 2 +- hugolib/content_render_hooks_test.go | 24 +++++- hugolib/filesystems/basefs.go | 13 +-- hugolib/hugo_modules_test.go | 120 +++++++++++++++++++++++++++ hugolib/hugo_sites.go | 16 ++++ hugolib/hugo_sites_build.go | 2 + hugolib/page__per_output.go | 2 + hugolib/testhelpers_test.go | 9 ++ 10 files changed, 186 insertions(+), 36 deletions(-) diff --git a/hugofs/fileinfo.go b/hugofs/fileinfo.go index 893436df1..c8a71bf21 100644 --- a/hugofs/fileinfo.go +++ b/hugofs/fileinfo.go @@ -34,9 +34,9 @@ import ( ) const ( - metaKeyFilename = "filename" - metaKeyPathFile = "pathFile" // Path of filename relative to a root. - metaKeyIsFileMount = "isFileMount" // Whether the source mount was a file. + metaKeyFilename = "filename" + + metaKeyBaseDir = "baseDir" // Abs base directory of source file. metaKeyMountRoot = "mountRoot" metaKeyOriginalFilename = "originalFilename" metaKeyName = "name" @@ -116,29 +116,19 @@ func (f FileMeta) Path() string { return f.stringV(metaKeyPath) } -// PathFile returns the relative file path for the file source. This -// will in most cases be the same as Path. +// PathFile returns the relative file path for the file source. func (f FileMeta) PathFile() string { - pf := f.stringV(metaKeyPathFile) - if f.isFileMount() { - return pf + base := f.stringV(metaKeyBaseDir) + if base == "" { + return "" } - mountRoot := f.mountRoot() - if mountRoot == pf { - return f.Path() - } - - return pf + (strings.TrimPrefix(f.Path(), mountRoot)) + return strings.TrimPrefix(strings.TrimPrefix(f.Filename(), base), filepathSeparator) } -func (f FileMeta) mountRoot() string { +func (f FileMeta) MountRoot() string { return f.stringV(metaKeyMountRoot) } -func (f FileMeta) isFileMount() bool { - return f.GetBool(metaKeyIsFileMount) -} - func (f FileMeta) Weight() int { return f.GetInt(metaKeyWeight) } diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go index dd60452fc..2196be8e0 100644 --- a/hugofs/rootmapping_fs.go +++ b/hugofs/rootmapping_fs.go @@ -57,12 +57,8 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { // Extract "blog" from "content/blog" rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, fromBase), filepathSeparator) if rm.Meta != nil { - rm.Meta[metaKeyIsFileMount] = !fi.IsDir() + rm.Meta[metaKeyBaseDir] = rm.ToBasedir rm.Meta[metaKeyMountRoot] = rm.path - if rm.ToBasedir != "" { - pathFile := strings.TrimPrefix(strings.TrimPrefix(rm.To, rm.ToBasedir), filepathSeparator) - rm.Meta[metaKeyPathFile] = pathFile - } } meta := copyFileMeta(rm.Meta) diff --git a/hugofs/rootmapping_fs_test.go b/hugofs/rootmapping_fs_test.go index 7d685c77e..f7637a61f 100644 --- a/hugofs/rootmapping_fs_test.go +++ b/hugofs/rootmapping_fs_test.go @@ -271,7 +271,7 @@ func TestRootMappingFsMount(t *testing.T) { c.Assert(singles, qt.HasLen, 2) for i, lang := range []string{"no", "sv"} { fi := singles[i].(FileMetaInfo) - c.Assert(fi.Meta().PathFile(), qt.Equals, lang+".txt") + c.Assert(fi.Meta().PathFile(), qt.Equals, filepath.FromSlash("themes/a/singlefiles/"+lang+".txt")) c.Assert(fi.Meta().Lang(), qt.Equals, lang) c.Assert(fi.Name(), qt.Equals, "p1.md") } diff --git a/hugolib/content_render_hooks_test.go b/hugolib/content_render_hooks_test.go index 8aba1dd8c..5290ebcbd 100644 --- a/hugolib/content_render_hooks_test.go +++ b/hugolib/content_render_hooks_test.go @@ -14,7 +14,10 @@ package hugolib import ( + "fmt" "testing" + + qt "github.com/frankban/quicktest" ) func TestRenderHooks(t *testing.T) { @@ -118,7 +121,20 @@ title: With RenderString {{< myshortcode5 >}}Inner Link: [Inner Link](https://www.gohugo.io "Hugo's Homepage"){{< /myshortcode5 >}} `) - b.Build(BuildCfg{}) + + for i := 1; i <= 30; i++ { + // Add some content with no shortcodes or links, i.e no templates needed. + b.WithContent(fmt.Sprintf("blog/notempl%d.md", i), `--- +title: No Template +--- + +## Content +`) + } + counters := &testCounters{} + b.Build(BuildCfg{testCounters: counters}) + b.Assert(int(counters.contentRenderCounter), qt.Equals, 50) + b.AssertFileContent("public/blog/p1/index.html", `

Cool Page|https://www.google.com|Title: Google's Homepage|Text: First Link|END

Text: Second @@ -149,7 +165,11 @@ SHORT3| "layouts/shortcodes/myshortcode3.html", `SHORT3_EDITED|`, ) - b.Build(BuildCfg{}) + counters = &testCounters{} + b.Build(BuildCfg{testCounters: counters}) + // Make sure that only content using the changed templates are re-rendered. + b.Assert(int(counters.contentRenderCounter), qt.Equals, 7) + b.AssertFileContent("public/customview/p1/index.html", `.Render: myrender: Custom View|P4: PARTIAL4_EDITED`) b.AssertFileContent("public/blog/p1/index.html", `

EDITED: https://www.google.com|

`, "SHORT3_EDITED|") b.AssertFileContent("public/blog/p2/index.html", `PARTIAL1_EDITED`) diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index cf9ff3c38..5cede88d0 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -295,21 +295,16 @@ func (d *SourceFilesystem) Contains(filename string) bool { return false } -// Path returns the relative path to the given filename if it is a member of +// Path returns the mount relative path to the given filename if it is a member of // of the current filesystem, an empty string if not. func (d *SourceFilesystem) Path(filename string) string { for _, dir := range d.Dirs { meta := dir.Meta() - if !dir.IsDir() { - if filename == meta.Filename() { - return meta.PathFile() - } - continue - } - if strings.HasPrefix(filename, meta.Filename()) { p := strings.TrimPrefix(strings.TrimPrefix(filename, meta.Filename()), filePathSeparator) - p = path.Join(meta.PathFile(), p) + if mountRoot := meta.MountRoot(); mountRoot != "" { + return filepath.Join(mountRoot, p) + } return p } } diff --git a/hugolib/hugo_modules_test.go b/hugolib/hugo_modules_test.go index 5c2b46b30..4b71a54c8 100644 --- a/hugolib/hugo_modules_test.go +++ b/hugolib/hugo_modules_test.go @@ -668,6 +668,126 @@ Readme Edit } +func TestMountsPaths(t *testing.T) { + c := qt.New(t) + + type test struct { + b *sitesBuilder + clean func() + workingDir string + } + + prepare := func(c *qt.C, mounts string) test { + workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-mounts-paths") + c.Assert(err, qt.IsNil) + + configTemplate := ` +baseURL = "https://example.com" +title = "My Modular Site" +workingDir = %q + +%s + +` + config := fmt.Sprintf(configTemplate, workingDir, mounts) + config = strings.Replace(config, "WORKING_DIR", workingDir, -1) + + b := newTestSitesBuilder(c).Running() + + b.Fs = hugofs.NewDefault(viper.New()) + + os.MkdirAll(filepath.Join(workingDir, "content", "blog"), 0777) + + b.WithWorkingDir(workingDir).WithConfigFile("toml", config) + + return test{ + b: b, + clean: clean, + workingDir: workingDir, + } + + } + + c.Run("Default", func(c *qt.C) { + mounts := `` + + test := prepare(c, mounts) + b := test.b + defer test.clean() + + b.WithContent("blog/p1.md", `--- +title: P1 +---`) + + b.Build(BuildCfg{}) + + p := b.GetPage("blog/p1.md") + f := p.File().FileInfo().Meta() + b.Assert(filepath.ToSlash(f.Path()), qt.Equals, "blog/p1.md") + b.Assert(filepath.ToSlash(f.PathFile()), qt.Equals, "content/blog/p1.md") + + b.Assert(b.H.BaseFs.Layouts.Path(filepath.Join(test.workingDir, "layouts", "_default", "single.html")), qt.Equals, filepath.FromSlash("_default/single.html")) + + }) + + c.Run("Mounts", func(c *qt.C) { + absDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-mounts-paths-abs") + c.Assert(err, qt.IsNil) + defer clean() + + mounts := `[module] + [[module.mounts]] + source = "README.md" + target = "content/_index.md" + [[module.mounts]] + source = "mycontent" + target = "content/blog" + [[module.mounts]] + source = "subdir/mypartials" + target = "layouts/partials" + [[module.mounts]] + source = %q + target = "layouts/shortcodes" +` + mounts = fmt.Sprintf(mounts, filepath.Join(absDir, "/abs/myshortcodes")) + + test := prepare(c, mounts) + b := test.b + defer test.clean() + + subContentDir := filepath.Join(test.workingDir, "mycontent", "sub") + os.MkdirAll(subContentDir, 0777) + myPartialsDir := filepath.Join(test.workingDir, "subdir", "mypartials") + os.MkdirAll(myPartialsDir, 0777) + + absShortcodesDir := filepath.Join(absDir, "abs", "myshortcodes") + os.MkdirAll(absShortcodesDir, 0777) + + b.WithSourceFile("README.md", "---\ntitle: Readme\n---") + b.WithSourceFile("mycontent/sub/p1.md", "---\ntitle: P1\n---") + + b.WithSourceFile(filepath.Join(absShortcodesDir, "myshort.html"), "MYSHORT") + b.WithSourceFile(filepath.Join(myPartialsDir, "mypartial.html"), "MYPARTIAL") + + b.Build(BuildCfg{}) + + p1_1 := b.GetPage("/blog/sub/p1.md") + p1_2 := b.GetPage("/mycontent/sub/p1.md") + b.Assert(p1_1, qt.Not(qt.IsNil)) + b.Assert(p1_2, qt.Equals, p1_1) + + f := p1_1.File().FileInfo().Meta() + b.Assert(filepath.ToSlash(f.Path()), qt.Equals, "blog/sub/p1.md") + b.Assert(filepath.ToSlash(f.PathFile()), qt.Equals, "mycontent/sub/p1.md") + b.Assert(b.H.BaseFs.Layouts.Path(filepath.Join(myPartialsDir, "mypartial.html")), qt.Equals, filepath.FromSlash("partials/mypartial.html")) + b.Assert(b.H.BaseFs.Layouts.Path(filepath.Join(absShortcodesDir, "myshort.html")), qt.Equals, filepath.FromSlash("shortcodes/myshort.html")) + b.Assert(b.H.BaseFs.Content.Path(filepath.Join(subContentDir, "p1.md")), qt.Equals, filepath.FromSlash("blog/sub/p1.md")) + b.Assert(b.H.BaseFs.Content.Path(filepath.Join(test.workingDir, "README.md")), qt.Equals, filepath.FromSlash("_index.md")) + + }) + +} + // https://github.com/gohugoio/hugo/issues/6299 func TestSiteWithGoModButNoModules(t *testing.T) { t.Parallel() diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index a0c62f01e..50694fbba 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -19,6 +19,7 @@ import ( "sort" "strings" "sync" + "sync/atomic" "github.com/gohugoio/hugo/identity" @@ -82,6 +83,19 @@ type HugoSites struct { init *hugoSitesInit *fatalErrorHandler + *testCounters +} + +// Only used in tests. +type testCounters struct { + contentRenderCounter uint64 +} + +func (h *testCounters) IncrContentRender() { + if h == nil { + return + } + atomic.AddUint64(&h.contentRenderCounter, 1) } type fatalErrorHandler struct { @@ -579,6 +593,8 @@ type BuildCfg struct { // Recently visited URLs. This is used for partial re-rendering. RecentlyVisited map[string]bool + + testCounters *testCounters } // shouldRender is used in the Fast Render Mode to determine if we need to re-render diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index 901941bda..cf7b14311 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -66,6 +66,8 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { h.Metrics.Reset() } + h.testCounters = config.testCounters + // Need a pointer as this may be modified. conf := &config diff --git a/hugolib/page__per_output.go b/hugolib/page__per_output.go index 59440c7cb..330b0d75d 100644 --- a/hugolib/page__per_output.go +++ b/hugolib/page__per_output.go @@ -80,6 +80,8 @@ func newPageContentOutput(p *pageState, po *pageOutput) (*pageContentOutput, err } initContent := func() (err error) { + p.s.h.IncrContentRender() + if p.cmap == nil { // Nothing to do. return nil diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index 376c0899c..7a87245a6 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -224,6 +224,9 @@ func (s *sitesBuilder) WithSourceFile(filenameContent ...string) *sitesBuilder { func (s *sitesBuilder) absFilename(filename string) string { filename = filepath.FromSlash(filename) + if filepath.IsAbs(filename) { + return filename + } if s.workingDir != "" && !strings.HasPrefix(filename, s.workingDir) { filename = filepath.Join(s.workingDir, filename) } @@ -736,6 +739,12 @@ func (s *sitesBuilder) CheckExists(filename string) bool { return destinationExists(s.Fs, filepath.Clean(filename)) } +func (s *sitesBuilder) GetPage(ref string) page.Page { + p, err := s.H.Sites[0].getPageNew(nil, ref) + s.Assert(err, qt.IsNil) + return p +} + func newTestHelper(cfg config.Provider, fs *hugofs.Fs, t testing.TB) testHelper { return testHelper{ Cfg: cfg,