Add build.UseResourceCacheWhen

Fixes #6993

config/commonConfig.go

@@ -14,10 +14,56 @@
 package config
 
 import (
+	"strings"
+
+	"github.com/gohugoio/hugo/common/herrors"
+	"github.com/mitchellh/mapstructure"
 	"github.com/spf13/cast"
 	jww "github.com/spf13/jwalterweatherman"
 )
 
+var DefaultBuild = Build{
+	UseResourceCacheWhen: "fallback",
+}
+
+// Build holds some build related condfiguration.
+type Build struct {
+	UseResourceCacheWhen string // never, fallback, always. Default is fallback
+}
+
+func (b Build) UseResourceCache(err error) bool {
+	if b.UseResourceCacheWhen == "never" {
+		return false
+	}
+
+	if b.UseResourceCacheWhen == "fallback" {
+		return err == herrors.ErrFeatureNotAvailable
+	}
+
+	return true
+}
+
+func DecodeBuild(cfg Provider) Build {
+	m := cfg.GetStringMap("build")
+	b := DefaultBuild
+	if m == nil {
+		return b
+	}
+
+	err := mapstructure.WeakDecode(m, &b)
+	if err != nil {
+		return DefaultBuild
+	}
+
+	b.UseResourceCacheWhen = strings.ToLower(b.UseResourceCacheWhen)
+	when := b.UseResourceCacheWhen
+	if when != "never" && when != "always" && when != "fallback" {
+		b.UseResourceCacheWhen = "fallback"
+	}
+
+	return b
+}
+
 // Sitemap configures the sitemap to be generated.
 type Sitemap struct {
 	ChangeFreq string

config/commonConfig_test.go

@@ -0,0 +1,60 @@
+// Copyright 2020 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/gohugoio/hugo/common/herrors"
+
+	qt "github.com/frankban/quicktest"
+
+	"github.com/spf13/viper"
+)
+
+func TestBuild(t *testing.T) {
+	c := qt.New(t)
+
+	v := viper.New()
+	v.Set("build", map[string]interface{}{
+		"useResourceCacheWhen": "always",
+	})
+
+	b := DecodeBuild(v)
+
+	c.Assert(b.UseResourceCacheWhen, qt.Equals, "always")
+
+	v.Set("build", map[string]interface{}{
+		"useResourceCacheWhen": "foo",
+	})
+
+	b = DecodeBuild(v)
+
+	c.Assert(b.UseResourceCacheWhen, qt.Equals, "fallback")
+
+	c.Assert(b.UseResourceCache(herrors.ErrFeatureNotAvailable), qt.Equals, true)
+	c.Assert(b.UseResourceCache(errors.New("err")), qt.Equals, false)
+
+	b.UseResourceCacheWhen = "always"
+	c.Assert(b.UseResourceCache(herrors.ErrFeatureNotAvailable), qt.Equals, true)
+	c.Assert(b.UseResourceCache(errors.New("err")), qt.Equals, true)
+	c.Assert(b.UseResourceCache(nil), qt.Equals, true)
+
+	b.UseResourceCacheWhen = "never"
+	c.Assert(b.UseResourceCache(herrors.ErrFeatureNotAvailable), qt.Equals, false)
+	c.Assert(b.UseResourceCache(errors.New("err")), qt.Equals, false)
+	c.Assert(b.UseResourceCache(nil), qt.Equals, false)
+
+}

docs/content/en/getting-started/configuration.md

@@ -87,6 +87,9 @@ baseURL
 blackfriday
 : See [Configure Blackfriday](/getting-started/configuration-markup#blackfriday)
 
+build
+: See [Configure Build](#configure-build)
+
 buildDrafts (false)
 : Include drafts when building.
 
@@ -288,6 +291,21 @@ enableemoji: true
 ```
 {{% /note %}}
 
+## Configure Build
+
+{{< new-in "0.66.0" >}}
+
+The `build` configuration section contains global build-realated configuration options.
+
+{{< code-toggle file="config">}}
+[build]
+useResourceCacheWhen="fallback"
+{{< /code-toggle >}}
+
+
+useResourceCacheWhen
+: When to use the cached resources in `/resources/_gen` for PostCSS and ToCSS. Valid values are `never`, `always` and `fallback`. The last value means that the cache will be tried if PostCSS/extended version is not available.
+
 ## Configure Title Case
 
 Set `titleCaseStyle` to specify the title style used by the [title](/functions/title/) template function and the automatic section titles in Hugo. It defaults to [AP Stylebook](https://www.apstylebook.com/) for title casing, but you can also set it to `Chicago` or `Go` (every word starts with a capital letter).

hugolib/resource_chain_test.go

@@ -753,20 +753,17 @@ h1 {
 	c.Assert(err, qt.IsNil)
 	defer clean()
 
-	v := viper.New()
+	newTestBuilder := func(v *viper.Viper) *sitesBuilder {
-	v.Set("workingDir", workDir)
+		v.Set("workingDir", workDir)
-	v.Set("disableKinds", []string{"taxonomyTerm", "taxonomy", "page"})
+		v.Set("disableKinds", []string{"taxonomyTerm", "taxonomy", "page"})
-	b := newTestSitesBuilder(t).WithLogger(loggers.NewWarningLogger())
+		b := newTestSitesBuilder(t).WithLogger(loggers.NewWarningLogger())
-	// Need to use OS fs for this.
+		// Need to use OS fs for this.
-	b.Fs = hugofs.NewDefault(v)
+		b.Fs = hugofs.NewDefault(v)
-	b.WithWorkingDir(workDir)
+		b.WithWorkingDir(workDir)
-	b.WithViper(v)
+		b.WithViper(v)
 
-	cssDir := filepath.Join(workDir, "assets", "css", "components")
+		b.WithContent("p1.md", "")
-	b.Assert(os.MkdirAll(cssDir, 0777), qt.IsNil)
+		b.WithTemplates("index.html", `
-
-	b.WithContent("p1.md", "")
-	b.WithTemplates("index.html", `
 {{ $options := dict "inlineImports" true }}
 {{ $styles := resources.Get "css/styles.css" | resources.PostCSS $options }}
 Styles RelPermalink: {{ $styles.RelPermalink }}
@@ -774,6 +771,15 @@ Styles RelPermalink: {{ $styles.RelPermalink }}
 Styles Content: Len: {{ len $styles.Content }}|
 
 `)
+
+		return b
+	}
+
+	b := newTestBuilder(viper.New())
+
+	cssDir := filepath.Join(workDir, "assets", "css", "components")
+	b.Assert(os.MkdirAll(cssDir, 0777), qt.IsNil)
+
 	b.WithSourceFile("assets/css/styles.css", tailwindCss)
 	b.WithSourceFile("assets/css/components/all.css", `
 @import "a.css";
@@ -810,9 +816,52 @@ Styles RelPermalink: /css/styles.css
 Styles Content: Len: 770878|
 `)
 
-	content := b.FileContent("public/css/styles.css")
+	assertCss := func(b *sitesBuilder) {
+		content := b.FileContent("public/css/styles.css")
 
-	b.Assert(strings.Contains(content, "class-in-a"), qt.Equals, true)
+		b.Assert(strings.Contains(content, "class-in-a"), qt.Equals, true)
-	b.Assert(strings.Contains(content, "class-in-b"), qt.Equals, true)
+		b.Assert(strings.Contains(content, "class-in-b"), qt.Equals, true)
+
+	}
+
+	assertCss(b)
+
+	build := func(s string, shouldFail bool) {
+		b.Assert(os.RemoveAll(filepath.Join(workDir, "public")), qt.IsNil)
+
+		v := viper.New()
+		v.Set("build", map[string]interface{}{
+			"useResourceCacheWhen": s,
+		})
+
+		b = newTestBuilder(v)
+
+		b.Assert(os.RemoveAll(filepath.Join(workDir, "public")), qt.IsNil)
+		b.Assert(os.RemoveAll(filepath.Join(workDir, "node_modules")), qt.IsNil)
+
+		if shouldFail {
+			b.BuildFail(BuildCfg{})
+		} else {
+			b.Build(BuildCfg{})
+			assertCss(b)
+		}
+	}
+
+	build("always", false)
+	build("fallback", false)
+
+	// Remove PostCSS
+	b.Assert(os.RemoveAll(filepath.Join(workDir, "node_modules")), qt.IsNil)
+
+	build("always", false)
+	build("fallback", false)
+	build("never", true)
+
+	// Remove cache
+	b.Assert(os.RemoveAll(filepath.Join(workDir, "resources")), qt.IsNil)
+
+	build("always", true)
+	build("fallback", true)
+	build("never", true)
 
 }

hugolib/site_render.go

@@ -59,7 +59,6 @@ func (s siteRenderContext) renderSingletonPages() bool {
 // renderPages renders pages each corresponding to a markdown file.
 // TODO(bep np doc
 func (s *Site) renderPages(ctx *siteRenderContext) error {
-
 	numWorkers := config.GetNumWorkerMultiplier()
 
 	results := make(chan error)

langs/language.go

@@ -36,6 +36,7 @@ var globalOnlySettings = map[string]bool{
 	strings.ToLower("multilingual"):                   true,
 	strings.ToLower("assetDir"):                       true,
 	strings.ToLower("resourceDir"):                    true,
+	strings.ToLower("build"):                          true,
 }
 
 // Language manages specific-language configuration.

resources/resource_spec.go

@@ -22,6 +22,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/gohugoio/hugo/config"
+
 	"github.com/gohugoio/hugo/hugofs"
 
 	"github.com/gohugoio/hugo/helpers"
@@ -69,6 +71,7 @@ func NewSpec(
 		MediaTypes:    mimeTypes,
 		OutputFormats: outputFormats,
 		Permalinks:    permalinks,
+		BuildConfig:   config.DecodeBuild(s.Cfg),
 		FileCaches:    fileCaches,
 		imageCache: newImageCache(
 			fileCaches.ImageCache(),
@@ -92,7 +95,8 @@ type Spec struct {
 
 	TextTemplates tpl.TemplateParseFinder
 
-	Permalinks page.PermalinkExpander
+	Permalinks  page.PermalinkExpander
+	BuildConfig config.Build
 
 	// Holds default filter settings etc.
 	imaging *images.ImageProcessor

resources/transform.go

@@ -21,12 +21,13 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/pkg/errors"
+
 	"github.com/gohugoio/hugo/resources/images/exif"
 	"github.com/spf13/afero"
 
 	bp "github.com/gohugoio/hugo/bufferpool"
 
-	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/gohugoio/hugo/common/hugio"
 	"github.com/gohugoio/hugo/common/maps"
 	"github.com/gohugoio/hugo/helpers"
@@ -369,8 +370,9 @@ func (r *resourceAdapter) transform(publish, setContent bool) error {
 			tctx.InMediaType = tctx.OutMediaType
 		}
 
+		mayBeCachedOnDisk := transformationsToCacheOnDisk[tr.Key().Name]
 		if !writeToFileCache {
-			writeToFileCache = transformationsToCacheOnDisk[tr.Key().Name]
+			writeToFileCache = mayBeCachedOnDisk
 		}
 
 		if i > 0 {
@@ -390,29 +392,44 @@ func (r *resourceAdapter) transform(publish, setContent bool) error {
 			}
 		}
 
-		if err = tr.Transform(tctx); err != nil {
+		notAvailableErr := func(err error) error {
-			if writeToFileCache && err == herrors.ErrFeatureNotAvailable {
+			errMsg := err.Error()
+			if tr.Key().Name == "postcss" {
 				// This transformation is not available in this
-				// Hugo installation (scss not compiled in, PostCSS not available etc.)
+				// Most likely because PostCSS is not installed.
-				// If a prepared bundle for this transformation chain is available, use that.
+				errMsg += ". Check your PostCSS installation; install with \"npm install postcss-cli\". See https://gohugo.io/hugo-pipes/postcss/"
-				f := r.target.tryTransformedFileCache(key, updates)
+			} else if tr.Key().Name == "tocss" {
-				if f == nil {
+				errMsg += ". Check your Hugo installation; you need the extended version to build SCSS/SASS."
-					errMsg := err.Error()
-					if tr.Key().Name == "postcss" {
-						errMsg = "PostCSS not found; install with \"npm install postcss-cli\". See https://gohugo.io/hugo-pipes/postcss/"
-					}
-					return fmt.Errorf("%s: failed to transform %q (%s): %s", strings.ToUpper(tr.Key().Name), tctx.InPath, tctx.InMediaType.Type(), errMsg)
-				}
-				transformedContentr = f
-				updates.sourceFs = cache.fileCache.Fs
-				defer f.Close()
-
-				// The reader above is all we need.
-				break
 			}
+			return fmt.Errorf("%s: failed to transform %q (%s): %s", strings.ToUpper(tr.Key().Name), tctx.InPath, tctx.InMediaType.Type(), errMsg)
 
-			// Abort.
+		}
-			return err
+
+		var tryFileCache bool
+
+		if mayBeCachedOnDisk && r.spec.BuildConfig.UseResourceCache(nil) {
+			tryFileCache = true
+		} else {
+			err = tr.Transform(tctx)
+			if mayBeCachedOnDisk {
+				tryFileCache = r.spec.BuildConfig.UseResourceCache(err)
+			}
+			if err != nil && !tryFileCache {
+				return notAvailableErr(err)
+			}
+		}
+
+		if tryFileCache {
+			f := r.target.tryTransformedFileCache(key, updates)
+			if f == nil {
+				return notAvailableErr(errors.Errorf("resource %q not found in file cache", key))
+			}
+			transformedContentr = f
+			updates.sourceFs = cache.fileCache.Fs
+			defer f.Close()
+
+			// The reader above is all we need.
+			break
 		}
 
 		if tctx.OutPath != "" {