Implement configuration in a directory for modules

Fixes #8654

config/configLoader.go

@@ -14,9 +14,14 @@
 package config
 
 import (
+	"os"
 	"path/filepath"
 	"strings"
 
+	"github.com/pkg/errors"
+
+	"github.com/gohugoio/hugo/common/paths"
+
 	"github.com/gohugoio/hugo/common/maps"
 	"github.com/gohugoio/hugo/parser/metadecoders"
 	"github.com/spf13/afero"
@@ -84,6 +89,102 @@ func loadConfigFromFile(fs afero.Fs, filename string) (map[string]interface{}, e
 	return m, nil
 }
 
+func LoadConfigFromDir(sourceFs afero.Fs, configDir, environment string) (Provider, []string, error) {
+	defaultConfigDir := filepath.Join(configDir, "_default")
+	environmentConfigDir := filepath.Join(configDir, environment)
+	cfg := New()
+
+	var configDirs []string
+	// Merge from least to most specific.
+	for _, dir := range []string{defaultConfigDir, environmentConfigDir} {
+		if _, err := sourceFs.Stat(dir); err == nil {
+			configDirs = append(configDirs, dir)
+		}
+	}
+
+	if len(configDirs) == 0 {
+		return nil, nil, nil
+	}
+
+	// Keep track of these so we can watch them for changes.
+	var dirnames []string
+
+	for _, configDir := range configDirs {
+		err := afero.Walk(sourceFs, configDir, func(path string, fi os.FileInfo, err error) error {
+			if fi == nil || err != nil {
+				return nil
+			}
+
+			if fi.IsDir() {
+				dirnames = append(dirnames, path)
+				return nil
+			}
+
+			if !IsValidConfigFilename(path) {
+				return nil
+			}
+
+			name := paths.Filename(filepath.Base(path))
+
+			item, err := metadecoders.Default.UnmarshalFileToMap(sourceFs, path)
+			if err != nil {
+				// This will be used in error reporting, use the most specific value.
+				dirnames = []string{path}
+				return errors.Wrapf(err, "failed to unmarshl config for path %q", path)
+			}
+
+			var keyPath []string
+
+			if name != "config" {
+				// Can be params.jp, menus.en etc.
+				name, lang := paths.FileAndExtNoDelimiter(name)
+
+				keyPath = []string{name}
+
+				if lang != "" {
+					keyPath = []string{"languages", lang}
+					switch name {
+					case "menu", "menus":
+						keyPath = append(keyPath, "menus")
+					case "params":
+						keyPath = append(keyPath, "params")
+					}
+				}
+			}
+
+			root := item
+			if len(keyPath) > 0 {
+				root = make(map[string]interface{})
+				m := root
+				for i, key := range keyPath {
+					if i >= len(keyPath)-1 {
+						m[key] = item
+					} else {
+						nm := make(map[string]interface{})
+						m[key] = nm
+						m = nm
+					}
+				}
+			}
+
+			// Migrate menu => menus etc.
+			RenameKeys(root)
+
+			// Set will overwrite keys with the same name, recursively.
+			cfg.Set("", root)
+
+			return nil
+		})
+		if err != nil {
+			return nil, dirnames, err
+		}
+
+	}
+
+	return cfg, dirnames, nil
+
+}
+
 var keyAliases maps.KeyRenamer
 
 func init() {

hugolib/config.go

@@ -79,10 +79,16 @@ func LoadConfig(d ConfigSourceDescriptor, doWithConfig ...func(cfg config.Provid
 	}
 
 	if d.AbsConfigDir != "" {
-		dirnames, err := l.loadConfigFromConfigDir()
+		dcfg, dirnames, err := config.LoadConfigFromDir(l.Fs, d.AbsConfigDir, l.Environment)
 		if err == nil {
-			configFiles = append(configFiles, dirnames...)
+			if len(dirnames) > 0 {
+				l.cfg.Set("", dcfg.Get(""))
+				configFiles = append(configFiles, dirnames...)
+			}
 		} else if err != ErrNoConfigFile {
+			if len(dirnames) > 0 {
+				return nil, nil, l.wrapFileError(err, dirnames[0])
+			}
 			return nil, nil, err
 		}
 	}
@@ -381,9 +387,9 @@ func (l configLoader) collectModules(modConfig modules.Config, v1 config.Provide
 
 	hook := func(m *modules.ModulesConfig) error {
 		for _, tc := range m.ActiveModules {
-			if tc.ConfigFilename() != "" {
+			if len(tc.ConfigFilenames()) > 0 {
 				if tc.Watch() {
-					configFilenames = append(configFilenames, tc.ConfigFilename())
+					configFilenames = append(configFilenames, tc.ConfigFilenames()...)
 				}
 
 				// Merge from theme config into v1 based on configured
@@ -406,6 +412,7 @@ func (l configLoader) collectModules(modConfig modules.Config, v1 config.Provide
 		HookBeforeFinalize: hook,
 		WorkingDir:         workingDir,
 		ThemesDir:          themesDir,
+		Environment:        l.Environment,
 		CacheDir:           filecacheConfigs.CacheDirModules(),
 		ModuleConfig:       modConfig,
 		IgnoreVendor:       ignoreVendor,
@@ -468,106 +475,6 @@ func (l configLoader) loadConfig(configName string) (string, error) {
 	return filename, nil
 }
 
-func (l configLoader) loadConfigFromConfigDir() ([]string, error) {
-	sourceFs := l.Fs
-	configDir := l.AbsConfigDir
-
-	if _, err := sourceFs.Stat(configDir); err != nil {
-		// Config dir does not exist.
-		return nil, nil
-	}
-
-	defaultConfigDir := filepath.Join(configDir, "_default")
-	environmentConfigDir := filepath.Join(configDir, l.Environment)
-
-	var configDirs []string
-	// Merge from least to most specific.
-	for _, dir := range []string{defaultConfigDir, environmentConfigDir} {
-		if _, err := sourceFs.Stat(dir); err == nil {
-			configDirs = append(configDirs, dir)
-		}
-	}
-
-	if len(configDirs) == 0 {
-		return nil, nil
-	}
-
-	// Keep track of these so we can watch them for changes.
-	var dirnames []string
-
-	for _, configDir := range configDirs {
-		err := afero.Walk(sourceFs, configDir, func(path string, fi os.FileInfo, err error) error {
-			if fi == nil || err != nil {
-				return nil
-			}
-
-			if fi.IsDir() {
-				dirnames = append(dirnames, path)
-				return nil
-			}
-
-			if !config.IsValidConfigFilename(path) {
-				return nil
-			}
-
-			name := cpaths.Filename(filepath.Base(path))
-
-			item, err := metadecoders.Default.UnmarshalFileToMap(sourceFs, path)
-			if err != nil {
-				return l.wrapFileError(err, path)
-			}
-
-			var keyPath []string
-
-			if name != "config" {
-				// Can be params.jp, menus.en etc.
-				name, lang := cpaths.FileAndExtNoDelimiter(name)
-
-				keyPath = []string{name}
-
-				if lang != "" {
-					keyPath = []string{"languages", lang}
-					switch name {
-					case "menu", "menus":
-						keyPath = append(keyPath, "menus")
-					case "params":
-						keyPath = append(keyPath, "params")
-					}
-				}
-			}
-
-			root := item
-			if len(keyPath) > 0 {
-				root = make(map[string]interface{})
-				m := root
-				for i, key := range keyPath {
-					if i >= len(keyPath)-1 {
-						m[key] = item
-					} else {
-						nm := make(map[string]interface{})
-						m[key] = nm
-						m = nm
-					}
-				}
-			}
-
-			// Migrate menu => menus etc.
-			config.RenameKeys(root)
-
-			// Set will overwrite keys with the same name, recursively.
-			l.cfg.Set("", root)
-
-			return nil
-		})
-		if err != nil {
-			return nil, err
-		}
-
-	}
-
-	return dirnames, nil
-}
-
 func (l configLoader) loadLanguageSettings(oldLangs langs.Languages) error {
 	_, err := langs.LoadLanguageSettings(l.cfg, oldLangs)
 	return err

hugolib/config_test.go

@@ -318,6 +318,59 @@ name = "menu-theme"
 
 }
 
+func TestLoadConfigFromThemeDir(t *testing.T) {
+	t.Parallel()
+
+	mainConfig := `
+theme = "test-theme"
+
+[params]
+m1 = "mv1"	
+`
+
+	themeConfig := `
+[params]
+t1 = "tv1"	
+t2 = "tv2"
+`
+
+	themeConfigDir := filepath.Join("themes", "test-theme", "config")
+	themeConfigDirDefault := filepath.Join(themeConfigDir, "_default")
+	themeConfigDirProduction := filepath.Join(themeConfigDir, "production")
+
+	projectConfigDir := "config"
+
+	b := newTestSitesBuilder(t)
+	b.WithConfigFile("toml", mainConfig).WithThemeConfigFile("toml", themeConfig)
+	b.Assert(b.Fs.Source.MkdirAll(themeConfigDirDefault, 0777), qt.IsNil)
+	b.Assert(b.Fs.Source.MkdirAll(themeConfigDirProduction, 0777), qt.IsNil)
+	b.Assert(b.Fs.Source.MkdirAll(projectConfigDir, 0777), qt.IsNil)
+
+	b.WithSourceFile(filepath.Join(projectConfigDir, "config.toml"), `[params]
+m2 = "mv2"
+`)
+	b.WithSourceFile(filepath.Join(themeConfigDirDefault, "config.toml"), `[params]
+t2 = "tv2d"
+t3 = "tv3d"
+`)
+
+	b.WithSourceFile(filepath.Join(themeConfigDirProduction, "config.toml"), `[params]
+t3 = "tv3p"
+`)
+
+	b.Build(BuildCfg{})
+
+	got := b.Cfg.Get("params").(maps.Params)
+
+	b.Assert(got, qt.DeepEquals, maps.Params{
+		"t3": "tv3p",
+		"m1": "mv1",
+		"t1": "tv1",
+		"t2": "tv2d",
+	})
+
+}
+
 func TestPrivacyConfig(t *testing.T) {
 	t.Parallel()
 

modules/client.go

@@ -653,6 +653,9 @@ type ClientConfig struct {
 	// Absolute path to the project's themes dir.
 	ThemesDir string
 
+	// Eg. "production"
+	Environment string
+
 	CacheDir     string // Module cache
 	ModuleConfig Config
 }

modules/collect.go

@@ -396,17 +396,16 @@ func (c *collector) applyMounts(moduleImport Import, mod *moduleAdapter) error {
 func (c *collector) applyThemeConfig(tc *moduleAdapter) error {
 	var (
 		configFilename string
-		cfg            config.Provider
 		themeCfg       map[string]interface{}
-		hasConfig      bool
+		hasConfigFile  bool
 		err            error
 	)
 
 	// Viper supports more, but this is the sub-set supported by Hugo.
 	for _, configFormats := range config.ValidConfigFileExtensions {
 		configFilename = filepath.Join(tc.Dir(), "config."+configFormats)
-		hasConfig, _ = afero.Exists(c.fs, configFilename)
+		hasConfigFile, _ = afero.Exists(c.fs, configFilename)
-		if hasConfig {
+		if hasConfigFile {
 			break
 		}
 	}
@@ -428,20 +427,38 @@ func (c *collector) applyThemeConfig(tc *moduleAdapter) error {
 		}
 	}
 
-	if hasConfig {
+	if hasConfigFile {
 		if configFilename != "" {
 			var err error
-			cfg, err = config.FromFile(c.fs, configFilename)
+			tc.cfg, err = config.FromFile(c.fs, configFilename)
 			if err != nil {
 				return errors.Wrapf(err, "failed to read module config for %q in %q", tc.Path(), configFilename)
 			}
 		}
 
-		tc.configFilename = configFilename
+		tc.configFilenames = append(tc.configFilenames, configFilename)
-		tc.cfg = cfg
+
 	}
 
-	config, err := decodeConfig(cfg, c.moduleConfig.replacementsMap)
+	// Also check for a config dir, which we overlay on top of the file configuration.
+	configDir := filepath.Join(tc.Dir(), "config")
+	dcfg, dirnames, err := config.LoadConfigFromDir(c.fs, configDir, c.ccfg.Environment)
+	if err != nil {
+		return err
+	}
+
+	if len(dirnames) > 0 {
+		tc.configFilenames = append(tc.configFilenames, dirnames...)
+
+		if hasConfigFile {
+			// Set will overwrite existing keys.
+			tc.cfg.Set("", dcfg.Get(""))
+		} else {
+			tc.cfg = dcfg
+		}
+	}
+
+	config, err := decodeConfig(tc.cfg, c.moduleConfig.replacementsMap)
 	if err != nil {
 		return err
 	}

modules/module.go

@@ -30,10 +30,10 @@ type Module interface {
 	// The decoded module config and mounts.
 	Config() Config
 
-	// Optional configuration filename (e.g. "/themes/mytheme/config.json").
+	// Optional configuration filenames (e.g. "/themes/mytheme/config.json").
 	// This will be added to the special configuration watch list when in
 	// server mode.
-	ConfigFilename() string
+	ConfigFilenames() []string
 
 	// Directory holding files for this module.
 	Dir() string
@@ -82,9 +82,9 @@ type moduleAdapter struct {
 
 	mounts []Mount
 
-	configFilename string
+	configFilenames []string
-	cfg            config.Provider
+	cfg             config.Provider
-	config         Config
+	config          Config
 
 	// Set if a Go module.
 	gomod *goModule
@@ -98,8 +98,8 @@ func (m *moduleAdapter) Config() Config {
 	return m.config
 }
 
-func (m *moduleAdapter) ConfigFilename() string {
+func (m *moduleAdapter) ConfigFilenames() []string {
-	return m.configFilename
+	return m.configFilenames
 }
 
 func (m *moduleAdapter) Dir() string {