Skip to content
Permalink
Browse files

Add HUGO_NUMWORKERMULTIPLIER

And use that to calculate number of workers, if set, else fall back to number of logical CPUs.

Also tweak the relevant related settings to match the new setup.

Also remove the setting of `runtime.GOMAXPROCS` as this has been the default behaviour since Go 1.5.

Fixes #5814
  • Loading branch information...
bep committed Apr 5, 2019
1 parent 415ca96 commit 87b16abd93ff60acd245776d5b0d914fd580c259
@@ -0,0 +1,33 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
"os"
"runtime"
"strconv"
)

// GetNumWorkerMultiplier returns the base value used to calculate the number
// of workers to use for Hugo's parallel execution.
// It returns the value in HUGO_NUMWORKERMULTIPLIER OS env variable if set to a
// positive integer, else the number of logical CPUs.
func GetNumWorkerMultiplier() int {
if gmp := os.Getenv("HUGO_NUMWORKERMULTIPLIER"); gmp != "" {
if p, err := strconv.Atoi(gmp); err == nil && p > 0 {
return p
}
}
return runtime.NumCPU()
}
@@ -297,6 +297,11 @@ enableemoji: true
```
{{% /note %}}

## Configuration Environment Variables

HUGO_NUMWORKERMULTIPLIER
: Can be set to increase or reduce the number of workers used in parallel processing in Hugo. If not set, the number of logical CPUs will be used.

## Configuration Lookup Order

Similar to the template [lookup order][], Hugo has a default set of rules for searching for a configuration file in the root of your website's source directory as a default behavior:
@@ -18,7 +18,8 @@ import (
"fmt"
"math"
"path/filepath"
"runtime"

"github.com/gohugoio/hugo/config"

_errors "github.com/pkg/errors"

@@ -73,10 +74,7 @@ func (s *siteContentProcessor) processAsset(asset pathLangFile) {
}

func newSiteContentProcessor(ctx context.Context, partialBuild bool, s *Site) *siteContentProcessor {
numWorkers := 12
if n := runtime.NumCPU() * 3; n > numWorkers {
numWorkers = n
}
numWorkers := config.GetNumWorkerMultiplier() * 3

numWorkers = int(math.Ceil(float64(numWorkers) / float64(len(s.h.Sites))))

@@ -19,7 +19,8 @@ import (
"os"
"path"
"path/filepath"
"runtime"

"github.com/gohugoio/hugo/config"

"github.com/gohugoio/hugo/common/loggers"
_errors "github.com/pkg/errors"
@@ -70,10 +71,7 @@ func newCapturer(
contentChanges *contentChangeMap,
filenames ...string) *capturer {

numWorkers := 4
if n := runtime.NumCPU(); n > numWorkers {
numWorkers = n
}
numWorkers := config.GetNumWorkerMultiplier()

// TODO(bep) the "index" vs "_index" check/strings should be moved in one place.
isBundleHeader := func(filename string) bool {
@@ -1881,15 +1881,6 @@ func (s *Site) newPage(kind string, sections ...string) *pageState {
return p
}

func getGoMaxProcs() int {
if gmp := os.Getenv("GOMAXPROCS"); gmp != "" {
if p, err := strconv.Atoi(gmp); err == nil {
return p
}
}
return 1
}

func (s *Site) shouldBuild(p page.Page) bool {
return shouldBuild(s.BuildFuture, s.BuildExpired,
s.BuildDrafts, p.Draft(), p.PublishDate(), p.ExpiryDate())
@@ -19,6 +19,8 @@ import (
"strings"
"sync"

"github.com/gohugoio/hugo/config"

"github.com/gohugoio/hugo/output"
"github.com/pkg/errors"

@@ -55,7 +57,7 @@ func (s siteRenderContext) renderSingletonPages() bool {
// TODO(bep np doc
func (s *Site) renderPages(ctx *siteRenderContext) error {

numWorkers := getGoMaxProcs() * 4
numWorkers := config.GetNumWorkerMultiplier()

results := make(chan error)
pages := make(chan *pageState, numWorkers) // buffered for performance
@@ -14,16 +14,12 @@
package main

import (
"runtime"

"os"

"github.com/gohugoio/hugo/commands"
)

func main() {

runtime.GOMAXPROCS(runtime.NumCPU())
resp := commands.Execute(os.Args[1:])

if resp.Err != nil {

0 comments on commit 87b16ab

Please sign in to comment.
You can’t perform that action at this time.