text/template, html/template: Multithread template parse - panic (concurrent map writes) #19170
Description
I have a problem with multi-threading template.Parse
Go template system (text/template/template & html/template/template) contains sync.Mutex to prevent
simultaneously access but Parse is still unsafe.
What version of Go are you using (go version)?
1.8 (and olders)
https://github.com/goatcms/goat-core/tree/issues-1-multithread-template-parse/goathtml/ghprovider
What operating system and processor architecture are you using (go env)?
set GOARCH=amd64
set GOBIN=
set GOEXE=.exe
set GOHOSTARCH=amd64
set GOHOSTOS=windows
set GOOS=windows
set GOPATH=C:\Users\Sebastian Pożoga/work
set GORACE=
set GOROOT=C:\Go
set GOTOOLDIR=C:\Go\pkg\tool\windows_amd64
set GCCGO=gccgo
set CC=gcc
set GOGCCFLAGS=-m64 -mthreads -fmessage-length=0 -fdebug-prefix-map=C:\Users\SEBAST~1\AppData\Local\Temp\go-build062317335=/tmp/go-build -gno-record-gcc-switches
set CXX=g++
set CGO_ENABLED=1
set PKG_CONFIG=pkg-config
set CGO_CFLAGS=-g -O2
set CGO_CPPFLAGS=
set CGO_CXXFLAGS=-g -O2
set CGO_FFLAGS=-g -O2
set CGO_LDFLAGS=-g -O2
What did you do?
I prepare code to simultaneouslytemplate parse.
Source: https://github.com/goatcms/goatcore/tree/issue-1-multithread-template-parse
( issue-1-multithread-template-parse branch is important )
and I take error after run tests file by
go test github.com/goatcms/goatcore/goathtml/ghprovider
Test code:
func TestLoadManyFiles(t *testing.T) {
var (
funcs = template.FuncMap{"join": strings.Join}
guardians = []string{"Gamora", "Groot", "Nebula", "Rocket", "Star-Lord"}
)
fs, err := memfs.NewFilespace()
if err != nil {
t.Error(err)
}
// create test data
if err := fs.MkdirAll("layouts/", 0777); err != nil {
t.Error(err)
return
}
if err := fs.MkdirAll("views/", 0777); err != nil {
t.Error(err)
return
}
if err := fs.WriteFile("layouts/default/main.gohtml", []byte(overlayTemplate), 0777); err != nil {
t.Error(err)
return
}
if err := fs.WriteFile("views/myview/file1.gohtml", []byte(templateFile1), 0777); err != nil {
t.Error(err)
return
}
if err := fs.WriteFile("views/myview/file2.gohtml", []byte(templateFile2), 0777); err != nil {
t.Error(err)
return
}
if err := fs.WriteFile("views/myview/file3.gohtml", []byte(templateFile3), 0777); err != nil {
t.Error(err)
return
}
if err := fs.WriteFile("views/myview/file4.gohtml", []byte(templateFile4), 0777); err != nil {
t.Error(err)
return
}
// test loop
for ti := 0; ti < workers.AsyncTestReapeat; ti++ {
provider := NewProvider(fs, goathtml.LayoutPath, goathtml.ViewPath, funcs)
view, errs := provider.View(goathtml.DefaultLayout, "myview", nil)
if errs != nil {
t.Errorf("Errors: %v", errs)
return
}
buf := new(bytes.Buffer)
if err := view.Execute(buf, guardians); err != nil {
t.Error(err)
return
}
if strings.Contains(buf.String(), "Gamora,") {
t.Errorf("layout template should be overwrited")
return
}
}
}
If possible, provide a recipe for reproducing the error.
git clone https://github.com/goatcms/goatcore
git checkout -b issues-1-multithread-template-parse
go test github.com/goatcms/goatcore/goathtml/ghprovider
What did you expect to see?
correct processed template (without panic).
What did you see instead?
stacktrace:
fatal error: concurrent map writes
fatal error: concurrent map writes
goroutine 22168 [running]:
runtime.throw(0x60bbcb, 0x15)
C:/Go/src/runtime/panic.go:596 +0x9c fp=0xc042379b20 sp=0xc042379b00
runtime.mapassign(0x5cf020, 0xc04264eb10, 0xc0422d5040, 0xa)
C:/Go/src/runtime/hashmap.go:499 +0x66e fp=0xc042379bc0 sp=0xc042379b20
text/template.(*Template).associate(0xc0426bc3c0, 0xc0422d5040, 0xc042665300, 0x2, 0x6a9c1644, 0xc042379d28)
C:/Go/src/text/template/template.go:223 +0xa4 fp=0xc042379c08 sp=0xc042379bc0
text/template.(*Template).AddParseTree(0xc0426bc3c0, 0xc0425e3f8a, 0xa, 0xc042665300, 0x0, 0x0, 0x0)
C:/Go/src/text/template/template.go:128 +0x117 fp=0xc042379c68 sp=0xc042379c08
text/template.(*Template).Parse(0xc0426bc3c0, 0xc0425e3f80, 0x2f, 0x2f, 0xc0425e3f80, 0x2f)
C:/Go/src/text/template/template.go:204 +0x1f7 fp=0xc042379d70 sp=0xc042379c68
html/template.(*Template).Parse(0xc04264ee10, 0xc0425e3f80, 0x2f, 0x0, 0x0, 0x0)
C:/Go/src/html/template/template.go:186 +0xad fp=0xc042379e20 sp=0xc042379d70
github.com/goatcms/goatcore/goathtml/ghprovider.(*TemplateLoader).Load(0xc042058038, 0x6e8d20, 0xc042058268, 0xc0424d22a0, 0x19, 0x592533, 0x20)
C:/Users/Sebastian Pożoga/work/src/github.com/goatcms/goatcore/goathtml/ghprovider/loader.go:24 +0xbd fp=0xc042379e78 sp=0xc042379e20
github.com/goatcms/goatcore/goathtml/ghprovider.(*TemplateLoader).Load-fm(0x6e8d20, 0xc042058268, 0xc0424d22a0, 0x19, 0xffffffffffffff01, 0x5419aa)
C:/Users/Sebastian Pożoga/work/src/github.com/goatcms/goatcore/goathtml/ghprovider/main.go:64 +0x59 fp=0xc042379ec0 sp=0xc042379e78
github.com/goatcms/goatcore/filesystem/fsloop.(*Consumer).Loop(0xc0424d2240)
C:/Users/Sebastian Pożoga/work/src/github.com/goatcms/goatcore/filesystem/fsloop/consumer.go:32 +0x297 fp=0xc042379fd8 sp=0xc042379ec0
runtime.goexit()
C:/Go/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc042379fe0 sp=0xc042379fd8
created by github.com/goatcms/goatcore/filesystem/fsloop.(*Loop).run
C:/Users/Sebastian Pożoga/work/src/github.com/goatcms/goatcore/filesystem/fsloop/loop.go:61 +0x294
goroutine 1 [chan receive]:
testing.(*T).Run(0xc04204c680, 0x609de6, 0x11, 0x6147d0, 0xc042035d01)
C:/Go/src/testing/testing.go:698 +0x2fb
testing.runTests.func1(0xc04204c680)
C:/Go/src/testing/testing.go:882 +0x6e
testing.tRunner(0xc04204c680, 0xc042035de0)
C:/Go/src/testing/testing.go:657 +0x9d
testing.runTests(0xc042046d40, 0x6f6980, 0x2, 0x2, 0xc042040ec0)
C:/Go/src/testing/testing.go:888 +0x2c8
testing.(*M).Run(0xc0423b7f20, 0xc042035f20)
C:/Go/src/testing/testing.go:822 +0x103
main.main()
github.com/goatcms/goatcore/goathtml/ghprovider/_test/_testmain.go:44 +0xfe
goroutine 22034 [semacquire]:
sync.runtime_Semacquire(0xc04264eee4)
C:/Go/src/runtime/sema.go:47 +0x3b
sync.(*WaitGroup).Wait(0xc04264eed8)
C:/Go/src/sync/waitgroup.go:131 +0x81
github.com/goatcms/goatcore/workers/jobsync.(*Pool).Wait(0xc04264eed0)
C:/Users/Sebastian Pożoga/work/src/github.com/goatcms/goatcore/workers/jobsync/pool.go:39 +0x38
github.com/goatcms/goatcore/filesystem/fsloop.(*Loop).Wait(0xc04264ee40)
C:/Users/Sebastian Pożoga/work/src/github.com/goatcms/goatcore/filesystem/fsloop/loop.go:73 +0x36
github.com/goatcms/goatcore/goathtml/ghprovider.(*Provider).view(0xc04203e0c0, 0x604fcc, 0x7, 0x604482, 0x6, 0xc04212a020, 0xe, 0x0, 0x0, 0x0, ...)
C:/Users/Sebastian Pożoga/work/src/github.com/goatcms/goatcore/goathtml/ghprovider/main.go:120 +0x3fc
github.com/goatcms/goatcore/goathtml/ghprovider.(*Provider).View(0xc04203e0c0, 0x604fcc, 0x7, 0x604482, 0x6, 0x0, 0x0, 0xc0425e3a70, 0x0, 0x0)
C:/Users/Sebastian Pożoga/work/src/github.com/goatcms/goatcore/goathtml/ghprovider/main.go:89 +0x193
github.com/goatcms/goatcore/goathtml/ghprovider.TestLoadManyFiles(0xc0422b8270)
C:/Users/Sebastian Pożoga/work/src/github.com/goatcms/goatcore/goathtml/ghprovider/main_test.go:110 +0x940
testing.tRunner(0xc0422b8270, 0x6147d0)
C:/Go/src/testing/testing.go:657 +0x9d
created by testing.(*T).Run
C:/Go/src/testing/testing.go:697 +0x2d1
goroutine 22164 [running]:
goroutine running on other thread; stack unavailable
created by github.com/goatcms/goatcore/filesystem/fsloop.(*Loop).run
C:/Users/Sebastian Pożoga/work/src/github.com/goatcms/goatcore/filesystem/fsloop/loop.go:61 +0x294
goroutine 22164 [running]:
runtime.throw(0x60bbcb, 0x15)
C:/Go/src/runtime/panic.go:596 +0x9c fp=0xc0421ddb20 sp=0xc0421ddb00
runtime.mapassign(0x5cf020, 0xc04264eb10, 0xc04268e340, 0xa)
C:/Go/src/runtime/hashmap.go:499 +0x66e fp=0xc0421ddbc0 sp=0xc0421ddb20
text/template.(*Template).associate(0xc0426bc3c0, 0xc04268e340, 0xc0420ea200, 0x2, 0x5117d3d4, 0xc0421ddd28)
C:/Go/src/text/template/template.go:223 +0xa4 fp=0xc0421ddc08 sp=0xc0421ddbc0
text/template.(*Template).AddParseTree(0xc0426bc3c0, 0xc04200881a, 0xa, 0xc0420ea200, 0x0, 0x0, 0x0)
C:/Go/src/text/template/template.go:128 +0x117 fp=0xc0421ddc68 sp=0xc0421ddc08
text/template.(*Template).Parse(0xc0426bc3c0, 0xc042008810, 0x2f, 0x2f, 0xc042008810, 0x2f)
C:/Go/src/text/template/template.go:204 +0x1f7 fp=0xc0421ddd70 sp=0xc0421ddc68
html/template.(*Template).Parse(0xc04264ee10, 0xc042008810, 0x2f, 0x0, 0x0, 0x0)
C:/Go/src/html/template/template.go:186 +0xad fp=0xc0421dde20 sp=0xc0421ddd70
github.com/goatcms/goatcore/goathtml/ghprovider.(*TemplateLoader).Load(0xc042058038, 0x6e8d20, 0xc042058268, 0xc0424d22c0, 0x19, 0x592533, 0x20)
C:/Users/Sebastian Pożoga/work/src/github.com/goatcms/goatcore/goathtml/ghprovider/loader.go:24 +0xbd fp=0xc0421dde78 sp=0xc0421dde20
github.com/goatcms/goatcore/goathtml/ghprovider.(*TemplateLoader).Load-fm(0x6e8d20, 0xc042058268, 0xc0424d22c0, 0x19, 0xffffffffffffff01, 0x4)
C:/Users/Sebastian Pożoga/work/src/github.com/goatcms/goatcore/goathtml/ghprovider/main.go:64 +0x59 fp=0xc0421ddec0 sp=0xc0421dde78
github.com/goatcms/goatcore/filesystem/fsloop.(*Consumer).Loop(0xc0424d2240)
C:/Users/Sebastian Pożoga/work/src/github.com/goatcms/goatcore/filesystem/fsloop/consumer.go:32 +0x297 fp=0xc0421ddfd8 sp=0xc0421ddec0
runtime.goexit()
C:/Go/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc0421ddfe0 sp=0xc0421ddfd8
created by github.com/goatcms/goatcore/filesystem/fsloop.(*Loop).run
C:/Users/Sebastian Pożoga/work/src/github.com/goatcms/goatcore/filesystem/fsloop/loop.go:61 +0x294
FAIL github.com/goatcms/goatcore/goathtml/ghprovider 0.183s