Skip to content
This repository has been archived by the owner on Mar 9, 2019. It is now read-only.

page already freed #731

Open
deepch opened this issue Sep 27, 2017 · 5 comments
Open

page already freed #731

deepch opened this issue Sep 27, 2017 · 5 comments

Comments

@deepch
Copy link

deepch commented Sep 27, 2017

curent version git i can upload db

Linux sr7-ipeye 4.4.0-96-generic #119-Ubuntu SMP Tue Sep 12 14:59:54 UTC 2017 x86_64 x86_64 x86_64 GNU/Linux

Description: Ubuntu 16.04.3 LTS

func (this *Database) Clean() {
	a := []string{}
	if this.db != nil {
		this.db.Update(func(tx *bolt.Tx) error {
			start := time.Now().AddDate(0, 0, -7)
			seek := []byte(start.Format(time.RFC3339))
			c := tx.Cursor()
			for k, _ := c.Seek(seek); k != nil; k, _ = c.Next() {
				a = append(a, string(k))
			}
			return nil
		})
		if len(a) > 0 {
			for _, v := range a {
				this.db.Update(func(tx *bolt.Tx) error {
					start := time.Now().AddDate(0, 0, -7)
					seek := []byte(start.Format(time.RFC3339))
					bkt := tx.Bucket([]byte(v))
					c := bkt.Cursor()
					for k, _ := c.First(); k != nil && bytes.Compare(k, seek) <= 0; k, _ = c.Next() {
						c.Delete()
					}
					return nil
				})
			}
		}
	}
}
panic: page 1890706 already freed

goroutine 120 [running]:
github.com/boltdb/bolt.(*freelist).free(0xc420459800, 0x187c83, 0x7f01a1992000)
        /home/deepweb/test/src/github.com/boltdb/bolt/freelist.go:121 +0x301
github.com/boltdb/bolt.(*node).spill(0xc819003960, 0xc4c4119960, 0x11342e0)
        /home/deepweb/test/src/github.com/boltdb/bolt/node.go:363 +0x210
github.com/boltdb/bolt.(*node).spill(0xc7be1c0150, 0xc4c4119940, 0x11342e0)
        /home/deepweb/test/src/github.com/boltdb/bolt/node.go:350 +0xbf
github.com/boltdb/bolt.(*node).spill(0xc7be1c00e0, 0xc430fc1500, 0xc6c31b9aa8)
        /home/deepweb/test/src/github.com/boltdb/bolt/node.go:350 +0xbf
github.com/boltdb/bolt.(*Bucket).spill(0xc55f250780, 0xc430fc1400, 0xc6c31b9d20)
        /home/deepweb/test/src/github.com/boltdb/bolt/bucket.go:570 +0x4d3
github.com/boltdb/bolt.(*Bucket).spill(0xc8795ba2b8, 0x20fec8a82ccae, 0x1166ce0)
        /home/deepweb/test/src/github.com/boltdb/bolt/bucket.go:537 +0x417
github.com/boltdb/bolt.(*Tx).Commit(0xc8795ba2a0, 0x0, 0x0)
        /home/deepweb/test/src/github.com/boltdb/bolt/tx.go:163 +0x129
github.com/boltdb/bolt.(*DB).Update(0xc420210000, 0xc6c31b9f80, 0x0, 0x0)
        /home/deepweb/test/src/github.com/boltdb/bolt/db.go:605 +0xf2
main.(*Database).Clean(0xc4203ee1e0)
        /home/deepweb/test/src/github.com/deepch/dd/ModuleDatabase.go:410 +0xf2
main.event_clener()
        /home/deepweb/test/src/github.com/deepch/dd/ModuleDatabase.go:499 +0x2d
created by main.event_init
        /home/deepweb/test/src/github.com/deepch/dd/ModuleDatabase.go:494 +0x99

@djadala
Copy link

djadala commented Dec 22, 2017

Same here:

panic: page 162597 already freed

goroutine 17 [running]:
github.com/boltdb/bolt.(*freelist).free(0xc420078330, 0x7b, 0x7efe5cb57000)
	/home/jambo/golang/src/github.com/boltdb/bolt/freelist.go:121 +0x301
github.com/boltdb/bolt.(*node).spill(0xc42381a3f0, 0x0, 0x0)
	/home/jambo/golang/src/github.com/boltdb/bolt/node.go:363 +0x210
github.com/boltdb/bolt.(*node).spill(0xc423d73030, 0x0, 0x0)
	/home/jambo/golang/src/github.com/boltdb/bolt/node.go:350 +0xbf
github.com/boltdb/bolt.(*node).spill(0xc423d72fc0, 0xc426b1a0c0, 0xc420046a80)
	/home/jambo/golang/src/github.com/boltdb/bolt/node.go:350 +0xbf
github.com/boltdb/bolt.(*Bucket).spill(0xc425316100, 0xc426b1a000, 0xc420046cf8)
	/home/jambo/golang/src/github.com/boltdb/bolt/bucket.go:570 +0x4d3
github.com/boltdb/bolt.(*Bucket).spill(0xc422ad40f8, 0x1f3dd3f0d0, 0x59e980)
	/home/jambo/golang/src/github.com/boltdb/bolt/bucket.go:537 +0x417
github.com/boltdb/bolt.(*Tx).Commit(0xc422ad40e0, 0x0, 0x0)
	/home/jambo/golang/src/github.com/boltdb/bolt/tx.go:163 +0x129
github.com/boltdb/bolt.(*DB).Update(0xc422a10000, 0xc42006dfa0, 0x0, 0x0)
	/home/jambo/golang/src/github.com/boltdb/bolt/db.go:605 +0xf2
main.Add(0xc420072060, 0x5087ae, 0x13)

I insert 40Gib bytes to new db file, average slice len=30 bytes,max=30000:

func Add(c chan []byte, bfile string) {
	const bname = `bc41`

	db, err := bolt.Open(bfile, 0600, nil)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	err = db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists([]byte(bname))
		if err != nil {
			return err //fmt.Errorf("create bucket: %s", err)
		}
		return nil
	})
	if err != nil {
		log.Fatal(err)
		panic(err)
	}

	for {
		err := db.Update(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(bname))
			for i := 0; i < 100000; i++ {
				v := <-c
				if v == nil {
					return nil
				}
				err = b.Put(v, []byte("0"))
				if err != nil {
					return err
				}
			}
			return nil
		})
		if err != nil {
			log.Fatal(err)
			panic(err)
		}
	}
}

@djadala
Copy link

djadala commented Dec 22, 2017

reproducer, crash when file is 28 MiB:

package main

import (
	"log"
	"math/rand"

	// "github.com/coreos/bbolt"
	"github.com/boltdb/bolt"
)

var BC = make(chan []byte)

func main() {

	go Add(BC, "/opt/test.db")

	s := make([]byte, 30)
	for i := 0; i < 100000000; i++ {
		rand.Read(s)
		BC <- s
	}

	close(BC)

}

func Add(c chan []byte, bfile string) {
	const bname = `bc41`

	db, err := bolt.Open(bfile, 0600, nil)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	err = db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists([]byte(bname))
		if err != nil {
			return err //fmt.Errorf("create bucket: %s", err)
		}
		return nil
	})
	if err != nil {
		log.Fatal(err)
		panic(err)
	}

	for {
		err := db.Update(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(bname))
			for i := 0; i < 100000; i++ {
				v := <-c
				if v == nil {
					return nil
				}
				err = b.Put(v, []byte("0"))
				if err != nil {
					return err
				}
			}
			return nil
		})
		if err != nil {
			log.Fatal(err)
			panic(err)
		}
	}
}
panic: page 420 already freed

goroutine 5 [running]:
github.com/boltdb/bolt.(*freelist).free(0xc4200741e0, 0x7, 0x7ff7e4a19000)
	/home/jambo/golang/src/github.com/boltdb/bolt/freelist.go:121 +0x301
github.com/boltdb/bolt.(*node).spill(0xc422ca07e0, 0x0, 0x0)
	/home/jambo/golang/src/github.com/boltdb/bolt/node.go:363 +0x210
github.com/boltdb/bolt.(*node).spill(0xc4202b7650, 0xc4201407e0, 0x55d920)
	/home/jambo/golang/src/github.com/boltdb/bolt/node.go:350 +0xbf
github.com/boltdb/bolt.(*node).spill(0xc420e503f0, 0xc4222d20c0, 0xc420049a80)
	/home/jambo/golang/src/github.com/boltdb/bolt/node.go:350 +0xbf
github.com/boltdb/bolt.(*Bucket).spill(0xc4200523c0, 0xc4222d2000, 0xc420049cf8)
	/home/jambo/golang/src/github.com/boltdb/bolt/bucket.go:570 +0x4d3
github.com/boltdb/bolt.(*Bucket).spill(0xc4200900f8, 0x20140c719, 0x570160)
	/home/jambo/golang/src/github.com/boltdb/bolt/bucket.go:537 +0x417
github.com/boltdb/bolt.(*Tx).Commit(0xc4200900e0, 0x0, 0x0)
	/home/jambo/golang/src/github.com/boltdb/bolt/tx.go:163 +0x129
github.com/boltdb/bolt.(*DB).Update(0xc420088000, 0xc42006dfa0, 0x0, 0x0)
	/home/jambo/golang/src/github.com/boltdb/bolt/db.go:605 +0xf2
main.Add(0xc420072060, 0x4e88fb, 0x12)
	/home/jambo/go/src/jambo/tests/error1/error1.go:49 +0x1d4
created by main.main
	/home/jambo/go/src/jambo/tests/error1/error1.go:15 +0x5a

@djadala
Copy link

djadala commented Dec 22, 2017

go version 
go version go1.9.1 linux/amd64

uname -a
Linux bee 4.9.0-0.bpo.4-amd64 #1 SMP Debian 4.9.51-1~bpo8+1 (2017-10-17) x86_64 GNU/Linux

@djadala
Copy link

djadala commented Dec 22, 2017

Opened issue in coreos/bbolt:
etcd-io/bbolt#72

@djadala
Copy link

djadala commented Jan 3, 2018

There is race in my example, modified example without race works fine.
@deepch , can you run your program under race detector ?

Sign up for free to subscribe to this conversation on GitHub. Already have an account? Sign in.
Labels
None yet
Projects
None yet
Development

No branches or pull requests

2 participants