From 3a24a15b769132006481279fba26a5b6de695ec6 Mon Sep 17 00:00:00 2001 From: Jan Mercl <0xjnml@gmail.com> Date: Sun, 4 Nov 2018 12:54:10 +0100 Subject: [PATCH] Add a new back end file format. To use the new format for newly created databases set the FileFormat field in *Options passed to OpenFile to value 2 or use the driver named "ql2" instead of "ql". - Both the old and new driver will properly open and use, read and write the old (V1) or new file (V2) format of an existing database. - V1 format has a record size limit of ~64 kB. V2 format record size limit is math.MaxInt32. - V1 format uncommitted transaction size is limited by memory resources. V2 format uncommitted transaction is limited by free disk space. - A direct consequence of the previous is that small transactions perform better using V1 format and big transactions perform better using V2 format. benchmark old ns/op new ns/op delta SelectOrderedFile1kBx1e4-4 1329006807 255517723 -80.77% CrossJoinFile1e3NoX1e4-4 3107615747 619794528 -80.06% CrossJoinFile1e4NoX1e3-4 3354802646 700357061 -79.12% SelectOrderedFile1kBx1e3-4 123908159 26627421 -78.51% CrossJoinFile1e2NoX1e3-4 27598600 6474143 -76.54% CrossJoinFile1e3X1e4-4 335359531 78790606 -76.51% SelectFile1kBx1e3-4 4562622 1092952 -76.05% SelectBoolFileNoX1e3Perc5-4 1781983 517165 -70.98% SelectBoolFileNoX1e3Perc50-4 1789031 519676 -70.95% SelectBoolFileX1e4Perc5-4 1199502 359018 -70.07% SelectFile1kBx1e4-4 40393899 12168884 -69.87% SelectFile1kBx1e5-4 398938019 122302661 -69.34% SelectBoolFileNoX1e4Perc50-4 17760045 5522018 -68.91% SelectBoolFileX1e3Perc50-4 1050091 328726 -68.70% SelectBoolFileNoX1e4Perc5-4 17723524 5808971 -67.22% SelectBoolFileX1e4Perc50-4 10402097 3697792 -64.45% InsertBoolFileX1e3-4 41825364 15918052 -61.94% SelectOrderedFile1kBx1e2-4 9774335 4492106 -54.04% CrossJoinFile1e3X1e2-4 1617783 785663 -51.44% InsertBoolFileX1e1-4 4927433 9665456 +96.16% InsertBoolFileNoX1e2-4 4638139 9045240 +95.02% SelectBoolFileX1e3Perc5-4 61143 31704 -48.15% InsertFile1kBn1e3t1e4-4 227403851 417962957 +83.80% InsertBoolFileX1e2-4 5205319 9355281 +79.73% SelectFile1kBx1e2-4 180977 101025 -44.18% InsertFile1kBn1e0t1e2-4 493998605 882267286 +78.60% CrossJoinFile1e2X1e3-4 1266888 711538 -43.84% SelectBoolFileX1e2Perc5-4 9774 5497 -43.76% CrossJoinFile1e3NoX1e2-4 11920217 6735563 -43.49% CrossJoinFile1e2X1e1-4 32795 19315 -41.10% InsertFile1kBn1e1t1e2-4 51884617 86750690 +67.20% CrossJoinFile1e1X1e2-4 30959 18643 -39.78% InsertBoolFileNoX1e1-4 4936511 8171014 +65.52% CrossJoinFile1e4X1e3-4 138960718 88406050 -36.38% InsertFile1kBn1e3t1e3-4 22921971 35507302 +54.91% SelectBoolFileX1e1Perc5-4 5363 3477 -35.17% SelectBoolFileX1e2Perc50-4 52708 34471 -34.60% InsertFile1kBn1e1t1e3-4 513335347 775494042 +51.07% InsertFile1kBn1e2t1e2-4 6725217 10065333 +49.67% CrossJoinFile1e1NoX1e2-4 241349 162557 -32.65% InsertFile1kBn1e2t1e4-4 834129466 1229457620 +47.39% SelectBoolFileX1e1Perc50-4 10724 7279 -32.12% InsertFile1kBn1e2t1e3-4 70257020 102477109 +45.86% SelectBoolFileNoX1e1Perc5-4 9691 6647 -31.41% CrossJoinFile1e2NoX1e1-4 178373 123161 -30.95% SelectBoolFileNoX1e2Perc50-4 79529 55208 -30.58% SelectBoolFileNoX1e1Perc50-4 9276 6542 -29.47% InsertFile1kBn1e3t1e5-4 2581234930 3629377500 +40.61% SelectBoolFileNoX1e2Perc5-4 82316 58945 -28.39% InsertBoolFileNoX1e3-4 8478206 10126515 +19.44% - V2 format uses substantially less memory. benchmark old bytes new bytes delta InsertFile1kBn1e3t1e3-4 4683629 214099 -95.43% InsertFile1kBn1e3t1e4-4 42326873 2174786 -94.86% InsertFile1kBn1e3t1e5-4 397886672 21182824 -94.68% InsertFile1kBn1e2t1e3-4 4111234 221634 -94.61% InsertFile1kBn1e2t1e4-4 40488844 2214520 -94.53% InsertFile1kBn1e2t1e2-4 400895 22224 -94.46% InsertFile1kBn1e1t1e3-4 5063280 411028 -91.88% InsertFile1kBn1e1t1e2-4 498687 41186 -91.74% InsertFile1kBn1e0t1e2-4 1571434 238228 -84.84% SelectBoolFileX1e3Perc5-4 52729 10235 -80.59% InsertBoolFileX1e1-4 33802 7251 -78.55% SelectOrderedFile1kBx1e2-4 1486433 373721 -74.86% SelectOrderedFile1kBx1e3-4 16538318 4318323 -73.89% SelectBoolFileX1e2Perc5-4 7307 1944 -73.40% SelectOrderedFile1kBx1e4-4 170936232 48409036 -71.68% CrossJoinFile1e3X1e2-4 814207 255715 -68.59% CrossJoinFile1e1X1e2-4 17059 5385 -68.43% CrossJoinFile1e2X1e3-4 573394 191340 -66.63% CrossJoinFile1e4X1e3-4 78447465 27922856 -64.41% CrossJoinFile1e2X1e1-4 19173 6954 -63.73% SelectFile1kBx1e3-4 3518328 1344859 -61.78% SelectFile1kBx1e5-4 351685469 134437012 -61.77% SelectFile1kBx1e4-4 35166002 13444622 -61.77% SelectFile1kBx1e2-4 351753 134910 -61.65% InsertBoolFileX1e3-4 2646597 1070045 -59.57% InsertBoolFileX1e2-4 179300 86247 -51.90% SelectBoolFileX1e3Perc50-4 220300 108229 -50.87% InsertBoolFileNoX1e1-4 9348 4622 -50.56% SelectBoolFileX1e2Perc50-4 22429 11300 -49.62% SelectBoolFileX1e4Perc5-4 221089 113039 -48.87% CrossJoinFile1e3X1e4-4 46730586 24064380 -48.50% SelectBoolFileX1e1Perc5-4 2521 1360 -46.05% SelectBoolFileX1e4Perc50-4 1929656 1055191 -45.32% SelectBoolFileX1e1Perc50-4 4530 2600 -42.60% SelectBoolFileNoX1e4Perc50-4 2403360 1521074 -36.71% SelectBoolFileNoX1e4Perc5-4 2403349 1521074 -36.71% SelectBoolFileNoX1e3Perc5-4 241282 153072 -36.56% SelectBoolFileNoX1e3Perc50-4 241280 153072 -36.56% CrossJoinFile1e2NoX1e3-4 2169344 1389074 -35.97% CrossJoinFile1e3NoX1e4-4 241111128 154415488 -35.96% CrossJoinFile1e4NoX1e3-4 278177376 178324552 -35.90% CrossJoinFile1e3NoX1e2-4 2714091 1752226 -35.44% SelectBoolFileNoX1e2Perc5-4 25076 16272 -35.11% SelectBoolFileNoX1e2Perc50-4 25076 16272 -35.11% CrossJoinFile1e1NoX1e2-4 57853 37680 -34.87% CrossJoinFile1e2NoX1e1-4 52762 35472 -32.77% SelectBoolFileNoX1e1Perc5-4 3472 2592 -25.35% SelectBoolFileNoX1e1Perc50-4 3472 2592 -25.35% InsertBoolFileNoX1e2-4 45394 39760 -12.41% InsertBoolFileNoX1e3-4 423187 391102 -7.58% modified: Makefile modified: all_test.go modified: doc.go modified: driver.go modified: driver/all_test.go modified: driver/driver.go new file: encode2.go modified: etc.go modified: file.go new file: file2.go new file: file2_test.go modified: plan.go modified: storage.go modified: storage_test.go new file: v2.sh --- Makefile | 8 +- all_test.go | 448 +++++++++++++++--- doc.go | 18 + driver.go | 46 +- driver/all_test.go | 99 ++++ driver/driver.go | 10 +- encode2.go | 485 +++++++++++++++++++ etc.go | 30 ++ file.go | 72 ++- file2.go | 1108 ++++++++++++++++++++++++++++++++++++++++++++ file2_test.go | 264 +++++++++++ plan.go | 4 +- storage.go | 4 - storage_test.go | 5 +- v2.sh | 4 + 15 files changed, 2512 insertions(+), 93 deletions(-) create mode 100644 encode2.go create mode 100644 file2.go create mode 100644 file2_test.go create mode 100755 v2.sh diff --git a/Makefile b/Makefile index 627329f..f9f7eaa 100644 --- a/Makefile +++ b/Makefile @@ -32,12 +32,16 @@ cover: t=$(shell mktemp) ; go test -coverprofile $$t && go tool cover -html $$t && unlink $$t cpu: clean - go test -run @ -bench . -cpuprofile cpu.out + go test -run @ -bench BenchmarkInsertBoolFileNoX1e2 -cpuprofile cpu.out -benchmem -benchtime 4s go tool pprof -lines *.test cpu.out edit: @ 1>/dev/null 2>/dev/null gvim -p Makefile *.l *.y *.go testdata.ql testdata.log +edit2: + touch log + @ 1>/dev/null 2>/dev/null gvim -p Makefile all_test.go log driver*.go encode2.go file*.go mem.go ql.go storage*.go testdata.ql testdata.log + editor: ql.y scanner.go parser.go coerce.go gofmt -s -l -w *.go go test -i @@ -51,7 +55,7 @@ later: @grep -n $(grep) MAYBE * || true mem: clean - go test -run @ -bench . -memprofile mem.out -memprofilerate 1 -timeout 24h + go test -run @ -bench BenchmarkInsertBoolFileNoX1e2 -memprofile mem.out -memprofilerate 1 -timeout 24h -benchmem -benchtime 4s go tool pprof -lines -web -alloc_space *.test mem.out nuke: clean diff --git a/all_test.go b/all_test.go index 0e2e8a4..a0f53cd 100644 --- a/all_test.go +++ b/all_test.go @@ -38,6 +38,7 @@ func init() { isTesting = true use(dieHard, caller, (*DB).dumpTables, dumpTables2, dumpTables3, dumpFields, dumpFlds, dumpCols, fldsString, typeof, stypeof) flag.IntVar(&yyDebug, "yydebug", 0, "") + use(dbg) } func dieHard(exitValue int) { @@ -223,6 +224,66 @@ func (m *fileTestDB) teardown(ctx *TCtx) (err error) { return err } +type file2TestDB struct { + db *DB + gmp0 int + m0 int64 +} + +func (m *file2TestDB) setup() (db *DB, err error) { + m.gmp0 = runtime.GOMAXPROCS(0) + f, err := ioutil.TempFile("", "ql-test-") + if err != nil { + return + } + + if m.db, err = OpenFile(f.Name(), &Options{FileFormat: 2}); err != nil { + return + } + + return m.db, nil +} + +func (m *file2TestDB) mark() (err error) { + m.m0, err = m.db.store.Verify() + if err != nil { + m.m0 = -1 + } + return +} + +func (m *file2TestDB) teardown(ctx *TCtx) (err error) { + runtime.GOMAXPROCS(m.gmp0) + defer func() { + f := m.db.store.(*storage2) + errSet(&err, m.db.Close()) + os.Remove(f.Name()) + if f.walName != "" { + os.Remove(f.walName) + } + }() + + if m.m0 < 0 { + return + } + + n, err := m.db.store.Verify() + if err != nil { + return + } + + if g, e := n, m.m0; g != e { + return fmt.Errorf("STORAGE LEAK: allocs: got %d, exp %d", g, e) + } + + if ctx == nil { + return nil + } + + _, _, err = m.db.Execute(ctx, txCommit) + return err +} + type osFileTestDB struct { db *DB gmp0 int @@ -292,7 +353,7 @@ func TestFileStorage(t *testing.T) { t.Skip("skipping test in short mode.") } - test(t, &fileTestDB{}) + test(t, &file2TestDB{}) } func TestOSFileStorage(t *testing.T) { @@ -303,6 +364,14 @@ func TestOSFileStorage(t *testing.T) { test(t, &osFileTestDB{}) } +func TestFile2Storage(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + test(t, &file2TestDB{}) +} + var ( compiledCommit = MustCompile("COMMIT;") compiledCreate = MustCompile("BEGIN TRANSACTION; CREATE TABLE t (i16 int16, s16 string, s string);") @@ -413,18 +482,34 @@ func BenchmarkSelectFile1kBx1e2(b *testing.B) { benchmarkSelect(b, 1e2, compiledSelect, &fileTestDB{}) } +func BenchmarkSelectFileV21kBx1e2(b *testing.B) { + benchmarkSelect(b, 1e2, compiledSelect, &file2TestDB{}) +} + func BenchmarkSelectFile1kBx1e3(b *testing.B) { benchmarkSelect(b, 1e3, compiledSelect, &fileTestDB{}) } +func BenchmarkSelectFileV21kBx1e3(b *testing.B) { + benchmarkSelect(b, 1e3, compiledSelect, &file2TestDB{}) +} + func BenchmarkSelectFile1kBx1e4(b *testing.B) { benchmarkSelect(b, 1e4, compiledSelect, &fileTestDB{}) } +func BenchmarkSelectFileV21kBx1e4(b *testing.B) { + benchmarkSelect(b, 1e4, compiledSelect, &file2TestDB{}) +} + func BenchmarkSelectFile1kBx1e5(b *testing.B) { benchmarkSelect(b, 1e5, compiledSelect, &fileTestDB{}) } +func BenchmarkSelectFileV21kBx1e5(b *testing.B) { + benchmarkSelect(b, 1e5, compiledSelect, &file2TestDB{}) +} + func BenchmarkSelectOrderedMem1kBx1e2(b *testing.B) { benchmarkSelect(b, 1e2, compiledSelectOrderBy, &memTestDB{}) } @@ -441,14 +526,26 @@ func BenchmarkSelectOrderedFile1kBx1e2(b *testing.B) { benchmarkSelect(b, 1e2, compiledSelectOrderBy, &fileTestDB{}) } +func BenchmarkSelectOrderedFileV21kBx1e2(b *testing.B) { + benchmarkSelect(b, 1e2, compiledSelectOrderBy, &file2TestDB{}) +} + func BenchmarkSelectOrderedFile1kBx1e3(b *testing.B) { benchmarkSelect(b, 1e3, compiledSelectOrderBy, &fileTestDB{}) } +func BenchmarkSelectOrderedFileV21kBx1e3(b *testing.B) { + benchmarkSelect(b, 1e3, compiledSelectOrderBy, &file2TestDB{}) +} + func BenchmarkSelectOrderedFile1kBx1e4(b *testing.B) { benchmarkSelect(b, 1e4, compiledSelectOrderBy, &fileTestDB{}) } +func BenchmarkSelectOrderedFileV21kBx1e4(b *testing.B) { + benchmarkSelect(b, 1e4, compiledSelectOrderBy, &file2TestDB{}) +} + func TestString(t *testing.T) { for _, v := range testdata { a := strings.Split(v, "\n|") @@ -538,6 +635,10 @@ func BenchmarkInsertFile1kBn1e0t1e2(b *testing.B) { benchmarkInsert(b, 1e0, 1e2, &fileTestDB{}) } +func BenchmarkInsertFileV21kBn1e0t1e2(b *testing.B) { + benchmarkInsert(b, 1e0, 1e2, &file2TestDB{}) +} + //============================================================================= func BenchmarkInsertMem1kBn1e1t1e2(b *testing.B) { @@ -548,6 +649,10 @@ func BenchmarkInsertFile1kBn1e1t1e2(b *testing.B) { benchmarkInsert(b, 1e1, 1e2, &fileTestDB{}) } +func BenchmarkInsertFileV21kBn1e1t1e2(b *testing.B) { + benchmarkInsert(b, 1e1, 1e2, &file2TestDB{}) +} + func BenchmarkInsertMem1kBn1e1t1e3(b *testing.B) { benchmarkInsert(b, 1e1, 1e3, &memTestDB{}) } @@ -556,6 +661,10 @@ func BenchmarkInsertFile1kBn1e1t1e3(b *testing.B) { benchmarkInsert(b, 1e1, 1e3, &fileTestDB{}) } +func BenchmarkInsertFileV21kBn1e1t1e3(b *testing.B) { + benchmarkInsert(b, 1e1, 1e3, &file2TestDB{}) +} + //============================================================================= func BenchmarkInsertMem1kBn1e2t1e2(b *testing.B) { @@ -566,6 +675,10 @@ func BenchmarkInsertFile1kBn1e2t1e2(b *testing.B) { benchmarkInsert(b, 1e2, 1e2, &fileTestDB{}) } +func BenchmarkInsertFileV21kBn1e2t1e2(b *testing.B) { + benchmarkInsert(b, 1e2, 1e2, &file2TestDB{}) +} + func BenchmarkInsertMem1kBn1e2t1e3(b *testing.B) { benchmarkInsert(b, 1e2, 1e3, &memTestDB{}) } @@ -574,6 +687,10 @@ func BenchmarkInsertFile1kBn1e2t1e3(b *testing.B) { benchmarkInsert(b, 1e2, 1e3, &fileTestDB{}) } +func BenchmarkInsertFileV21kBn1e2t1e3(b *testing.B) { + benchmarkInsert(b, 1e2, 1e3, &file2TestDB{}) +} + func BenchmarkInsertMem1kBn1e2t1e4(b *testing.B) { benchmarkInsert(b, 1e2, 1e4, &memTestDB{}) } @@ -582,6 +699,10 @@ func BenchmarkInsertFile1kBn1e2t1e4(b *testing.B) { benchmarkInsert(b, 1e2, 1e4, &fileTestDB{}) } +func BenchmarkInsertFileV21kBn1e2t1e4(b *testing.B) { + benchmarkInsert(b, 1e2, 1e4, &file2TestDB{}) +} + //============================================================================= func BenchmarkInsertMem1kBn1e3t1e3(b *testing.B) { @@ -592,6 +713,10 @@ func BenchmarkInsertFile1kBn1e3t1e3(b *testing.B) { benchmarkInsert(b, 1e3, 1e3, &fileTestDB{}) } +func BenchmarkInsertFileV21kBn1e3t1e3(b *testing.B) { + benchmarkInsert(b, 1e3, 1e3, &file2TestDB{}) +} + func BenchmarkInsertMem1kBn1e3t1e4(b *testing.B) { benchmarkInsert(b, 1e3, 1e4, &memTestDB{}) } @@ -600,6 +725,10 @@ func BenchmarkInsertFile1kBn1e3t1e4(b *testing.B) { benchmarkInsert(b, 1e3, 1e4, &fileTestDB{}) } +func BenchmarkInsertFileV21kBn1e3t1e4(b *testing.B) { + benchmarkInsert(b, 1e3, 1e4, &file2TestDB{}) +} + func BenchmarkInsertMem1kBn1e3t1e5(b *testing.B) { benchmarkInsert(b, 1e3, 1e5, &memTestDB{}) } @@ -608,7 +737,14 @@ func BenchmarkInsertFile1kBn1e3t1e5(b *testing.B) { benchmarkInsert(b, 1e3, 1e5, &fileTestDB{}) } -func TestReopen(t *testing.T) { +func BenchmarkInsertFileV21kBn1e3t1e5(b *testing.B) { + benchmarkInsert(b, 1e3, 1e5, &file2TestDB{}) +} + +func TestReopen(t *testing.T) { testReopen(t, 0) } +func TestReopen2(t *testing.T) { testReopen(t, 2) } + +func testReopen(t *testing.T, ff int) { if testing.Short() { t.Skip("skipping test in short mode.") } @@ -629,7 +765,7 @@ func TestReopen(t *testing.T) { } }() - db, err := OpenFile(nm, &Options{}) + db, err := OpenFile(nm, &Options{FileFormat: ff}) if err != nil { t.Error(err) return @@ -1359,14 +1495,14 @@ func BenchmarkInsertBoolMemX1e5(b *testing.B) { benchmarkInsertBoolMem(b, 1e5, 0.5, true) } -func benchmarkInsertBoolFile(b *testing.B, size int, sel float64, index bool) { +func benchmarkInsertBoolFile(b *testing.B, size int, sel float64, index bool, ver int) { dir, err := ioutil.TempDir("", "ql-bench-") if err != nil { b.Fatal(err) } n := runtime.GOMAXPROCS(0) - db, err := OpenFile(filepath.Join(dir, "ql.db"), &Options{CanCreate: true}) + db, err := OpenFile(filepath.Join(dir, "ql.db"), &Options{CanCreate: true, FileFormat: ver}) if err != nil { b.Fatal(err) } @@ -1379,27 +1515,51 @@ func benchmarkInsertBoolFile(b *testing.B, size int, sel float64, index bool) { } func BenchmarkInsertBoolFileNoX1e1(b *testing.B) { - benchmarkInsertBoolFile(b, 1e1, 0.5, false) + benchmarkInsertBoolFile(b, 1e1, 0.5, false, 0) +} + +func BenchmarkInsertBoolFileV2NoX1e1(b *testing.B) { + benchmarkInsertBoolFile(b, 1e1, 0.5, false, 2) } func BenchmarkInsertBoolFileX1e1(b *testing.B) { - benchmarkInsertBoolFile(b, 1e1, 0.5, true) + benchmarkInsertBoolFile(b, 1e1, 0.5, true, 0) +} + +func BenchmarkInsertBoolFileV2X1e1(b *testing.B) { + benchmarkInsertBoolFile(b, 1e1, 0.5, true, 2) } func BenchmarkInsertBoolFileNoX1e2(b *testing.B) { - benchmarkInsertBoolFile(b, 1e2, 0.5, false) + benchmarkInsertBoolFile(b, 1e2, 0.5, false, 0) +} + +func BenchmarkInsertBoolFileV2NoX1e2(b *testing.B) { + benchmarkInsertBoolFile(b, 1e2, 0.5, false, 2) } func BenchmarkInsertBoolFileX1e2(b *testing.B) { - benchmarkInsertBoolFile(b, 1e2, 0.5, true) + benchmarkInsertBoolFile(b, 1e2, 0.5, true, 0) +} + +func BenchmarkInsertBoolFileV2X1e2(b *testing.B) { + benchmarkInsertBoolFile(b, 1e2, 0.5, true, 2) } func BenchmarkInsertBoolFileNoX1e3(b *testing.B) { - benchmarkInsertBoolFile(b, 1e3, 0.5, false) + benchmarkInsertBoolFile(b, 1e3, 0.5, false, 0) +} + +func BenchmarkInsertBoolFileV2NoX1e3(b *testing.B) { + benchmarkInsertBoolFile(b, 1e3, 0.5, false, 2) } func BenchmarkInsertBoolFileX1e3(b *testing.B) { - benchmarkInsertBoolFile(b, 1e3, 0.5, true) + benchmarkInsertBoolFile(b, 1e3, 0.5, true, 0) +} + +func BenchmarkInsertBoolFileV2X1e3(b *testing.B) { + benchmarkInsertBoolFile(b, 1e3, 0.5, true, 2) } var benchmarkSelectBoolOnce = map[string]bool{} @@ -1583,14 +1743,14 @@ func BenchmarkSelectBoolMemX1e5Perc5(b *testing.B) { benchmarkSelectBoolMem(b, 1e5, 0.05, true) } -func benchmarkSelectBoolFile(b *testing.B, size int, sel float64, index bool) { +func benchmarkSelectBoolFile(b *testing.B, size int, sel float64, index bool, ver int) { dir, err := ioutil.TempDir("", "ql-bench-") if err != nil { b.Fatal(err) } n := runtime.GOMAXPROCS(0) - db, err := OpenFile(filepath.Join(dir, "ql.db"), &Options{CanCreate: true}) + db, err := OpenFile(filepath.Join(dir, "ql.db"), &Options{CanCreate: true, FileFormat: ver}) if err != nil { b.Fatal(err) } @@ -1605,69 +1765,133 @@ func benchmarkSelectBoolFile(b *testing.B, size int, sel float64, index bool) { // ---- func BenchmarkSelectBoolFileNoX1e1Perc50(b *testing.B) { - benchmarkSelectBoolFile(b, 1e1, 0.5, false) + benchmarkSelectBoolFile(b, 1e1, 0.5, false, 0) +} + +func BenchmarkSelectBoolFileV2NoX1e1Perc50(b *testing.B) { + benchmarkSelectBoolFile(b, 1e1, 0.5, false, 2) } func BenchmarkSelectBoolFileX1e1Perc50(b *testing.B) { - benchmarkSelectBoolFile(b, 1e1, 0.5, true) + benchmarkSelectBoolFile(b, 1e1, 0.5, true, 0) +} + +func BenchmarkSelectBoolFileV2X1e1Perc50(b *testing.B) { + benchmarkSelectBoolFile(b, 1e1, 0.5, true, 2) } func BenchmarkSelectBoolFileNoX1e2Perc50(b *testing.B) { - benchmarkSelectBoolFile(b, 1e2, 0.5, false) + benchmarkSelectBoolFile(b, 1e2, 0.5, false, 0) +} + +func BenchmarkSelectBoolFileV2NoX1e2Perc50(b *testing.B) { + benchmarkSelectBoolFile(b, 1e2, 0.5, false, 2) } func BenchmarkSelectBoolFileX1e2Perc50(b *testing.B) { - benchmarkSelectBoolFile(b, 1e2, 0.5, true) + benchmarkSelectBoolFile(b, 1e2, 0.5, true, 0) +} + +func BenchmarkSelectBoolFileV2X1e2Perc50(b *testing.B) { + benchmarkSelectBoolFile(b, 1e2, 0.5, true, 2) } func BenchmarkSelectBoolFileNoX1e3Perc50(b *testing.B) { - benchmarkSelectBoolFile(b, 1e3, 0.5, false) + benchmarkSelectBoolFile(b, 1e3, 0.5, false, 0) +} + +func BenchmarkSelectBoolFileV2NoX1e3Perc50(b *testing.B) { + benchmarkSelectBoolFile(b, 1e3, 0.5, false, 2) } func BenchmarkSelectBoolFileX1e3Perc50(b *testing.B) { - benchmarkSelectBoolFile(b, 1e3, 0.5, true) + benchmarkSelectBoolFile(b, 1e3, 0.5, true, 0) +} + +func BenchmarkSelectBoolFileV2X1e3Perc50(b *testing.B) { + benchmarkSelectBoolFile(b, 1e3, 0.5, true, 2) } func BenchmarkSelectBoolFileNoX1e4Perc50(b *testing.B) { - benchmarkSelectBoolFile(b, 1e4, 0.5, false) + benchmarkSelectBoolFile(b, 1e4, 0.5, false, 0) +} + +func BenchmarkSelectBoolFileV2NoX1e4Perc50(b *testing.B) { + benchmarkSelectBoolFile(b, 1e4, 0.5, false, 2) } func BenchmarkSelectBoolFileX1e4Perc50(b *testing.B) { - benchmarkSelectBoolFile(b, 1e4, 0.5, true) + benchmarkSelectBoolFile(b, 1e4, 0.5, true, 0) +} + +func BenchmarkSelectBoolFileV2X1e4Perc50(b *testing.B) { + benchmarkSelectBoolFile(b, 1e4, 0.5, true, 2) } // ---- func BenchmarkSelectBoolFileNoX1e1Perc5(b *testing.B) { - benchmarkSelectBoolFile(b, 1e1, 0.05, false) + benchmarkSelectBoolFile(b, 1e1, 0.05, false, 0) +} + +func BenchmarkSelectBoolFileV2NoX1e1Perc5(b *testing.B) { + benchmarkSelectBoolFile(b, 1e1, 0.05, false, 2) } func BenchmarkSelectBoolFileX1e1Perc5(b *testing.B) { - benchmarkSelectBoolFile(b, 1e1, 0.05, true) + benchmarkSelectBoolFile(b, 1e1, 0.05, true, 0) +} + +func BenchmarkSelectBoolFileV2X1e1Perc5(b *testing.B) { + benchmarkSelectBoolFile(b, 1e1, 0.05, true, 2) } func BenchmarkSelectBoolFileNoX1e2Perc5(b *testing.B) { - benchmarkSelectBoolFile(b, 1e2, 0.05, false) + benchmarkSelectBoolFile(b, 1e2, 0.05, false, 0) +} + +func BenchmarkSelectBoolFileV2NoX1e2Perc5(b *testing.B) { + benchmarkSelectBoolFile(b, 1e2, 0.05, false, 2) } func BenchmarkSelectBoolFileX1e2Perc5(b *testing.B) { - benchmarkSelectBoolFile(b, 1e2, 0.05, true) + benchmarkSelectBoolFile(b, 1e2, 0.05, true, 0) +} + +func BenchmarkSelectBoolFileV2X1e2Perc5(b *testing.B) { + benchmarkSelectBoolFile(b, 1e2, 0.05, true, 2) } func BenchmarkSelectBoolFileNoX1e3Perc5(b *testing.B) { - benchmarkSelectBoolFile(b, 1e3, 0.05, false) + benchmarkSelectBoolFile(b, 1e3, 0.05, false, 0) +} + +func BenchmarkSelectBoolFileV2NoX1e3Perc5(b *testing.B) { + benchmarkSelectBoolFile(b, 1e3, 0.05, false, 2) } func BenchmarkSelectBoolFileX1e3Perc5(b *testing.B) { - benchmarkSelectBoolFile(b, 1e3, 0.05, true) + benchmarkSelectBoolFile(b, 1e3, 0.05, true, 0) +} + +func BenchmarkSelectBoolFileV2X1e3Perc5(b *testing.B) { + benchmarkSelectBoolFile(b, 1e3, 0.05, true, 2) } func BenchmarkSelectBoolFileNoX1e4Perc5(b *testing.B) { - benchmarkSelectBoolFile(b, 1e4, 0.05, false) + benchmarkSelectBoolFile(b, 1e4, 0.05, false, 0) +} + +func BenchmarkSelectBoolFileV2NoX1e4Perc5(b *testing.B) { + benchmarkSelectBoolFile(b, 1e4, 0.05, false, 2) } func BenchmarkSelectBoolFileX1e4Perc5(b *testing.B) { - benchmarkSelectBoolFile(b, 1e4, 0.05, true) + benchmarkSelectBoolFile(b, 1e4, 0.05, true, 0) +} + +func BenchmarkSelectBoolFileV2X1e4Perc5(b *testing.B) { + benchmarkSelectBoolFile(b, 1e4, 0.05, true, 2) } func TestIndex(t *testing.T) { @@ -1832,14 +2056,14 @@ func benchmarkCrossJoinMem(b *testing.B, size1, size2 int, index bool) { benchmarkCrossJoin(b, db, xjoinCreate, xjoinSel, size1, size2, index, nil) } -func benchmarkCrossJoinFile(b *testing.B, size1, size2 int, index bool) { +func benchmarkCrossJoinFile(b *testing.B, size1, size2 int, index bool, ver int) { dir, err := ioutil.TempDir("", "ql-bench-") if err != nil { b.Fatal(err) } n := runtime.GOMAXPROCS(0) - db, err := OpenFile(filepath.Join(dir, "ql.db"), &Options{CanCreate: true}) + db, err := OpenFile(filepath.Join(dir, "ql.db"), &Options{CanCreate: true, FileFormat: ver}) if err != nil { b.Fatal(err) } @@ -1902,51 +2126,99 @@ func BenchmarkCrossJoinMem1e4X1e3(b *testing.B) { // ---- func BenchmarkCrossJoinFile1e1NoX1e2(b *testing.B) { - benchmarkCrossJoinFile(b, 1e1, 1e2, false) + benchmarkCrossJoinFile(b, 1e1, 1e2, false, 0) +} + +func BenchmarkCrossJoinFileV21e1NoX1e2(b *testing.B) { + benchmarkCrossJoinFile(b, 1e1, 1e2, false, 2) } func BenchmarkCrossJoinFile1e1X1e2(b *testing.B) { - benchmarkCrossJoinFile(b, 1e1, 1e2, true) + benchmarkCrossJoinFile(b, 1e1, 1e2, true, 0) +} + +func BenchmarkCrossJoinFileV21e1X1e2(b *testing.B) { + benchmarkCrossJoinFile(b, 1e1, 1e2, true, 2) } func BenchmarkCrossJoinFile1e2NoX1e3(b *testing.B) { - benchmarkCrossJoinFile(b, 1e2, 1e3, false) + benchmarkCrossJoinFile(b, 1e2, 1e3, false, 0) +} + +func BenchmarkCrossJoinFileV21e2NoX1e3(b *testing.B) { + benchmarkCrossJoinFile(b, 1e2, 1e3, false, 2) } func BenchmarkCrossJoinFile1e2X1e3(b *testing.B) { - benchmarkCrossJoinFile(b, 1e2, 1e3, true) + benchmarkCrossJoinFile(b, 1e2, 1e3, true, 0) +} + +func BenchmarkCrossJoinFileV21e2X1e3(b *testing.B) { + benchmarkCrossJoinFile(b, 1e2, 1e3, true, 2) } func BenchmarkCrossJoinFile1e3NoX1e4(b *testing.B) { - benchmarkCrossJoinFile(b, 1e3, 1e4, false) + benchmarkCrossJoinFile(b, 1e3, 1e4, false, 0) +} + +func BenchmarkCrossJoinFileV21e3NoX1e4(b *testing.B) { + benchmarkCrossJoinFile(b, 1e3, 1e4, false, 2) } func BenchmarkCrossJoinFile1e3X1e4(b *testing.B) { - benchmarkCrossJoinFile(b, 1e3, 1e4, true) + benchmarkCrossJoinFile(b, 1e3, 1e4, true, 0) +} + +func BenchmarkCrossJoinFileV21e3X1e4(b *testing.B) { + benchmarkCrossJoinFile(b, 1e3, 1e4, true, 2) } func BenchmarkCrossJoinFile1e2NoX1e1(b *testing.B) { - benchmarkCrossJoinFile(b, 1e2, 1e1, false) + benchmarkCrossJoinFile(b, 1e2, 1e1, false, 0) +} + +func BenchmarkCrossJoinFileV21e2NoX1e1(b *testing.B) { + benchmarkCrossJoinFile(b, 1e2, 1e1, false, 2) } func BenchmarkCrossJoinFile1e2X1e1(b *testing.B) { - benchmarkCrossJoinFile(b, 1e2, 1e1, true) + benchmarkCrossJoinFile(b, 1e2, 1e1, true, 0) +} + +func BenchmarkCrossJoinFileV21e2X1e1(b *testing.B) { + benchmarkCrossJoinFile(b, 1e2, 1e1, true, 2) } func BenchmarkCrossJoinFile1e3NoX1e2(b *testing.B) { - benchmarkCrossJoinFile(b, 1e3, 1e2, false) + benchmarkCrossJoinFile(b, 1e3, 1e2, false, 0) +} + +func BenchmarkCrossJoinFileV21e3NoX1e2(b *testing.B) { + benchmarkCrossJoinFile(b, 1e3, 1e2, false, 2) } func BenchmarkCrossJoinFile1e3X1e2(b *testing.B) { - benchmarkCrossJoinFile(b, 1e3, 1e2, true) + benchmarkCrossJoinFile(b, 1e3, 1e2, true, 0) +} + +func BenchmarkCrossJoinFileV21e3X1e2(b *testing.B) { + benchmarkCrossJoinFile(b, 1e3, 1e2, true, 2) } func BenchmarkCrossJoinFile1e4NoX1e3(b *testing.B) { - benchmarkCrossJoinFile(b, 1e4, 1e3, false) + benchmarkCrossJoinFile(b, 1e4, 1e3, false, 0) +} + +func BenchmarkCrossJoinFileV21e4NoX1e3(b *testing.B) { + benchmarkCrossJoinFile(b, 1e4, 1e3, false, 2) } func BenchmarkCrossJoinFile1e4X1e3(b *testing.B) { - benchmarkCrossJoinFile(b, 1e4, 1e3, true) + benchmarkCrossJoinFile(b, 1e4, 1e3, true, 0) +} + +func BenchmarkCrossJoinFileV21e4X1e3(b *testing.B) { + benchmarkCrossJoinFile(b, 1e4, 1e3, true, 2) } func TestIssue35(t *testing.T) { @@ -1997,12 +2269,16 @@ func TestIssue35(t *testing.T) { } } -func TestIssue28(t *testing.T) { +func TestIssue28(t *testing.T) { testIssue28(t, "ql") } +func TestIssue28v2(t *testing.T) { testIssue28(t, "ql2") } + +func testIssue28(t *testing.T, drv string) { if testing.Short() { t.Skip("skipping test in short mode.") } RegisterDriver() + RegisterDriver2() dir, err := ioutil.TempDir("", "ql-test-") if err != nil { t.Fatal(err) @@ -2010,7 +2286,7 @@ func TestIssue28(t *testing.T) { defer os.RemoveAll(dir) pth := filepath.Join(dir, "ql.db") - sdb, err := sql.Open("ql", "file://"+pth) + sdb, err := sql.Open(drv, "file://"+pth) if err != nil { t.Fatal(err) } @@ -2034,7 +2310,7 @@ func TestIssue28(t *testing.T) { } pth = filepath.Join(dir, "mem.db") - mdb, err := sql.Open("ql", "memory://"+pth) + mdb, err := sql.Open(drv, "memory://"+pth) if err != nil { t.Fatal(err) } @@ -2065,7 +2341,10 @@ func dumpFields(f []*fld) string { return strings.Join(a, ", ") } -func TestIssue50(t *testing.T) { // https://github.com/cznic/ql/issues/50 +func TestIssue50(t *testing.T) { testIssue50(t, "ql") } +func TestIssue50v2(t *testing.T) { testIssue50(t, "ql2") } + +func testIssue50(t *testing.T, drv string) { // https://github.com/cznic/ql/issues/50 if testing.Short() { t.Skip("skipping test in short mode.") } @@ -2119,7 +2398,7 @@ $9 // create db t.Log("Opening db.") RegisterDriver() - db, err := sql.Open("ql", filepath.Join(dir, dbFileName)) + db, err := sql.Open(drv, filepath.Join(dir, dbFileName)) if err != nil { t.Fatal(err) } @@ -2209,7 +2488,10 @@ $9 t.Log("Done:", scans) } -func TestIssue56(t *testing.T) { +func TestIssue56(t *testing.T) { testIssue56(t, "ql") } +func TestIssue56v2(t *testing.T) { testIssue56(t, "ql2") } + +func testIssue56(t *testing.T, drv string) { if testing.Short() { t.Skip("skipping test in short mode.") } @@ -2225,6 +2507,7 @@ CREATE INDEX IF NOT EXISTS bIdx ON Test (B); ` RegisterDriver() + RegisterDriver2() dir, err := ioutil.TempDir("", "ql-test-") if err != nil { t.Fatal(err) @@ -2232,7 +2515,7 @@ CREATE INDEX IF NOT EXISTS bIdx ON Test (B); defer os.RemoveAll(dir) pth := filepath.Join(dir, "test.db") - db, err := sql.Open("ql", "file://"+pth) + db, err := sql.Open(drv, "file://"+pth) if err != nil { t.Fatal(err) } @@ -2497,6 +2780,29 @@ func TestIssue66File(t *testing.T) { t.Log(err) } +func TestIssue66File2(t *testing.T) { + dir, err := ioutil.TempDir("", "ql-test-") + if err != nil { + t.Fatal(err) + } + + defer os.RemoveAll(dir) + + db, err := OpenFile(filepath.Join(dir, "test.db"), &Options{CanCreate: true, FileFormat: 2}) + if err != nil { + t.Fatal(err) + } + + defer db.Close() + + _, _, err = db.Execute(NewRWCtx(), issue66) + if err == nil { + t.Fatal(err) + } + + t.Log(err) +} + func TestIssue66MemDriver(t *testing.T) { RegisterMemDriver() db, err := sql.Open("ql-mem", "TestIssue66MemDriver-"+fmt.Sprintf("%d", time.Now().UnixNano())) @@ -2546,6 +2852,34 @@ func TestIssue66FileDriver(t *testing.T) { t.Log(err) } +func TestIssue66File2Driver(t *testing.T) { + RegisterDriver2() + dir, err := ioutil.TempDir("", "ql-test-") + if err != nil { + t.Fatal(err) + } + + defer os.RemoveAll(dir) + + db, err := sql.Open("ql2", filepath.Join(dir, "TestIssue66MemDriver")) + if err != nil { + t.Fatal(err) + } + + defer db.Close() + + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + + if _, err = tx.Exec(issue66Src); err == nil { + t.Fatal(err) + } + + t.Log(err) +} + func Example_lIKE() { db, err := OpenMem() if err != nil { @@ -2595,7 +2929,10 @@ func Example_lIKE() { // ---- } -func TestIssue73(t *testing.T) { +func TestIssue73(t *testing.T) { testIssue73(t, "ql") } +func TestIssue73v2(t *testing.T) { testIssue73(t, "ql2") } + +func testIssue73(t *testing.T, drv string) { if testing.Short() { t.Skip("skipping test in short mode.") } @@ -2616,7 +2953,7 @@ func TestIssue73(t *testing.T) { var row *sql.Row var name string - if db, err = sql.Open("ql", pth); err != nil { + if db, err = sql.Open(drv, pth); err != nil { t.Fatal("sql.Open: ", err) } @@ -3386,8 +3723,11 @@ func TestIssue109(t *testing.T) { (issue109{T: t}).test(true) } +func TestIssue142(t *testing.T) { testIssue142(t, "ql") } +func TestIssue142v2(t *testing.T) { testIssue142(t, "ql2") } + // https://github.com/cznic/ql/issues/142 -func TestIssue142(t *testing.T) { +func testIssue142(t *testing.T, drv string) { cwd, err := os.Getwd() if err != nil { t.Fatal(err) @@ -3409,7 +3749,7 @@ func TestIssue142(t *testing.T) { RegisterDriver() for _, nm := range []string{"test.db", "./test.db", "another.db"} { t.Log(nm) - db, err := sql.Open("ql", nm) + db, err := sql.Open(drv, nm) if err != nil { t.Fatal(err) } diff --git a/doc.go b/doc.go index bcf9633..ce012aa 100644 --- a/doc.go +++ b/doc.go @@ -14,6 +14,24 @@ // // Change list // +// 2018-11-04: Back end file format V2 is now released. To use the new format +// for newly created databases set the FileFormat field in *Options passed to +// OpenFile to value 2 or use the driver named "ql2" instead of "ql". +// +// - Both the old and new driver will properly open and use, read and write the +// old (V1) or new file (V2) format of an existing database. +// +// - V1 format has a record size limit of ~64 kB. V2 format record size limit +// is math.MaxInt32. +// +// - V1 format uncommitted transaction size is limited by memory resources. V2 +// format uncommitted transaction is limited by free disk space. +// +// - A direct consequence of the previous is that small transactions perform +// better using V1 format and big transactions perform better using V2 format. +// +// - V2 format uses substantially less memory. +// // 2018-08-02: Release v1.2.0 adds initial support for Go modules. // // 2017-01-10: Release v1.1.0 fixes some bugs and adds a configurable WAL diff --git a/driver.go b/driver.go index 0251831..28022b8 100644 --- a/driver.go +++ b/driver.go @@ -73,10 +73,12 @@ func params(args []driver.Value) []interface{} { } var ( - fileDriver = &sqlDriver{dbs: map[string]*driverDB{}} - fileDriverOnce sync.Once - memDriver = &sqlDriver{isMem: true, dbs: map[string]*driverDB{}} - memDriverOnce sync.Once + file2Driver = &sqlDriver{dbs: map[string]*driverDB{}} + file2DriverOnce sync.Once + fileDriver = &sqlDriver{dbs: map[string]*driverDB{}} + fileDriverOnce sync.Once + memDriver = &sqlDriver{isMem: true, dbs: map[string]*driverDB{}} + memDriverOnce sync.Once ) // RegisterDriver registers a QL database/sql/driver[0] named "ql". The name @@ -96,11 +98,39 @@ var ( // the prefix is stripped before interpreting it as a name of a memory-only, // volatile DB. // +// The ql2 driver can open both the original (V1) files and the new (V2) ones. +// It defaults to V1 on creating a new database. +// // [0]: http://golang.org/pkg/database/sql/driver/ func RegisterDriver() { fileDriverOnce.Do(func() { sql.Register("ql", fileDriver) }) } +// RegisterDriver2 registers a QL database/sql/driver[0] named "ql2". The name +// parameter of +// +// sql.Open("ql2", name) +// +// is interpreted as a path name to a named DB file which will be created if +// not present. The underlying QL database data are persisted on db.Close(). +// RegisterDriver can be safely called multiple times, it'll register the +// driver only once. +// +// The name argument can be optionally prefixed by "file://". In that case the +// prefix is stripped before interpreting it as a file name. +// +// The name argument can be optionally prefixed by "memory://". In that case +// the prefix is stripped before interpreting it as a name of a memory-only, +// volatile DB. +// +// The ql2 driver can open both the original (V1) files and the new (V2) ones. +// It defaults to V2 on creating a new database. +// +// [0]: http://golang.org/pkg/database/sql/driver/ +func RegisterDriver2() { + file2DriverOnce.Do(func() { sql.Register("ql2", file2Driver) }) +} + // RegisterMemDriver registers a QL memory database/sql/driver[0] named // "ql-mem". The name parameter of // @@ -152,7 +182,7 @@ func (d *sqlDriver) lock() func() { // headroom Size of the WAL headroom. See https://github.com/cznic/ql/issues/140. func (d *sqlDriver) Open(name string) (driver.Conn, error) { switch { - case d == fileDriver: + case d == fileDriver || d == file2Driver: if !strings.Contains(name, "://") && !strings.HasPrefix(name, "file") { name = "file://" + name } @@ -191,6 +221,10 @@ func (d *sqlDriver) Open(name string) (driver.Conn, error) { } } + ff := 0 + if d == file2Driver { + ff = 2 + } defer d.lock()() db := d.dbs[name] if db == nil { @@ -200,7 +234,7 @@ func (d *sqlDriver) Open(name string) (driver.Conn, error) { case true: db0, err = OpenMem() default: - db0, err = OpenFile(name, &Options{CanCreate: true, Headroom: headroom}) + db0, err = OpenFile(name, &Options{CanCreate: true, Headroom: headroom, FileFormat: ff}) } if err != nil { return nil, err diff --git a/driver/all_test.go b/driver/all_test.go index db28d1d..131efe6 100644 --- a/driver/all_test.go +++ b/driver/all_test.go @@ -111,6 +111,105 @@ func Example_testFile() { // OK } +func Example_testFile2() { + dir, err := ioutil.TempDir("", "ql-driver-test") + if err != nil { + return + } + + defer func() { + os.RemoveAll(dir) + }() + + db, err := sql.Open("ql2", filepath.Join(dir, "ql2.db")) + if err != nil { + return + } + + defer func() { + if err := db.Close(); err != nil { + return + } + + fmt.Println("OK") + }() + + tx, err := db.Begin() + if err != nil { + return + } + + if _, err := tx.Exec("CREATE TABLE t (Qty int, Name string);"); err != nil { + return + } + + result, err := tx.Exec(` + INSERT INTO t VALUES + ($1, $2), + ($3, $4), + ; + `, + 42, "foo", + 314, "bar", + ) + if err != nil { + return + } + + if err = tx.Commit(); err != nil { + return + } + + id, err := result.LastInsertId() + if err != nil { + return + } + + aff, err := result.RowsAffected() + if err != nil { + return + } + + fmt.Printf("LastInsertId %d, RowsAffected %d\n", id, aff) + + rows, err := db.Query("SELECT * FROM t;") + if err != nil { + return + } + + cols, err := rows.Columns() + if err != nil { + return + } + + fmt.Printf("Columns: %v\n", cols) + + var data struct { + Qty int + Name string + } + + for rows.Next() { + if err = rows.Scan(&data.Qty, &data.Name); err != nil { + rows.Close() + break + } + + fmt.Printf("%+v\n", data) + } + + if err = rows.Err(); err != nil { + return + } + + // Output: + // LastInsertId 2, RowsAffected 2 + // Columns: [Qty Name] + // {Qty:314 Name:bar} + // {Qty:42 Name:foo} + // OK +} + func Example_testMem() { db, err := sql.Open("ql-mem", "mem.db") if err != nil { diff --git a/driver/driver.go b/driver/driver.go index 557c70d..f3532fa 100644 --- a/driver/driver.go +++ b/driver/driver.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. /* -Package driver registers a QL sql/driver named "ql" and a memory driver named "ql-mem". +Package driver registers QL sql/drivers named "ql", "ql2" and a memory driver named "ql-mem". See also [0], [1] and [3]. @@ -28,6 +28,13 @@ A skeleton program using ql/driver. // and/or + // Disk file DB using V2 format + db, err := sql.Open("ql2", "ql.db") + // alternatively + db, err := sql.Open("ql2", "file://ql.db") + + // and/or + // RAM DB mdb, err := sql.Open("ql-mem", "mem.db") // alternatively @@ -57,5 +64,6 @@ import "github.com/cznic/ql" func init() { ql.RegisterDriver() + ql.RegisterDriver2() ql.RegisterMemDriver() } diff --git a/encode2.go b/encode2.go new file mode 100644 index 0000000..b598ac0 --- /dev/null +++ b/encode2.go @@ -0,0 +1,485 @@ +// Copyright 2018 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ql + +import ( + "encoding/binary" + "fmt" + "math" + "math/big" + "time" + + "github.com/cznic/internal/buffer" +) + +const ( + tag2null = iota + + tag2bigInt + tag2bigIntZero + tag2bigRat + tag2bigRatZero + tag2bin + tag2binZero + tag2c128 + tag2c64 + tag2duration + tag2durationZero + tag2f32 + tag2f64 + tag2false + tag2i16 + tag2i16Zero + tag2i32 + tag2i32Zero + tag2i64 + tag2i64Zero + tag2i8 + tag2i8Zero + tag2string + tag2stringZero + tag2time + tag2timeZero + tag2true + tag2u16 + tag2u16Zero + tag2u32 + tag2u32Zero + tag2u64 + tag2u64Zero + tag2u8 + tag2u8Zero +) + +func encode2(data []interface{}) (r buffer.Bytes, error error) { + p := buffer.Get(2*binary.MaxVarintLen64 + 1) + + defer buffer.Put(p) + + b := *p + for _, v := range data { + switch x := v.(type) { + case nil: + r.WriteByte(tag2null) + case bool: + switch x { + case false: + r.WriteByte(tag2false) + case true: + r.WriteByte(tag2true) + } + case complex64: + b[0] = tag2c64 + n := binary.PutUvarint(b[1:], uint64(math.Float32bits(real(x)))) + n += binary.PutUvarint(b[1+n:], uint64(math.Float32bits(imag(x)))) + r.Write(b[:n+1]) + case complex128: + b[0] = tag2c128 + n := binary.PutUvarint(b[1:], math.Float64bits(real(x))) + n += binary.PutUvarint(b[1+n:], math.Float64bits(imag(x))) + r.Write(b[:n+1]) + case float32: + b[0] = tag2f32 + n := binary.PutUvarint(b[1:], uint64(math.Float32bits(x))) + r.Write(b[:n+1]) + case float64: + b[0] = tag2f64 + n := binary.PutUvarint(b[1:], math.Float64bits(x)) + r.Write(b[:n+1]) + case int: + switch { + case x == 0: + r.WriteByte(tag2i64Zero) + default: + b[0] = tag2i64 + n := binary.PutVarint(b[1:], int64(x)) + r.Write(b[:n+1]) + } + case int8: + switch { + case x == 0: + r.WriteByte(tag2i8Zero) + default: + b[0] = tag2i8 + n := binary.PutVarint(b[1:], int64(x)) + r.Write(b[:n+1]) + } + case int16: + switch { + case x == 0: + r.WriteByte(tag2i16Zero) + default: + b[0] = tag2i16 + n := binary.PutVarint(b[1:], int64(x)) + r.Write(b[:n+1]) + } + case int32: + switch { + case x == 0: + r.WriteByte(tag2i32Zero) + default: + b[0] = tag2i32 + n := binary.PutVarint(b[1:], int64(x)) + r.Write(b[:n+1]) + } + case int64: + switch { + case x == 0: + r.WriteByte(tag2i64Zero) + default: + b[0] = tag2i64 + n := binary.PutVarint(b[1:], x) + r.Write(b[:n+1]) + } + case idealInt: + switch { + case x == 0: + r.WriteByte(tag2i64Zero) + default: + b[0] = tag2i64 + n := binary.PutVarint(b[1:], int64(x)) + r.Write(b[:n+1]) + } + case string: + switch { + case x == "": + r.WriteByte(tag2stringZero) + default: + b[0] = tag2string + n := binary.PutUvarint(b[1:], uint64(len(x))) + r.Write(b[:n+1]) + r.WriteString(x) + } + case uint8: + switch { + case x == 0: + r.WriteByte(tag2u8Zero) + default: + b[0] = tag2u8 + n := binary.PutUvarint(b[1:], uint64(x)) + r.Write(b[:n+1]) + } + case uint16: + switch { + case x == 0: + r.WriteByte(tag2u16Zero) + default: + b[0] = tag2u16 + n := binary.PutUvarint(b[1:], uint64(x)) + r.Write(b[:n+1]) + } + case uint32: + switch { + case x == 0: + r.WriteByte(tag2u32Zero) + default: + b[0] = tag2u32 + n := binary.PutUvarint(b[1:], uint64(x)) + r.Write(b[:n+1]) + } + case uint64: + switch { + case x == 0: + r.WriteByte(tag2u64Zero) + default: + b[0] = tag2u64 + n := binary.PutUvarint(b[1:], x) + r.Write(b[:n+1]) + } + case []byte: + switch { + case len(x) == 0: + r.WriteByte(tag2binZero) + default: + b[0] = tag2bin + n := binary.PutUvarint(b[1:], uint64(len(x))) + r.Write(b[:n+1]) + r.Write(x) + } + case *big.Int: + switch { + case x.Sign() == 0: + r.WriteByte(tag2bigIntZero) + default: + b[0] = tag2bigInt + buf, err := x.GobEncode() + if err != nil { + return r, err + } + + n := binary.PutUvarint(b[1:], uint64(len(buf))) + r.Write(b[:n+1]) + r.Write(buf) + } + case *big.Rat: + switch { + case x.Sign() == 0: + r.WriteByte(tag2bigRatZero) + default: + b[0] = tag2bigRat + buf, err := x.GobEncode() + if err != nil { + return r, err + } + + n := binary.PutUvarint(b[1:], uint64(len(buf))) + r.Write(b[:n+1]) + r.Write(buf) + } + case time.Time: + switch { + case x.IsZero(): + r.WriteByte(tag2timeZero) + default: + b[0] = tag2time + buf, err := x.GobEncode() + if err != nil { + return r, err + } + + n := binary.PutUvarint(b[1:], uint64(len(buf))) + r.Write(b[:n+1]) + r.Write(buf) + } + case time.Duration: + switch { + case x == 0: + r.WriteByte(tag2durationZero) + default: + b[0] = tag2duration + n := binary.PutVarint(b[1:], int64(x)) + r.Write(b[:n+1]) + } + default: + return r, fmt.Errorf("encode2: unexpected data %T(%v)", x, x) + } + } + return r, nil +} + +func decode2(dst []interface{}, b []byte) ([]interface{}, error) { + dst = dst[:0] + for len(b) != 0 { + tag := b[0] + b = b[1:] + switch tag { + case tag2null: + dst = append(dst, nil) + case tag2false: + dst = append(dst, false) + case tag2true: + dst = append(dst, true) + case tag2c64: + n, nlen := binary.Uvarint(b) + if nlen <= 0 || n > math.MaxUint32 { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + n2, nlen2 := binary.Uvarint(b[nlen:]) + if nlen2 <= 0 || n2 > math.MaxUint32 { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + dst = append(dst, complex(math.Float32frombits(uint32(n)), math.Float32frombits(uint32(n2)))) + b = b[nlen+nlen2:] + case tag2c128: + n, nlen := binary.Uvarint(b) + if nlen <= 0 { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + n2, nlen2 := binary.Uvarint(b[nlen:]) + if nlen2 <= 0 { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + dst = append(dst, complex(math.Float64frombits(n), math.Float64frombits(n2))) + b = b[nlen+nlen2:] + case tag2f32: + n, nlen := binary.Uvarint(b) + if nlen <= 0 || n > math.MaxUint32 { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + dst = append(dst, math.Float32frombits(uint32(n))) + b = b[nlen:] + case tag2f64: + n, nlen := binary.Uvarint(b) + if nlen <= 0 { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + dst = append(dst, math.Float64frombits(n)) + b = b[nlen:] + case tag2i8Zero: + dst = append(dst, int8(0)) + case tag2i8: + n, nlen := binary.Varint(b) + if nlen <= 0 || n < math.MinInt16 || n > math.MaxInt8 { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + dst = append(dst, int8(n)) + b = b[nlen:] + case tag2i16Zero: + dst = append(dst, int16(0)) + case tag2i16: + n, nlen := binary.Varint(b) + if nlen <= 0 || n < math.MinInt16 || n > math.MaxInt16 { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + dst = append(dst, int16(n)) + b = b[nlen:] + case tag2i32Zero: + dst = append(dst, int32(0)) + case tag2i32: + n, nlen := binary.Varint(b) + if nlen <= 0 || n < math.MinInt32 || n > math.MaxInt32 { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + dst = append(dst, int32(n)) + b = b[nlen:] + case tag2i64Zero: + dst = append(dst, int64(0)) + case tag2i64: + n, nlen := binary.Varint(b) + if nlen <= 0 { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + dst = append(dst, n) + b = b[nlen:] + case tag2stringZero: + dst = append(dst, "") + case tag2string: + n, nlen := binary.Uvarint(b) + if nlen <= 0 { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + b = b[nlen:] + if uint64(len(b)) < n { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + dst = append(dst, string(b[:n])) + b = b[n:] + case tag2u8Zero: + dst = append(dst, byte(0)) + case tag2u8: + n, nlen := binary.Uvarint(b) + if nlen <= 0 || n > math.MaxUint8 { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + dst = append(dst, byte(n)) + b = b[nlen:] + case tag2u16Zero: + dst = append(dst, uint16(0)) + case tag2u16: + n, nlen := binary.Uvarint(b) + if nlen <= 0 || n > math.MaxUint16 { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + dst = append(dst, uint16(n)) + b = b[nlen:] + case tag2u32Zero: + dst = append(dst, uint32(0)) + case tag2u32: + n, nlen := binary.Uvarint(b) + if nlen <= 0 || n > math.MaxUint32 { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + dst = append(dst, uint32(n)) + b = b[nlen:] + case tag2u64Zero: + dst = append(dst, uint64(0)) + case tag2u64: + n, nlen := binary.Uvarint(b) + if nlen <= 0 { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + dst = append(dst, n) + b = b[nlen:] + case tag2binZero: + dst = append(dst, []byte(nil)) + case tag2bin: + n, nlen := binary.Uvarint(b) + if nlen <= 0 { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + b = b[nlen:] + dst = append(dst, append([]byte(nil), b[:n]...)) + b = b[n:] + case tag2bigIntZero: + dst = append(dst, big.NewInt(0)) + case tag2bigInt: + n, nlen := binary.Uvarint(b) + if nlen <= 0 { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + b = b[nlen:] + var z big.Int + if err := z.GobDecode(b[:n]); err != nil { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + dst = append(dst, &z) + b = b[n:] + case tag2bigRatZero: + dst = append(dst, &big.Rat{}) + case tag2bigRat: + n, nlen := binary.Uvarint(b) + if nlen <= 0 { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + b = b[nlen:] + var q big.Rat + if err := q.GobDecode(b[:n]); err != nil { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + dst = append(dst, &q) + b = b[n:] + case tag2timeZero: + dst = append(dst, time.Time{}) + case tag2time: + n, nlen := binary.Uvarint(b) + if nlen <= 0 { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + b = b[nlen:] + var t time.Time + if err := t.GobDecode(b[:n]); err != nil { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + dst = append(dst, t) + b = b[n:] + case tag2durationZero: + dst = append(dst, time.Duration(0)) + case tag2duration: + n, nlen := binary.Varint(b) + if nlen <= 0 { + return nil, fmt.Errorf("decode2: corrupted DB") + } + + dst = append(dst, time.Duration(n)) + b = b[nlen:] + default: + return nil, fmt.Errorf("decode2: unexpected tag %v", tag) + } + } + return dst, nil +} diff --git a/etc.go b/etc.go index a4ea697..381f7f4 100644 --- a/etc.go +++ b/etc.go @@ -2362,6 +2362,36 @@ func collate1(a, b interface{}) int { default: panic("internal error 023") } + case int: + switch y := b.(type) { + case nil: + return 1 + case int64: + if int64(x) < y { + return -1 + } + + if int64(x) == y { + return 0 + } + + return 1 + case idealInt: + { + x, y := int64(x), int64(y) + if x < y { + return -1 + } + + if x == y { + return 0 + } + + return 1 + } + default: + panic("internal error 024") + } case int64: switch y := b.(type) { case nil: diff --git a/file.go b/file.go index 95d74c0..05f45fb 100644 --- a/file.go +++ b/file.go @@ -9,6 +9,7 @@ package ql import ( + "bytes" "crypto/sha1" "fmt" "io" @@ -82,6 +83,13 @@ func OpenFile(name string, opt *Options) (db *DB, err error) { return nil, err } + switch opt.FileFormat { + case 0, 1, 2: + // ok + default: + return nil, fmt.Errorf("OpenFile: invalid option.FileFormat value: %v", opt.FileFormat) + } + f, err = os.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666) if err != nil { return nil, err @@ -89,7 +97,35 @@ func OpenFile(name string, opt *Options) (db *DB, err error) { } } - fi, err := newFileFromOSFile(f, opt.Headroom) // always ACID + nfo, err := f.Stat() + if err != nil { + return nil, err + } + + var new bool + switch nfo.Size() { + case 0: + new = true + if opt.FileFormat == 2 { + return openFile2(name, f, opt, new) + } + default: + b := make([]byte, 16) + if _, err := f.ReadAt(b, 0); err != nil { + return nil, err + } + + switch { + case bytes.Equal(b[:len(magic)], []byte(magic)): + // ok + case bytes.Equal(b[:len(magic2)], []byte(magic2)): + return openFile2(name, f, opt, new) + default: + return nil, fmt.Errorf("OpenFile: unrecognized file format") + } + } + + fi, err := newFileFromOSFile(f, opt.Headroom, new) if err != nil { return } @@ -141,12 +177,22 @@ func OpenFile(name string, opt *Options) (db *DB, err error) { // // RemoveEmptyWAL controls whether empty WAL files should be deleted on // clean exit. +// +// FileVersion +// +// Select DB backend format when creating a new DB file. +// +// Supported values +// +// 0, 1 The original file format (version 1) +// 2 File format version 2 type Options struct { CanCreate bool OSFile lldb.OSFile TempFile func(dir, prefix string) (f lldb.OSFile, err error) Headroom int64 RemoveEmptyWAL bool + FileFormat int } type fileBTreeIterator struct { @@ -305,9 +351,7 @@ type fileTemp struct { t *lldb.BTree } -func (t *fileTemp) BeginTransaction() error { - return nil -} +func (t *fileTemp) BeginTransaction() error { return nil } func (t *fileTemp) Get(k []interface{}) (v []interface{}, err error) { if err = expand(k); err != nil { @@ -404,7 +448,7 @@ type file struct { removeEmptyWAL bool // Whether empty WAL files should be removed on close } -func newFileFromOSFile(f lldb.OSFile, headroom int64) (fi *file, err error) { +func newFileFromOSFile(f lldb.OSFile, headroom int64, new bool) (fi *file, err error) { nm := lockName(f.Name()) lck, err := lock.Lock(nm) if err != nil { @@ -455,13 +499,8 @@ func newFileFromOSFile(f lldb.OSFile, headroom int64) (fi *file, err error) { closew = st.Size() == 0 } - info, err := f.Stat() - if err != nil { - return nil, err - } - - switch sz := info.Size(); { - case sz == 0: + switch { + case new: b := make([]byte, 16) copy(b, []byte(magic)) if _, err := f.Write(b); err != nil { @@ -513,15 +552,6 @@ func newFileFromOSFile(f lldb.OSFile, headroom int64) (fi *file, err error) { close, closew = false, false return s, s.Commit() default: - b := make([]byte, 16) - if _, err := f.Read(b); err != nil { - return nil, err - } - - if string(b[:len(magic)]) != magic { - return nil, fmt.Errorf("(file-002) unknown file format") - } - filer := lldb.Filer(lldb.NewOSFiler(f)) filer = lldb.NewInnerFiler(filer, 16) if filer, err = lldb.NewACIDFiler(filer, w, lldb.MinWAL(headroom)); err != nil { diff --git a/file2.go b/file2.go new file mode 100644 index 0000000..2c63bb6 --- /dev/null +++ b/file2.go @@ -0,0 +1,1108 @@ +// Copyright 2018 The ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ql + +import ( + "encoding/binary" + "fmt" + "io" + "io/ioutil" + "math" + "os" + + "github.com/cznic/db" + cfile "github.com/cznic/file" + "github.com/cznic/internal/buffer" + "github.com/cznic/lldb" + "github.com/cznic/ql/vendored/github.com/camlistore/go4/lock" +) + +var ( + _ btreeIndex = (*btreeIndex2)(nil) + _ btreeIterator = (*btreeIterator2)(nil) + _ db.Storage = (*dbStorage)(nil) + _ indexIterator = (*indexIterator2)(nil) + _ storage = (*storage2)(nil) + _ temp = (*temp2)(nil) + + zeroInt64 = []interface{}{int64(0)} +) + +func init() { + if al := cfile.AllocAlign; al != 16 || al <= binary.MaxVarintLen64 { + panic("internal error") + } +} + +const ( + // These can be tuned. + btree2ND = 512 + btree2NX = 1024 + + // Do not touch after release + magic2 = "\x61\xdbql" + szBuf = 32 + szKey = 2 * cfile.AllocAlign + szVal = 2 * cfile.AllocAlign + wal2PageLog = 16 +) + +func handle2off(h int64) int64 { return (h-1)<<4 + cfile.LowestAllocationOffset } +func off2handle(off int64) int64 { return (off-cfile.LowestAllocationOffset)>>4 + 1 } +func roundup2(n int64) int64 { return (n + cfile.AllocAlign - 1) &^ (cfile.AllocAlign - 1) } + +func read(f cfile.File, b []byte, off int64) (int, error) { + n, err := f.ReadAt(b, off) + if n == len(b) { + err = nil + } + return n, err +} + +func openFile2(name string, f cfile.File, opt *Options, new bool) (db *DB, err error) { + tempFile := opt.TempFile + if tempFile == nil { + tempFile = func(dir, prefix string) (f lldb.OSFile, err error) { return ioutil.TempFile(dir, prefix) } + } + + s, err := newStorage2(f, name, opt.Headroom, tempFile, new) + if err != nil { + if new { + f.Close() + os.Remove(name) + } + return + } + + s.removeEmptyWAL = opt.RemoveEmptyWAL + return newDB(s) +} + +type storage2 struct { + db *db.DB + dbs dbStorage + id int64 + lck io.Closer + name string + tempFile func(dir, prefix string) (f lldb.OSFile, err error) + walName string + + varIntBuf [binary.MaxVarintLen64]byte + + idDirty bool + removeEmptyWAL bool // Whether empty WAL files should be removed on close +} + +func newStorage2(f cfile.File, name string, headroom int64, tempFile func(dir, prefix string) (f lldb.OSFile, err error), new bool) (s *storage2, err error) { + if headroom != 0 { + return nil, fmt.Errorf("v2 back end does not yet support headroom") + } + + var ( + f1 cfile.File + lck io.Closer + w *os.File + ) + + defer func() { + if lck != nil { + lck.Close() + } + if w != nil { + w.Close() + } + if f1 != nil { + f1.Close() + } + }() + + if lck, err = lock.Lock(lockName(name)); err != nil { + return nil, err + } + + if x, ok := f.(*os.File); ok { + f1, err = cfile.Map(x) + if err != nil { + return nil, err + } + + f = f1 + } + + wn := walName(name) + if w, err = os.OpenFile(wn, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666); err != nil { + if !os.IsExist(err) { + return nil, err + } + + if w, err = os.OpenFile(wn, os.O_EXCL|os.O_RDWR, 0666); err != nil { + return nil, err + } + } + + w1, err := cfile.Map(w) + if err != nil { + return nil, err + } + + wal, err := cfile.NewWAL(f, w1, 0, wal2PageLog) + if err != nil { + return nil, err + } + + a, err := cfile.NewAllocator(wal) + if err != nil { + return nil, err + } + + a.SetAutoFlush(false) + s = &storage2{ + dbs: newDBStorage(a, f, wal, tempFile), + lck: lck, + name: name, + tempFile: tempFile, + walName: wn, + } + d, err := db.NewDB(&s.dbs) + if err != nil { + return nil, err + } + s.db = d + + switch { + case new: + if err := s.BeginTransaction(); err != nil { + return nil, err + } + + b := make([]byte, 16) + copy(b, []byte(magic2)) + if _, err := s.dbs.WriteAt(b, 0); err != nil { + return nil, err + } + + off, err := s.dbs.Calloc(16) + if err != nil { + return nil, err + } + + if h := off2handle(off); h != 1 { // root + return nil, fmt.Errorf("unexpected root handle %#x", h) + } + + if off, err = s.dbs.Calloc(8); err != nil { + return nil, err + } + + if h := off2handle(off); h != 2 { // id + return nil, fmt.Errorf("unexpected ID handle %#x", h) + } + + if err := s.Commit(); err != nil { + return nil, err + } + default: + b := s.varIntBuf[:] + n, err := read(&s.dbs, b, handle2off(2)) + if err != nil { + return nil, err + } + + s.id, n = binary.Varint(b[:n]) + if n <= 0 { + return nil, fmt.Errorf("%T.newStorage2: corrupted DB", s) + } + } + lck = nil + w = nil + f1 = nil + return s, nil +} + +func (s *storage2) Acid() bool { return s.dbs.wal != nil } + +func (s *storage2) BeginTransaction() (err error) { return s.dbs.beginTransaction() } + +func (s *storage2) Close() (err error) { + if err := s.dbs.wal.F.Close(); err != nil { + return err + } + + if err := s.dbs.wal.W.Close(); err != nil { + return err + } + + if s.removeEmptyWAL && s.dbs.txLevel == 0 { + if err := os.Remove(s.walName); err != nil { + return err + } + } + + return s.lck.Close() +} + +func (s *storage2) Commit() (err error) { + if s.dbs.txLevel == 0 { + return fmt.Errorf("%T.commit: not in transaction", s) + } + + if s.idDirty { + b := s.varIntBuf[:] + n := binary.PutVarint(b, s.id) + if _, err := s.dbs.WriteAt(b[:n], handle2off(2)); err != nil { + return err + } + + s.idDirty = false + } + return s.dbs.commit() +} + +func (s *storage2) Create(data ...interface{}) (h int64, err error) { + if s.dbs.txLevel == 0 { + return 0, fmt.Errorf("%T.Create: not in transaction", s) + } + + off, err := s.dbs.create(data...) + if err != nil { + return 0, err + } + + return off2handle(off), nil +} + +func (s *storage2) CreateIndex(unique bool) (handle int64, x btreeIndex, err error) { + bt, err := s.db.NewBTree(btree2ND, btree2NX, szKey, szVal) + if err != nil { + return 0, nil, err + } + + return off2handle(bt.Off), &btreeIndex2{ + bt: btree2{ + bt: bt, + dbs: &s.dbs, + k: 1, + }, + unique: unique, + }, nil +} + +func (s *storage2) CreateTemp(asc bool) (t temp, err error) { + var ( + f lldb.OSFile + f1 cfile.File + ) + + defer func() { + if f != nil { + f.Close() + os.Remove(f.Name()) + } + if f1 != nil { + f1.Close() + } + }() + + defer func() { + if f1 != nil { + f1.Close() + } + }() + + if f, err = s.tempFile("", ""); err != nil { + return nil, err + } + + fn := f.Name() + f1 = f + f = nil + if x, ok := f1.(*os.File); ok { + if f1, err = cfile.Map(x); err != nil { + return nil, err + } + } + + a, err := cfile.NewAllocator(f1) + if err != nil { + return nil, err + } + + a.SetAutoFlush(false) + r := &temp2{ + dbs: newDBStorage(a, f1, nil, nil), + name: fn, + } + + d, err := db.NewDB(&r.dbs) + if err != nil { + return nil, err + } + + bt, err := d.NewBTree(btree2ND, btree2NX, szKey, szVal) + if err != nil { + return nil, err + } + + k := 1 + if !asc { + k = -1 + } + r.bt = btree2{ + bt: bt, + dbs: &r.dbs, + k: k, + } + f1 = nil + return r, nil +} + +func (s *storage2) Delete(h int64, blobCols ...*col) (err error) { + if s.dbs.txLevel == 0 { + return fmt.Errorf("%T.Delete: not in transaction", s) + } + + b := s.varIntBuf[:] + off := handle2off(h) + if _, err := read(&s.dbs, b, off); err != nil { + return err + } + + sz, n := binary.Varint(b) + if n <= 0 || sz > math.MaxInt32 { + return fmt.Errorf("%T.Delete: corrupted DB", s) + } + + if sz < 0 { // redirect + if err := s.dbs.Free(-sz); err != nil { + return err + } + } + return s.dbs.Free(off) +} + +func (s *storage2) ID() (id int64, err error) { + if s.dbs.txLevel == 0 { + return 0, fmt.Errorf("%T.ID(): not in transaction", s) + } + + s.id++ + s.idDirty = true + return s.id, nil +} + +func (s *storage2) Name() string { return s.name } + +func (s *storage2) OpenIndex(unique bool, handle int64) (btreeIndex, error) { + off := handle2off(handle) + bt, err := s.db.OpenBTree(off) + if err != nil { + return nil, err + } + + return &btreeIndex2{ + bt: btree2{ + bt: bt, + dbs: &s.dbs, + k: 1, + }, + unique: unique, + }, nil +} + +func (s *storage2) Read(dst []interface{}, h int64, cols ...*col) (data []interface{}, err error) { + if data, err = s.dbs.read(s.dbs.buf[:], dst, handle2off(h)); err != nil { + return nil, err + } + + if cols != nil { + for n, dn := len(cols)+2, len(data); dn < n; dn++ { + data = append(data, nil) + } + } + return data, nil +} + +func (s *storage2) ResetID() (err error) { + if s.dbs.txLevel == 0 { + return fmt.Errorf("%T.ResetID: not in transaction", s) + } + + s.id = 0 + s.idDirty = true + return nil +} + +func (s *storage2) Rollback() (err error) { + if s.dbs.txLevel == 0 { + return fmt.Errorf("%T.Rollback: not in transaction", s) + } + + return s.dbs.pop() +} + +func (s *storage2) Update(h int64, data ...interface{}) (err error) { + if s.dbs.txLevel == 0 { + return fmt.Errorf("%T.Update: not in transaction", s) + } + + off := handle2off(h) + b := s.varIntBuf[:] + if _, err = read(&s.dbs, b, off); err != nil { + return err + } + + sz, n := binary.Varint(b) + if n <= 0 || sz > math.MaxInt32 { + return fmt.Errorf("%T.Update: corrupted DB", s) + } + + if sz < 0 { // redirect + if err := s.dbs.Free(-sz); err != nil { + return err + } + + sz = 0 + } + + have := roundup2(int64(n) + sz) + buf, err := encode2(data) + if err != nil { + return err + } + + if buf.Len() > math.MaxInt32 { + return fmt.Errorf("%T.Update: data bigger than 2 GB", s) + } + + n = binary.PutVarint(b, int64(buf.Len())) + need := int64(n) + int64(buf.Len()) + if have < need { + have, err := s.dbs.UsableSize(off) + if err != nil { + return err + } + + if have < need { + off2, err := s.dbs.Alloc(int64(n) + int64(buf.Len())) + if err != nil { + return err + } + + if _, err = s.dbs.WriteAt(b[:n], off2); err != nil { + return err + } + + if _, err = s.dbs.WriteAt(buf.Bytes(), off2+int64(n)); err != nil { + return err + } + + n = binary.PutVarint(b, -off2) + _, err = s.dbs.WriteAt(b[:n], off) + return err + } + } + + // have >= need + if _, err := s.dbs.WriteAt(b[:n], off); err != nil { + return err + } + + _, err = s.dbs.WriteAt(buf.Bytes(), off+int64(n)) + return err +} + +func (s *storage2) UpdateRow(h int64, blobCols []*col, data ...interface{}) (err error) { + if s.dbs.txLevel == 0 { + return fmt.Errorf("%T.UpdateRow: not in transaction", s) + } + + return s.Update(h, data...) +} + +func (s *storage2) Verify() (allocs int64, err error) { + var opt cfile.VerifyOptions + if err := s.dbs.Verify(&opt); err != nil { + return 0, err + } + + return opt.Allocs, nil +} + +type btreeIndex2 struct { + bt btree2 + + unique bool +} + +func (x *btreeIndex2) Clear() (err error) { + return x.bt.bt.Clear(x.free) +} + +func (x *btreeIndex2) Create(indexedValues []interface{}, h int64) (err error) { + switch { + case !x.unique: + k := append(indexedValues, h) + return x.bt.set(k, zeroInt64) + case isIndexNull(indexedValues): // unique, NULL + k := []interface{}{nil, h} + return x.bt.set(k, zeroInt64) + default: // unique, non NULL + k := append(indexedValues, int64(0)) + _, ok, err := x.bt.bt.Get(x.bt.cmp(k)) + if err != nil { + return err + } + + if ok { + return fmt.Errorf("cannot insert into unique index: duplicate value(s): %v", indexedValues) + } + + return x.bt.set(k, []interface{}{h}) + } +} + +func (x *btreeIndex2) Delete(indexedValues []interface{}, h int64) (err error) { + var k []interface{} + switch { + case !x.unique: + k = append(indexedValues, h) + case isIndexNull(indexedValues): // unique, NULL + k = []interface{}{nil, h} + default: // unique, non NULL + k = append(indexedValues, int64(0)) + } + _, err = x.bt.bt.Delete(x.bt.cmp(k), x.free) + return err +} + +func (x *btreeIndex2) free(koff, voff int64) error { + if err := x.free1(koff); err != nil { + return err + } + + return x.free1(voff) +} + +func (x *btreeIndex2) free1(off int64) error { + b := x.bt.dbs.varIntBuf[:] + _, err := read(x.bt.dbs, b, off) + if err != nil { + return err + } + + sz, n := binary.Varint(b) + if n <= 0 || sz > math.MaxInt32 { + return fmt.Errorf("%T.free: corrupted DB", x) + } + + if sz < 0 { + if err = x.bt.dbs.Free(-sz); err != nil { + return err + } + + n = binary.PutVarint(b, 0) + _, err = x.bt.dbs.WriteAt(b[:n], off) + } + return nil +} + +func (x *btreeIndex2) Drop() (err error) { + return x.bt.bt.Remove(x.free) +} + +func (x *btreeIndex2) Seek(indexedValues []interface{}) (iter indexIterator, hit bool, err error) { + k := append(indexedValues, int64(0)) + c, hit, err := x.bt.bt.Seek(x.bt.cmp(k)) + if err != nil { + return nil, false, err + } + + return &indexIterator2{ + bt: &x.bt, + c: c, + unique: x.unique, + }, hit, nil +} + +func (x *btreeIndex2) SeekFirst() (iter indexIterator, err error) { + c, err := x.bt.bt.SeekFirst() + if err != nil { + return nil, err + } + + return &indexIterator2{ + bt: &x.bt, + c: c, + unique: x.unique, + }, nil +} + +func (x *btreeIndex2) SeekLast() (iter indexIterator, err error) { + c, err := x.bt.bt.SeekLast() + if err != nil { + return nil, err + } + + return &indexIterator2{ + bt: &x.bt, + c: c, + unique: x.unique, + }, nil +} + +type indexIterator2 struct { + bt *btree2 + c *db.BTreeCursor + + unique bool +} + +func (b *indexIterator2) nextPrev(mv func() bool) (k []interface{}, h int64, err error) { + if mv() { + if k, err = b.bt.dbs.read(b.bt.dbs.kbuf[:], nil, b.c.K); err != nil { + return nil, 0, err + } + + v, err := b.bt.dbs.read(b.bt.dbs.vbuf[:], nil, b.c.V) + if err != nil { + return nil, 0, err + } + + if len(v) != 1 { + return nil, 0, fmt.Errorf("%T.Next(): corrupted DB", b) + } + + var ok bool + if h, ok = v[0].(int64); !ok { + return nil, 0, fmt.Errorf("%T.Next(): corrupted DB", b) + } + + if b.unique { + if isIndexNull(k[:len(k)-1]) { + if h, ok = k[len(k)-1].(int64); !ok { + return nil, 0, fmt.Errorf("%T.Next(): corrupted DB", b) + } + + return nil, h, nil + } + + return k[:len(k)-1], h, nil + } + + if h, ok = k[len(k)-1].(int64); !ok { + return nil, 0, fmt.Errorf("%T.Next(): corrupted DB", b) + } + + return k[:len(k)-1], h, nil + } + + return nil, 0, io.EOF +} + +func (b *indexIterator2) Next() (k []interface{}, h int64, err error) { return b.nextPrev(b.c.Next) } + +func (b *indexIterator2) Prev() (k []interface{}, h int64, err error) { return b.nextPrev(b.c.Prev) } + +type temp2 struct { + bt btree2 + dbs dbStorage + name string +} + +func (t *temp2) BeginTransaction() error { return nil } + +func (t *temp2) Create(data ...interface{}) (h int64, err error) { + off, err := t.dbs.create(data...) + if err != nil { + return 0, err + } + + return off2handle(off), nil +} + +func (t *temp2) Drop() (err error) { + if err := t.dbs.File.Close(); err != nil { + return err + } + + return os.Remove(t.name) +} + +func (t *temp2) Get(k []interface{}) (v []interface{}, err error) { + off, ok, err := t.bt.bt.Get(t.bt.cmp(k)) + if err != nil { + return nil, err + } + + if !ok { + return nil, nil + } + + return t.dbs.read(t.bt.dbs.vbuf[:], nil, off) +} + +func (t *temp2) Read(dst []interface{}, h int64, cols ...*col) (data []interface{}, err error) { + if data, err = t.dbs.read(t.dbs.buf[:], dst, handle2off(h)); err != nil { + return nil, err + } + + if cols != nil { + for n, dn := len(cols)+2, len(data); dn < n; dn++ { + data = append(data, nil) + } + } + return data, nil +} + +func (t *temp2) SeekFirst() (e btreeIterator, err error) { + it, err := t.bt.bt.SeekFirst() + if err != nil { + return nil, err + } + + return &btreeIterator2{ + it: it, + t: t, + }, nil +} + +func (t *temp2) Set(k, v []interface{}) (err error) { + return t.bt.set(k, v) +} + +type btree2 struct { + bt *db.BTree + dbs *dbStorage + k int +} + +func (t *btree2) set(k []interface{}, v []interface{}) (err error) { + ek, err := encode2(k) + + defer ek.Close() + + if err != nil { + return err + } + + ev, err := encode2(v) + + defer ev.Close() + + if err != nil { + return err + } + + var free bool + koff, voff, err := t.bt.Set(t.cmp(k), t.free(&free)) + if err != nil { + return err + } + + if !free { + if err := t.setK(koff, ek); err != nil { + return err + } + } + + switch { + case free: + return t.replace(voff, ev) + default: + return t.setV(voff, ev) + } +} + +func (t *btree2) replace(off int64, buf buffer.Bytes) error { + b := t.dbs.varIntBuf[:] + if _, err := read(t.dbs, b, off); err != nil { + return err + } + + sz, n := binary.Varint(b) + if n <= 0 || sz > math.MaxInt32 { + return fmt.Errorf("%T.replace: corrupted DB", t) + } + + if sz < 0 { // Redirected. + if err := t.dbs.Free(-sz); err != nil { + return err + } + } + + return t.setV(off, buf) +} + +func (t *btree2) setK(off int64, buf buffer.Bytes) error { return t.set1(t.dbs.kbuf[:], off, buf) } +func (t *btree2) setV(off int64, buf buffer.Bytes) error { return t.set1(t.dbs.vbuf[:], off, buf) } + +func (t *btree2) set1(b []byte, off int64, buf buffer.Bytes) error { + if buf.Len() > math.MaxInt32 { + return fmt.Errorf("%T.set1: data bigger than 2 GB", t) + } + + n := binary.PutVarint(b, int64(buf.Len())) + if n+buf.Len() <= len(b) { + n += copy(b[n:], buf.Bytes()) + _, err := t.dbs.WriteAt(b[:n], off) + return err + } + + // Redirect. + off2, err := t.dbs.Alloc(int64(n) + int64(buf.Len())) + if err != nil { + return err + } + + if _, err = t.dbs.WriteAt(b[:n], off2); err != nil { + return err + } + + if _, err = t.dbs.WriteAt(buf.Bytes(), off2+int64(n)); err != nil { + return err + } + + n = binary.PutVarint(b, -off2) + _, err = t.dbs.WriteAt(b[:n], off) + return err +} + +func (t *btree2) free(free *bool) func(off int64) error { + return func(off int64) error { + *free = true + return nil + } +} + +func (t *btree2) cmp(k []interface{}) func(koff int64) (int, error) { + return func(koff int64) (int, error) { + k2, err := t.dbs.read(t.dbs.kbuf[:], nil, koff) + if err != nil { + return 0, err + } + + return t.k * collate(k, k2), nil + } +} + +type btreeIterator2 struct { + it *db.BTreeCursor + t *temp2 +} + +func (b *btreeIterator2) Next() (k, v []interface{}, err error) { + if !b.it.Next() { + err := b.it.Err() + if err == nil { + err = io.EOF + } + return nil, nil, err + } + + if k, err = b.t.dbs.read(b.t.dbs.kbuf[:], nil, b.it.K); err != nil { + return nil, nil, err + } + + if v, err = b.t.dbs.read(b.t.dbs.vbuf[:], nil, b.it.V); err != nil { + return nil, nil, err + } + + return k, v, nil +} + +type walStack struct { + f cfile.File + wal *cfile.WAL + walName string +} + +type dbStorage struct { + *cfile.Allocator + cfile.File + stack []walStack + tempFile func(dir, prefix string) (f lldb.OSFile, err error) + txLevel int + wal *cfile.WAL + walName string + + buf [szBuf]byte + kbuf [szKey]byte + varIntBuf [binary.MaxVarintLen64]byte + vbuf [szVal]byte +} + +func newDBStorage(a *cfile.Allocator, f cfile.File, wal *cfile.WAL, tempFile func(dir, prefix string) (f lldb.OSFile, err error)) dbStorage { + return dbStorage{ + Allocator: a, + File: f, + tempFile: tempFile, + wal: wal, + } +} + +func (s *dbStorage) Close() error { return s.File.Close() } + +func (s *dbStorage) Root() (int64, error) { return -1, fmt.Errorf("not implemented") } + +func (s *dbStorage) SetRoot(root int64) error { return fmt.Errorf("not implemented") } + +func (s *dbStorage) beginTransaction() error { + if err := s.Flush(); err != nil { + return err + } + + s.stack = append(s.stack, walStack{ + f: s.File, + wal: s.wal, + walName: s.walName, + }) + s.txLevel++ + if s.txLevel == 1 { + s.File = s.wal + return s.SetFile(s.File) + } + + w, err := s.tempFile("", "") + if err != nil { + return err + } + + w1 := cfile.File(w) + if x, ok := w.(*os.File); ok { + if w1, err = cfile.Map(x); err != nil { + os.Remove(w.Name()) + return err + } + } + + wal, err := cfile.NewWAL(s.wal, w1, 0, wal2PageLog) + if err != nil { + w.Close() + os.Remove(w.Name()) + return err + } + + if err := s.SetFile(wal); err != nil { + w.Close() + os.Remove(w.Name()) + return err + } + + s.wal = wal + s.walName = w.Name() + s.File = s.wal + return nil +} + +func (s *dbStorage) commit() error { + if err := s.Flush(); err != nil { + return err + } + + s.wal.DoSync = s.txLevel == 1 + if err := s.wal.Commit(); err != nil { + return err + } + + return s.pop() +} + +func (s *dbStorage) pop() error { + if s.txLevel > 1 { + if err := s.wal.W.Close(); err != nil { + return err + } + + if err := os.Remove(s.walName); err != nil { + return err + } + } + n := len(s.stack) + x := s.stack[n-1] + s.stack = s.stack[:n-1] + s.File = x.f + s.wal = x.wal + s.walName = x.walName + s.txLevel-- + return s.SetFile(s.File) +} + +func (s *dbStorage) read(b []byte, dst []interface{}, off int64) ([]interface{}, error) { + n, err := read(s, b, off) + if err != nil { + if n < cfile.AllocAlign { + return nil, err + } + + b = b[:n] + } + + sz, n := binary.Varint(b) + if n <= 0 || sz > math.MaxInt32 { + return nil, fmt.Errorf("%T.read: corrupted DB", s) + } + + if sz < 0 { + off = -sz + if _, err := read(s, b, off); err != nil { + return nil, err + } + + sz, n = binary.Varint(b) + if n <= 0 || sz > math.MaxInt32 { + return nil, fmt.Errorf("%T.read: corrupted DB", s) + } + + if sz < 0 { + return nil, fmt.Errorf("%T.read: corrupted DB", s) + } + + } + + if sz <= int64(len(b)-n) { + return decode2(dst, b[n:n+int(sz)]) + } + + p := buffer.Get(int(sz)) + + defer buffer.Put(p) + + c := *p + m := copy(c, b[n:]) + if _, err := read(s, c[m:], off+int64(len(b))); err != nil { + return nil, err + } + + return decode2(dst, c) +} + +func (s *dbStorage) create(data ...interface{}) (off int64, err error) { + buf, err := encode2(data) + if buf.Len() > math.MaxInt32 { + return 0, fmt.Errorf("%T.create: data bigger than 2 GB", s) + } + + defer buf.Close() + + if err != nil { + return 0, err + } + + b := s.varIntBuf[:] + n := binary.PutVarint(b, int64(buf.Len())) + if off, err = s.Alloc(int64(n) + int64(buf.Len())); err != nil { + return 0, err + } + + if _, err := s.WriteAt(b[:n], off); err != nil { + return 0, err + } + + if _, err := s.WriteAt(buf.Bytes(), off+int64(n)); err != nil { + return 0, err + } + + return off, nil +} diff --git a/file2_test.go b/file2_test.go new file mode 100644 index 0000000..96e6ff8 --- /dev/null +++ b/file2_test.go @@ -0,0 +1,264 @@ +// Copyright (c) 2018 ql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ql + +import ( + "bytes" + "database/sql" + "fmt" + "io/ioutil" + "os" + "strings" + "testing" +) + +func TestWALRemoval2(t *testing.T) { + const tempDBName = "./_test_was_removal.db" + wName := walName(tempDBName) + defer os.Remove(tempDBName) + defer os.Remove(wName) + + db, err := OpenFile(tempDBName, &Options{CanCreate: true, FileFormat: 2}) + if err != nil { + t.Fatalf("Cannot open db %s: %s\n", tempDBName, err) + } + db.Close() + if !fileExists(wName) { + t.Fatalf("Expect WAL file %s to exist but it doesn't", wName) + } + + db, err = OpenFile(tempDBName, &Options{CanCreate: true, FileFormat: 2, RemoveEmptyWAL: true}) + if err != nil { + t.Fatalf("Cannot open db %s: %s\n", tempDBName, err) + } + db.Close() + if fileExists(wName) { + t.Fatalf("Expect WAL file %s to be removed but it still exists", wName) + } +} + +func detectVersion(f *os.File) (int, error) { + b := make([]byte, 16) + if _, err := f.ReadAt(b, 0); err != nil { + return 0, err + } + + switch { + case bytes.Equal(b[:len(magic)], []byte(magic)): + return 1, nil + case bytes.Equal(b[:len(magic2)], []byte(magic2)): + return 2, nil + default: + return 0, fmt.Errorf("unrecognized file format") + } +} + +func TestV2(t *testing.T) { + RegisterDriver() + RegisterDriver2() + + f1, err := ioutil.TempFile("", "") + if err != nil { + t.Fatal(err) + } + + nm1 := f1.Name() + + defer func() { + f1.Close() + os.Remove(nm1) + }() + + db1, err := sql.Open("ql", nm1) + if err != nil { + t.Fatal(err) + } + + tx, err := db1.Begin() + if err != nil { + t.Fatal(err) + } + + if _, err = tx.Exec("create table t (c int); insert into t values (1)"); err != nil { + t.Fatal(err) + } + + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + + vn, err := detectVersion(f1) + if err != nil { + t.Fatal(err) + } + + if vn != 1 { + t.Fatal(vn) + } + + if err = db1.Close(); err != nil { + t.Fatal(err) + } + + f2, err := ioutil.TempFile("", "") + if err != nil { + t.Fatal(err) + } + + nm2 := f2.Name() + + defer func() { + f2.Close() + os.Remove(nm2) + }() + + db2, err := sql.Open("ql2", nm2) + if err != nil { + t.Fatal(err) + } + + if tx, err = db2.Begin(); err != nil { + t.Fatal(err) + } + + if _, err = tx.Exec("create table t (c int); insert into t values (2)"); err != nil { + t.Fatal(err) + } + + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + + if vn, err = detectVersion(f2); err != nil { + t.Fatal(err) + } + + if vn != 2 { + t.Fatal(vn) + } + + if err = db2.Close(); err != nil { + t.Fatal(err) + } + + db, err := sql.Open("ql2", f1.Name()) + if err != nil { + t.Fatal(err) + } + + row := db.QueryRow("select * from t") + if row == nil { + t.Fatal(err) + } + + var n int64 + if err = row.Scan(&n); err != nil { + t.Fatal(err) + } + + if n != 1 { + t.Fatal(n) + } + + if err = db.Close(); err != nil { + t.Fatal(err) + } + + if db, err = sql.Open("ql", f2.Name()); err != nil { + t.Fatal(err) + } + + if row = db.QueryRow("select * from t"); row == nil { + t.Fatal(err) + } + + if err = row.Scan(&n); err != nil { + t.Fatal(err) + } + + if n != 2 { + t.Fatal(n) + } + + if err = db.Close(); err != nil { + t.Fatal(err) + } +} + +func TestBigRec(t *testing.T) { + RegisterDriver() + RegisterDriver2() + + f1, err := ioutil.TempFile("", "") + if err != nil { + t.Fatal(err) + } + + nm1 := f1.Name() + + defer func() { + f1.Close() + os.Remove(nm1) + }() + + db, err := sql.Open("ql", nm1) + if err != nil { + t.Fatal(err) + } + + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + + _, err = tx.Exec("create table t (s string); insert into t values(__testString(1<<20))") // 1 MB string not possible with V1 format + if err == nil { + t.Fatal("unexpected success") + } + + if !strings.Contains(err.Error(), "limit") { + t.Fatal(err) + } + + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + + if err = db.Close(); err != nil { + t.Fatal(err) + } + + f2, err := ioutil.TempFile("", "") + if err != nil { + t.Fatal(err) + } + + nm2 := f2.Name() + + defer func() { + f2.Close() + os.Remove(nm2) + }() + + if db, err = sql.Open("ql2", nm2); err != nil { + t.Fatal(err) + } + + if tx, err = db.Begin(); err != nil { + t.Fatal(err) + } + + if _, err = tx.Exec("create table t (s string); insert into t values(__testString(1<<20))"); err != nil { // 1 MB string possible with V2 format + t.Fatal(err) + } + + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + + if err = db.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/plan.go b/plan.go index 65132d4..4149f49 100644 --- a/plan.go +++ b/plan.go @@ -623,7 +623,7 @@ func (r *indexPlan) do(ctx *execCtx, f func(interface{}, []interface{}) (bool, e return r.doIntervalCO(ctx, f) default: //dbg("", r.kind) - panic("internal error 072") + panic("internal error 068") } } @@ -662,7 +662,7 @@ func (r *indexPlan) explain(w strutil.Formatter) { w.Format(" > %v && %s <= %v", value{r.lval}, r.cname, value{r.hval}) default: //dbg("", r.kind) - panic("internal error 073") + panic("internal error 053") } w.Format("\nâ””Output field names %v\n", qnames(r.fieldNames())) } diff --git a/storage.go b/storage.go index 61818a5..fd7209f 100644 --- a/storage.go +++ b/storage.go @@ -837,10 +837,6 @@ func newRoot(store storage) (r *root, err error) { tables: map[string]*table{}, }, nil case 1: // existing DB, load tables - if len(data) != 1 { - return nil, fmt.Errorf("corrupted DB: root is an %d-scalar", len(data)) - } - p, ok := data[0].(int64) if !ok { return nil, fmt.Errorf("corrupted DB: root head has type %T", data[0]) diff --git a/storage_test.go b/storage_test.go index 707dcf6..426aa6e 100644 --- a/storage_test.go +++ b/storage_test.go @@ -326,7 +326,6 @@ func test(t *testing.T, s testDB) (panicked error) { max = n } for itest, test := range testdata[*oN:max] { - //dbg("------------------------------------------------------------- ( itest %d ) ----", itest) var re *regexp.Regexp a := strings.Split(test+"|", "|") q, rset := a[0], strings.TrimSpace(a[1]) @@ -420,7 +419,7 @@ func test(t *testing.T, s testDB) (panicked error) { nfo, err := db.Info() if err != nil { - dbg("", err) + //dbg("", err) panic(err) } @@ -452,7 +451,7 @@ func test(t *testing.T, s testDB) (panicked error) { }() if err = s.mark(); err != nil { - t.Error(err) + t.Error(itest, err) return } diff --git a/v2.sh b/v2.sh new file mode 100755 index 0000000..c799bce --- /dev/null +++ b/v2.sh @@ -0,0 +1,4 @@ +grep -v FileV2 $1 > old +grep FileV2 $1 | sed s/FileV2/File/ > new +benchcmp -mag -changed old new | sed s/Benchmark// +rm -f old new