Skip to content
Permalink
Browse files

init public history

  • Loading branch information...
bictorman committed Feb 29, 2016
0 parents commit 6c7c627f8c2c13d42490c2ea7011c968c3508ab5
Showing with 18,598 additions and 0 deletions.
  1. +373 −0 LICENSE
  2. +5 −0 README.md
  3. +54 −0 biglog/README.md
  4. +115 −0 biglog/benchmark_test.go
  5. +565 −0 biglog/biglog.go
  6. +170 −0 biglog/biglog_test.go
  7. +152 −0 biglog/example/example.go
  8. +274 −0 biglog/index_reader.go
  9. +158 −0 biglog/index_reader_test.go
  10. +42 −0 biglog/info.go
  11. +156 −0 biglog/reader.go
  12. +127 −0 biglog/reader_test.go
  13. +233 −0 biglog/scanner.go
  14. +105 −0 biglog/scanner_test.go
  15. +524 −0 biglog/segment.go
  16. +125 −0 biglog/segment_test.go
  17. +124 −0 biglog/streamer.go
  18. +97 −0 biglog/streamer_test.go
  19. +72 −0 biglog/util_test.go
  20. +37 −0 biglog/watcher.go
  21. +76 −0 biglog/watcher_test.go
  22. +75 −0 cmd/netlog/main.go
  23. +98 −0 errors.go
  24. +105 −0 message.go
  25. +225 −0 netlog.go
  26. +44 −0 segment_monitor.go
  27. +82 −0 streamer_atomicmap.go
  28. +258 −0 topic.go
  29. +80 −0 topic_atomicmap.go
  30. +99 −0 topic_test.go
  31. +101 −0 topicscanner.go
  32. +73 −0 topicscanner_atomicmap.go
  33. +22 −0 topicscanner_test.go
  34. +78 −0 transport/benchmark_test.go
  35. +324 −0 transport/http_transport.go
  36. +77 −0 transport/payload_test.go
  37. +54 −0 transport/util_test.go
  38. +48 −0 util_test.go
  39. +22 −0 vendor/comail.io/go/colog/LICENSE
  40. +159 −0 vendor/comail.io/go/colog/README.md
  41. +534 −0 vendor/comail.io/go/colog/colog.go
  42. +35 −0 vendor/comail.io/go/colog/interfaces.go
  43. +120 −0 vendor/comail.io/go/colog/json_formatter.go
  44. +61 −0 vendor/comail.io/go/colog/std_extractor.go
  45. +226 −0 vendor/comail.io/go/colog/std_formatter.go
  46. +31 −0 vendor/comail.io/go/colog/tty.go
  47. +7 −0 vendor/comail.io/go/colog/tty_bsd.go
  48. +5 −0 vendor/comail.io/go/colog/tty_linux.go
  49. +62 −0 vendor/comail.io/go/colog/tty_windows.go
  50. +27 −0 vendor/github.com/comail/go-uuid/uuid/LICENSE
  51. +84 −0 vendor/github.com/comail/go-uuid/uuid/dce.go
  52. +8 −0 vendor/github.com/comail/go-uuid/uuid/doc.go
  53. +53 −0 vendor/github.com/comail/go-uuid/uuid/hash.go
  54. +30 −0 vendor/github.com/comail/go-uuid/uuid/json.go
  55. +101 −0 vendor/github.com/comail/go-uuid/uuid/node.go
  56. +132 −0 vendor/github.com/comail/go-uuid/uuid/time.go
  57. +43 −0 vendor/github.com/comail/go-uuid/uuid/util.go
  58. +163 −0 vendor/github.com/comail/go-uuid/uuid/uuid.go
  59. +41 −0 vendor/github.com/comail/go-uuid/uuid/version1.go
  60. +25 −0 vendor/github.com/comail/go-uuid/uuid/version4.go
  61. +24 −0 vendor/github.com/julienschmidt/httprouter/LICENSE
  62. +289 −0 vendor/github.com/julienschmidt/httprouter/README.md
  63. +123 −0 vendor/github.com/julienschmidt/httprouter/path.go
  64. +411 −0 vendor/github.com/julienschmidt/httprouter/router.go
  65. +649 −0 vendor/github.com/julienschmidt/httprouter/tree.go
  66. +22 −0 vendor/github.com/ninibe/bigduration/LICENSE
  67. +11 −0 vendor/github.com/ninibe/bigduration/README.md
  68. +155 −0 vendor/github.com/ninibe/bigduration/bigduration.go
  69. +27 −0 vendor/golang.org/x/net/LICENSE
  70. +22 −0 vendor/golang.org/x/net/PATENTS
  71. +447 −0 vendor/golang.org/x/net/context/context.go
  72. +51 −0 vendor/golang.org/x/net/http2/Dockerfile
  73. +3 −0 vendor/golang.org/x/net/http2/Makefile
  74. +20 −0 vendor/golang.org/x/net/http2/README
  75. +225 −0 vendor/golang.org/x/net/http2/client_conn_pool.go
  76. +89 −0 vendor/golang.org/x/net/http2/configure_transport.go
  77. +122 −0 vendor/golang.org/x/net/http2/errors.go
  78. +60 −0 vendor/golang.org/x/net/http2/fixed_buffer.go
  79. +50 −0 vendor/golang.org/x/net/http2/flow.go
  80. +1,485 −0 vendor/golang.org/x/net/http2/frame.go
  81. +11 −0 vendor/golang.org/x/net/http2/go15.go
  82. +170 −0 vendor/golang.org/x/net/http2/gotrack.go
  83. +78 −0 vendor/golang.org/x/net/http2/headermap.go
  84. +251 −0 vendor/golang.org/x/net/http2/hpack/encode.go
  85. +542 −0 vendor/golang.org/x/net/http2/hpack/hpack.go
  86. +190 −0 vendor/golang.org/x/net/http2/hpack/huffman.go
  87. +352 −0 vendor/golang.org/x/net/http2/hpack/tables.go
  88. +429 −0 vendor/golang.org/x/net/http2/http2.go
  89. +11 −0 vendor/golang.org/x/net/http2/not_go15.go
  90. +13 −0 vendor/golang.org/x/net/http2/not_go16.go
  91. +147 −0 vendor/golang.org/x/net/http2/pipe.go
  92. +2,149 −0 vendor/golang.org/x/net/http2/server.go
  93. +1,662 −0 vendor/golang.org/x/net/http2/transport.go
  94. +263 −0 vendor/golang.org/x/net/http2/write.go
  95. +283 −0 vendor/golang.org/x/net/http2/writesched.go
  96. +29 −0 vendor/launchpad.net/gommap/LICENSE
  97. +53 −0 vendor/launchpad.net/gommap/consts.c.txt
  98. +47 −0 vendor/launchpad.net/gommap/consts.go
  99. +4 −0 vendor/launchpad.net/gommap/consts.sh
  100. +159 −0 vendor/launchpad.net/gommap/gommap.go
  101. +8 −0 vendor/launchpad.net/gommap/mmap_darwin_amd64.go
  102. +12 −0 vendor/launchpad.net/gommap/mmap_linux_386.go
  103. +8 −0 vendor/launchpad.net/gommap/mmap_linux_amd64.go
  104. +46 −0 vendor/vendor.json
373 LICENSE

Large diffs are not rendered by default.

@@ -0,0 +1,5 @@
# NetLog
A lightweight, HTTP-centric, log-based (Kafka-style) message queue.

### Work-in-progress
To peek at the internals start with [BigLog](https://github.com/ninibe/netlog/tree/master/biglog).
@@ -0,0 +1,54 @@
# BigLog

BigLog is a high level abstraction on top os.File designed to store large amounts of data. BigLog is NetLog's core component, which can be embedded in any application.

In a log-based queue, logs must be append-only. But most people eventually need to delete data, so instead of a single file we have several "segments". Every segment is just a data file with blob of bytes written to it and a companion index file. Indexes are preallocated in a fixed size every time a segment is created and memory-mapped. Each entry in the index has the format.

### Index entry format

```
+---------------------------------------------------------------+
| relative offset | unix timestamp | data file offset |
+---------------------------------------------------------------+
| (4bytes)[uint32] | (4bytes)[uint32] | (8bytes)[int64] |
+---------------------------------------------------------------+
```

Every segment has a base offset, and the index stores the relative offset to that base, with the first offset being always 1. The index can be sparse. The last element of the index is always the NEXT offset to be written, who's timestamp is not set.

### Index example

```
+-----------------------------------+
| 1 | 1456514390 | 0 | <- first RO, offset 0 in data file
+-----------------------------------+
| 2 | 1456514391 | 32 | <- RO 2 starts 32 bytes later
+-----------------------------------+
| 4 | 1456514392 | 96 | <- RO 4 starts 64 bytes later
+-----------------------------------+ RO 3 is embedded somewhere between position 32 and 96
| 11 | 0 | 320 | <- Next available offset is RO11 which goes at position 320
+-----------------------------------+ (size of the data file)
```

The segment with the highest base offset is the "hot" segment, the only one who gets writes under the hood via Write() [io.Writer interface] for a single offset or WriteN() for N offsets. You can create a new hot segment calling Split(), and discard the oldest one calling Trim().

There are 2 reading primitives, a Reader [io.Reader] which reads over the data files returning byte blobs, and an Index Reader which reads index files retuning entries. Both are initialized (multiple instances allowed) and operate separately.

```
+------------------+
-> | index reader | ->
+------------------+
+-----------------------------------+ +-----------------------------------+
| index file 0 | | index file 1 |
+-----------------------------------+ +-----------------------------------+
+-----------------------------------+ +-----------------------------------+
| data file 0 | | data file 1 |
+-----------------------------------+ +-----------------------------------+
+------------------+
-> | data reader | ->
+------------------+
```

Readers will transparently jump through segments until their buffer is full or EOF is reached, they can be instantiated to start in any give offset with an specific entry in the index, if an embedded offset it requested the reader will start in the previous known offset position.

Based on these 2 readers, BigLog provides another 2 higher abstractions, Scanner and Streamer. [See the godocs](https://godoc.org/github.com/ninibe/netlog/biglog).
@@ -0,0 +1,115 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package biglog

import (
"fmt"
"math/rand"
"os"
"path/filepath"
"testing"
)

var data = []byte(`Lorem ipsum dolor sit amet, consectetur adipiscing elit. Etiam volutpat ante in rhoncus commodo.
Morbi id ipsum rutrum, elementum tortor ac, pellentesque libero. Vestibulum tincidunt porta orci.
Vestibulum mattis, augue non mattis malesuada, lectus justo iaculis ipsum, eget euismod sapien nunc vitae nisi.
Nullam eu mattis dui. Etiam faucibus egestas erat sit amet semper. Praesent tincidunt mattis blandit.
Ut sed magna eget lacus ullamcorper semper sed quis felis. Aenean id consequat orci. In hac habitasse platea dictumst.
Nam tincidunt ipsum vitae egestas sollicitudin.`)

func BenchmarkFileWrite(b *testing.B) {
tmpfile := filepath.Join(os.TempDir(), "writetest")
f, _ := os.Create(tmpfile)
b.ReportAllocs()
b.SetBytes(int64(len(data)))
for i := 0; i <= b.N; i++ {
f.Write(data)
}
f.Close()
os.Remove(tmpfile)
}

func BenchmarkFileWriteSync(b *testing.B) {
tmpfile := filepath.Join(os.TempDir(), "writetest")
f, _ := os.Create(tmpfile)
b.ReportAllocs()
b.SetBytes(int64(len(data)))
for i := 0; i <= b.N; i++ {
f.Write(data)
f.Sync()
}
f.Close()
os.Remove(tmpfile)
}

func BenchmarkSegWrite(b *testing.B) {
b.StopTimer()

seg, _ := createSegment(os.TempDir(), 10*1024*1024, rand.Int63())
b.ReportAllocs()
b.SetBytes(int64(len(data)))
b.StartTimer()
for i := 0; i <= b.N; i++ {
seg.WriteN(data, 1)
}
b.StopTimer()
seg.Delete(true)
}

func BenchmarkSegWriteSync(b *testing.B) {
b.StopTimer()
seg, _ := createSegment(os.TempDir(), 10*1024*1024, rand.Int63())
b.ReportAllocs()
b.SetBytes(int64(len(data)))
b.StartTimer()
for i := 0; i <= b.N; i++ {
seg.WriteN(data, 1)
seg.Sync()
}
b.StopTimer()
seg.Delete(true)
}

func BenchmarkBiglogWrite(b *testing.B) {
b.StopTimer()

bl, _ := Create(filepath.Join(os.TempDir(), fmt.Sprintf("biglogtest-%d", rand.Int63())), 10*1024*1024)
b.ReportAllocs()
b.SetBytes(int64(len(data)))
b.StartTimer()
for i := 0; i <= b.N; i++ {
bl.Write(data)
}
b.StopTimer()
bl.Delete(true)
}

func BenchmarkBiglogWrite4Kbuf(b *testing.B) {
b.StopTimer()
bl, _ := Create(filepath.Join(os.TempDir(), fmt.Sprintf("biglogtest-%d", rand.Int63())), 10*1024*1024)
bl.SetOpts(BufioWriter(4 * 1024))
b.ReportAllocs()
b.SetBytes(int64(len(data)))
b.StartTimer()
for i := 0; i <= b.N; i++ {
bl.Write(data)
}
b.StopTimer()
bl.Delete(true)
}

func BenchmarkBiglogWrite64Kbuf(b *testing.B) {
b.StopTimer()
bl, _ := Create(filepath.Join(os.TempDir(), fmt.Sprintf("biglogtest-%d", rand.Int63())), 10*1024*1024)
bl.SetOpts(BufioWriter(64 * 1024))
b.ReportAllocs()
b.SetBytes(int64(len(data)))
b.StartTimer()
for i := 0; i <= b.N; i++ {
bl.Write(data)
}
b.StopTimer()
bl.Delete(true)
}

0 comments on commit 6c7c627

Please sign in to comment.
You can’t perform that action at this time.