diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index dc37146a..eb692193 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -204,9 +204,23 @@ "ImportPath": "google.golang.org/grpc/peer", "Rev": "ab0be5212fb225475f2087566eded7da5d727960" }, + { + "ImportPath": "github.com/RoaringBitmap/roaring", + "Comment": "v0.2.8-7-gc3e090c", + "Rev": "c3e090c41e857e2efc2b85af31c3c23bde0e9574" + }, { "ImportPath": "google.golang.org/grpc/transport", "Rev": "ab0be5212fb225475f2087566eded7da5d727960" + }, + { + "ImportPath": "github.com/mschoch/smat", + "Rev": "90eadee771aeab36e8bf796039b8c261bebebe4f" + }, + { + "ImportPath": "github.com/willf/bitset", + "Comment": "v1.0.0-64-g85301ea", + "Rev": "85301ea3894aa6d640cbc2afccc0994239444300" } ] } diff --git a/containerd/containerd.go b/containerd/containerd.go index c25b3e95..30f8d903 100644 --- a/containerd/containerd.go +++ b/containerd/containerd.go @@ -59,6 +59,7 @@ var ContainerdCommand = cli.Command{ driver := context.GlobalString("driver") kernel := context.GlobalString("kernel") initrd := context.GlobalString("initrd") + vsock := context.GlobalBool("vsock") template := context.GlobalString("template") stateDir := context.String("state-dir") containerdDir := context.String("containerd-dir") @@ -111,7 +112,7 @@ var ContainerdCommand = cli.Command{ if template != "" { f = singlefactory.New(templatefactory.NewFromExisted(tconfig)) } else { - f = factory.NewFromConfigs(kernel, initrd, nil) + f = factory.NewFromConfigs(kernel, initrd, vsock, nil) } sv, err := supervisor.New(stateDir, containerdDir, f, context.GlobalInt("default_cpus"), context.GlobalInt("default_memory")) diff --git a/driverloader/driverloader_linux.go b/driverloader/driverloader_linux.go index 47a86b51..46dcab2a 100644 --- a/driverloader/driverloader_linux.go +++ b/driverloader/driverloader_linux.go @@ -9,9 +9,15 @@ import ( "github.com/hyperhq/runv/hypervisor/libvirt" "github.com/hyperhq/runv/hypervisor/qemu" "github.com/hyperhq/runv/hypervisor/xen" + "github.com/hyperhq/runv/lib/vsock" ) -func Probe(driver string) (hypervisor.HypervisorDriver, error) { +func Probe(driver string) (hd hypervisor.HypervisorDriver, err error) { + defer func() { + if hd != nil && hypervisor.VsockCidManager == nil { + hypervisor.VsockCidManager = vsock.NewDefaultVsockCidAllocator() + } + }() switch strings.ToLower(driver) { case "libvirt": ld := libvirt.InitDriver() diff --git a/factory/direct/direct.go b/factory/direct/direct.go index 2b72bc7b..7136a295 100644 --- a/factory/direct/direct.go +++ b/factory/direct/direct.go @@ -10,11 +10,12 @@ type directFactory struct { config hypervisor.BootConfig } -func New(cpu, mem int, kernel, initrd string) base.Factory { +func New(cpu, mem int, kernel, initrd string, vsock bool) base.Factory { b := hypervisor.BootConfig{ CPU: cpu, Memory: mem, HotAddCpuMem: true, + EnableVsock: vsock, Kernel: kernel, Initrd: initrd, } diff --git a/factory/factory.go b/factory/factory.go index e4109236..527e914f 100644 --- a/factory/factory.go +++ b/factory/factory.go @@ -6,6 +6,7 @@ package factory import ( "encoding/json" + "github.com/golang/glog" "github.com/hyperhq/runv/factory/base" "github.com/hyperhq/runv/factory/cache" @@ -28,21 +29,21 @@ type FactoryConfig struct { Memory int `json:"memory"` } -func NewFromConfigs(kernel, initrd string, configs []FactoryConfig) Factory { +func NewFromConfigs(kernel, initrd string, vsock bool, configs []FactoryConfig) Factory { bases := make([]base.Factory, len(configs)) for i, c := range configs { var b base.Factory if c.Template { - b = template.New(hypervisor.BaseDir+"/template", c.Cpu, c.Memory, kernel, initrd) + b = template.New(hypervisor.BaseDir+"/template", c.Cpu, c.Memory, kernel, initrd, vsock) } else { - b = direct.New(c.Cpu, c.Memory, kernel, initrd) + b = direct.New(c.Cpu, c.Memory, kernel, initrd, vsock) } bases[i] = cache.New(c.Cache, b) } if len(bases) == 0 { // skip GetVm from the base factory - return single.New(direct.New(1000000, 1000000, kernel, initrd)) + return single.New(direct.New(1000000, 1000000, kernel, initrd, vsock)) } else if len(bases) == 1 { return single.New(bases[0]) } else { @@ -52,12 +53,12 @@ func NewFromConfigs(kernel, initrd string, configs []FactoryConfig) Factory { // vmFactoryPolicy = [FactoryConfig,]*FactoryConfig // FactoryConfig = {["cache":NUMBER,]["template":true|false,]"cpu":NUMBER,"memory":NUMBER} -func NewFromPolicy(kernel, initrd string, policy string) Factory { +func NewFromPolicy(kernel, initrd string, vsock bool, policy string) Factory { var configs []FactoryConfig jsonString := "[" + policy + "]" err := json.Unmarshal([]byte(jsonString), &configs) if err != nil && policy != "none" { glog.Errorf("Incorrect policy: %s", policy) } - return NewFromConfigs(kernel, initrd, configs) + return NewFromConfigs(kernel, initrd, vsock, configs) } diff --git a/factory/single/single.go b/factory/single/single.go index a40695db..9bdeb158 100644 --- a/factory/single/single.go +++ b/factory/single/single.go @@ -18,10 +18,11 @@ func (f Factory) GetVm(cpu, mem int) (*hypervisor.Vm, error) { if config.CPU > cpu || config.Memory > mem { // also strip unrelated option from @config boot := &hypervisor.BootConfig{ - CPU: cpu, - Memory: mem, - Kernel: config.Kernel, - Initrd: config.Initrd, + CPU: cpu, + Memory: mem, + Kernel: config.Kernel, + Initrd: config.Initrd, + EnableVsock: config.EnableVsock, } return hypervisor.GetVm("", boot, false) } diff --git a/factory/template/template.go b/factory/template/template.go index 80157950..a4101ac1 100644 --- a/factory/template/template.go +++ b/factory/template/template.go @@ -16,7 +16,7 @@ type templateFactory struct { s *template.TemplateVmConfig } -func New(templateRoot string, cpu, mem int, kernel, initrd string) base.Factory { +func New(templateRoot string, cpu, mem int, kernel, initrd string, vsock bool) base.Factory { var vmName string for { @@ -25,11 +25,11 @@ func New(templateRoot string, cpu, mem int, kernel, initrd string) base.Factory break } } - s, err := template.CreateTemplateVM(templateRoot+"/"+vmName, vmName, cpu, mem, kernel, initrd) + s, err := template.CreateTemplateVM(templateRoot+"/"+vmName, vmName, cpu, mem, kernel, initrd, vsock) if err != nil { glog.Infof("failed to create template factory: %v", err) glog.Infof("use direct factory instead") - return direct.New(cpu, mem, kernel, initrd) + return direct.New(cpu, mem, kernel, initrd, vsock) } return &templateFactory{s: s} } diff --git a/hyperstart/api/json/constants.go b/hyperstart/api/json/constants.go index 37b37564..c347ebd4 100644 --- a/hyperstart/api/json/constants.go +++ b/hyperstart/api/json/constants.go @@ -33,3 +33,6 @@ const ( // "hyperstart" is the special container ID for adding processes. const HYPERSTART_EXEC_CONTAINER = "hyperstart" + +const HYPER_VSOCK_CTL_PORT = 2718 +const HYPER_VSOCK_MSG_PORT = 2719 diff --git a/hyperstart/libhyperstart/serial.go b/hyperstart/libhyperstart/json.go similarity index 84% rename from hyperstart/libhyperstart/serial.go rename to hyperstart/libhyperstart/json.go index d521d6b6..7860a066 100644 --- a/hyperstart/libhyperstart/serial.go +++ b/hyperstart/libhyperstart/json.go @@ -23,7 +23,7 @@ type pState struct { stderrPipe io.ReadCloser } -type serialBasedHyperstart struct { +type jsonBasedHyperstart struct { sync.RWMutex vmAPIVersion uint32 closed bool @@ -44,8 +44,8 @@ type hyperstartCmd struct { result chan<- error } -func NewSerialJsonBasedHyperstart(ctlSock, streamSock string, lastStreamSeq uint64, waitReady bool) Hyperstart { - h := &serialBasedHyperstart{ +func NewJsonBasedHyperstart(ctlSock, streamSock string, lastStreamSeq uint64, waitReady bool) Hyperstart { + h := &jsonBasedHyperstart{ procs: make(map[pKey]*pState), lastStreamSeq: lastStreamSeq, streamOuts: make(map[uint64]streamOut), @@ -58,11 +58,11 @@ func NewSerialJsonBasedHyperstart(ctlSock, streamSock string, lastStreamSeq uint return h } -func (h *serialBasedHyperstart) Close() { +func (h *jsonBasedHyperstart) Close() { h.Lock() defer h.Unlock() if !h.closed { - glog.Info("close serialBasedHyperstart") + glog.Info("close jsonBasedHyperstart") for pk := range h.procs { h.processAsyncEvents <- hyperstartapi.ProcessAsyncEvent{Container: pk.c, Process: pk.p, Event: "finished", Status: 255} } @@ -78,11 +78,11 @@ func (h *serialBasedHyperstart) Close() { } } -func (h *serialBasedHyperstart) ProcessAsyncEvents() (<-chan hyperstartapi.ProcessAsyncEvent, error) { +func (h *jsonBasedHyperstart) ProcessAsyncEvents() (<-chan hyperstartapi.ProcessAsyncEvent, error) { return h.processAsyncEvents, nil } -func (h *serialBasedHyperstart) LastStreamSeq() uint64 { +func (h *jsonBasedHyperstart) LastStreamSeq() uint64 { return h.lastStreamSeq } @@ -129,9 +129,11 @@ func readVmMessage(conn io.Reader) (*hyperstartapi.DecodedMessage, error) { }, nil } -func handleCtlSock(h *serialBasedHyperstart, ctlSock string, waitReady bool) error { - conn, err := utils.UnixSocketConnect(ctlSock) +func handleCtlSock(h *jsonBasedHyperstart, ctlSock string, waitReady bool) error { + conn, err := utils.SocketConnect(ctlSock) if err != nil { + glog.Error("Cannot connect to ctl socket ", ctlSock, err.Error()) + h.Close() return err } @@ -162,7 +164,7 @@ func handleCtlSock(h *serialBasedHyperstart, ctlSock string, waitReady bool) err return err } -func (h *serialBasedHyperstart) hyperstartCommandWithRetMsg(code uint32, msg interface{}) (retMsg []byte, err error) { +func (h *jsonBasedHyperstart) hyperstartCommandWithRetMsg(code uint32, msg interface{}) (retMsg []byte, err error) { defer func() { if recover() != nil { err = fmt.Errorf("send ctl channel error, the hyperstart might have closed") @@ -179,12 +181,12 @@ func (h *serialBasedHyperstart) hyperstartCommandWithRetMsg(code uint32, msg int return vcmd.retMsg, err } -func (h *serialBasedHyperstart) hyperstartCommand(code uint32, msg interface{}) error { +func (h *jsonBasedHyperstart) hyperstartCommand(code uint32, msg interface{}) error { _, err := h.hyperstartCommandWithRetMsg(code, msg) return err } -func handleMsgToHyperstart(h *serialBasedHyperstart, conn io.WriteCloser) { +func handleMsgToHyperstart(h *jsonBasedHyperstart, conn io.WriteCloser) { looping := true cmds := []*hyperstartCmd{} @@ -297,7 +299,7 @@ func handleMsgToHyperstart(h *serialBasedHyperstart, conn io.WriteCloser) { } } -func handleMsgFromHyperstart(h *serialBasedHyperstart, conn io.Reader) { +func handleMsgFromHyperstart(h *jsonBasedHyperstart, conn io.Reader) { for { res, err := readVmMessage(conn) if err == nil { @@ -355,7 +357,7 @@ func readTtyMessage(conn io.Reader) (*hyperstartapi.TtyMessage, error) { }, nil } -func handleStreamToHyperstart(h *serialBasedHyperstart, conn io.WriteCloser) { +func handleStreamToHyperstart(h *jsonBasedHyperstart, conn io.WriteCloser) { for { msg, ok := <-h.streamChan if !ok { @@ -372,10 +374,10 @@ func handleStreamToHyperstart(h *serialBasedHyperstart, conn io.WriteCloser) { } } -func handleStreamSock(h *serialBasedHyperstart, streamSock string) error { - conn, err := utils.UnixSocketConnect(streamSock) +func handleStreamSock(h *jsonBasedHyperstart, streamSock string) error { + conn, err := utils.SocketConnect(streamSock) if err != nil { - glog.Error("Cannot connect to stream socket ", err.Error()) + glog.Error("Cannot connect to stream socket ", streamSock, err.Error()) h.Close() return err } @@ -387,7 +389,7 @@ func handleStreamSock(h *serialBasedHyperstart, streamSock string) error { return nil } -func handleStreamFromHyperstart(h *serialBasedHyperstart, conn io.Reader) { +func handleStreamFromHyperstart(h *jsonBasedHyperstart, conn io.Reader) { for { res, err := readTtyMessage(conn) if err != nil { @@ -423,7 +425,7 @@ func handleStreamFromHyperstart(h *serialBasedHyperstart, conn io.Reader) { } } -func (h *serialBasedHyperstart) sendProcessAsyncEvent(pae hyperstartapi.ProcessAsyncEvent) { +func (h *jsonBasedHyperstart) sendProcessAsyncEvent(pae hyperstartapi.ProcessAsyncEvent) { h.Lock() defer h.Unlock() pk := pKey{c: pae.Container, p: pae.Process} @@ -433,7 +435,7 @@ func (h *serialBasedHyperstart) sendProcessAsyncEvent(pae hyperstartapi.ProcessA } } -func (h *serialBasedHyperstart) sendProcessAsyncEvent4242(stdioSeq uint64, code uint8) { +func (h *jsonBasedHyperstart) sendProcessAsyncEvent4242(stdioSeq uint64, code uint8) { h.Lock() defer h.Unlock() for pk, ps := range h.procs { @@ -444,7 +446,7 @@ func (h *serialBasedHyperstart) sendProcessAsyncEvent4242(stdioSeq uint64, code } } -func (h *serialBasedHyperstart) removeStreamOut(seq uint64) { +func (h *jsonBasedHyperstart) removeStreamOut(seq uint64) { h.Lock() defer h.Unlock() // simple version: delete(h.streamOuts, seq), but the serial-based hyperstart @@ -459,7 +461,7 @@ func (h *serialBasedHyperstart) removeStreamOut(seq uint64) { type streamIn struct { streamSeq uint64 - h *serialBasedHyperstart + h *jsonBasedHyperstart } func (s *streamIn) Write(data []byte) (ret int, err error) { @@ -513,7 +515,7 @@ type streamOut struct { ps *pState // required for removeStreamOut() } -func (h *serialBasedHyperstart) APIVersion() (uint32, error) { +func (h *jsonBasedHyperstart) APIVersion() (uint32, error) { retMsg, err := h.hyperstartCommandWithRetMsg(hyperstartapi.INIT_VERSION, nil) if err != nil { glog.Infof("get hyperstart API version error: %v\n", err) @@ -526,7 +528,7 @@ func (h *serialBasedHyperstart) APIVersion() (uint32, error) { return binary.BigEndian.Uint32(retMsg[:4]), nil } -func (h *serialBasedHyperstart) WriteFile(container, path string, data []byte) error { +func (h *jsonBasedHyperstart) WriteFile(container, path string, data []byte) error { writeCmd, _ := json.Marshal(hyperstartapi.FileCommand{ Container: container, File: path, @@ -535,18 +537,18 @@ func (h *serialBasedHyperstart) WriteFile(container, path string, data []byte) e return h.hyperstartCommand(hyperstartapi.INIT_WRITEFILE, writeCmd) } -func (h *serialBasedHyperstart) ReadFile(container, path string) ([]byte, error) { +func (h *jsonBasedHyperstart) ReadFile(container, path string) ([]byte, error) { return h.hyperstartCommandWithRetMsg(hyperstartapi.INIT_READFILE, &hyperstartapi.FileCommand{ Container: container, File: path, }) } -func (h *serialBasedHyperstart) AddRoute(r []hyperstartapi.Route) error { +func (h *jsonBasedHyperstart) AddRoute(r []hyperstartapi.Route) error { return h.hyperstartCommand(hyperstartapi.INIT_SETUPROUTE, hyperstartapi.Routes{Routes: r}) } -func (h *serialBasedHyperstart) UpdateInterface(dev, ip, mask string) error { +func (h *jsonBasedHyperstart) UpdateInterface(dev, ip, mask string) error { return h.hyperstartCommand(hyperstartapi.INIT_SETUPINTERFACE, hyperstartapi.NetworkInf{ Device: dev, IpAddress: ip, @@ -554,7 +556,7 @@ func (h *serialBasedHyperstart) UpdateInterface(dev, ip, mask string) error { }) } -func (h *serialBasedHyperstart) TtyWinResize4242(container, process string, row, col uint16) error { +func (h *jsonBasedHyperstart) TtyWinResize4242(container, process string, row, col uint16) error { h.RLock() p, ok := h.procs[pKey{c: container, p: process}] h.RUnlock() @@ -572,7 +574,7 @@ func (h *serialBasedHyperstart) TtyWinResize4242(container, process string, row, return h.hyperstartCommand(hyperstartapi.INIT_WINSIZE, cmd) } -func (h *serialBasedHyperstart) TtyWinResize(container, process string, row, col uint16) error { +func (h *jsonBasedHyperstart) TtyWinResize(container, process string, row, col uint16) error { if h.vmAPIVersion <= 4242 { return h.TtyWinResize4242(container, process, row, col) } @@ -585,17 +587,17 @@ func (h *serialBasedHyperstart) TtyWinResize(container, process string, row, col return h.hyperstartCommand(hyperstartapi.INIT_WINSIZE, cmd) } -func (h *serialBasedHyperstart) OnlineCpuMem() error { +func (h *jsonBasedHyperstart) OnlineCpuMem() error { return h.hyperstartCommand(hyperstartapi.INIT_ONLINECPUMEM, nil) } -func (h *serialBasedHyperstart) allocStreamSeq() uint64 { +func (h *jsonBasedHyperstart) allocStreamSeq() uint64 { seq := h.lastStreamSeq h.lastStreamSeq++ return seq } -func (h *serialBasedHyperstart) setupProcessIo(ps *pState, terminal bool) { +func (h *jsonBasedHyperstart) setupProcessIo(ps *pState, terminal bool) { ps.stdioSeq = h.allocStreamSeq() stdoutPipe, stdout := io.Pipe() // TODO: make StreamOut nonblockable ps.stdoutPipe = stdoutPipe @@ -609,7 +611,7 @@ func (h *serialBasedHyperstart) setupProcessIo(ps *pState, terminal bool) { ps.stdinPipe = streamIn{streamSeq: ps.stdioSeq, h: h} } -func (h *serialBasedHyperstart) removeProcess(container, process string) { +func (h *jsonBasedHyperstart) removeProcess(container, process string) { h.Lock() defer h.Unlock() pk := pKey{c: container, p: process} @@ -624,7 +626,7 @@ func (h *serialBasedHyperstart) removeProcess(container, process string) { } } -func (h *serialBasedHyperstart) NewContainer(c *hyperstartapi.Container) (io.WriteCloser, io.ReadCloser, io.ReadCloser, error) { +func (h *jsonBasedHyperstart) NewContainer(c *hyperstartapi.Container) (io.WriteCloser, io.ReadCloser, io.ReadCloser, error) { h.Lock() if _, existed := h.procs[pKey{c: c.Id, p: c.Process.Id}]; existed { h.Unlock() @@ -645,7 +647,7 @@ func (h *serialBasedHyperstart) NewContainer(c *hyperstartapi.Container) (io.Wri return &ps.stdinPipe, ps.stdoutPipe, ps.stderrPipe, err } -func (h *serialBasedHyperstart) AddProcess(container string, p *hyperstartapi.Process) (io.WriteCloser, io.ReadCloser, io.ReadCloser, error) { +func (h *jsonBasedHyperstart) AddProcess(container string, p *hyperstartapi.Process) (io.WriteCloser, io.ReadCloser, io.ReadCloser, error) { h.Lock() if _, existed := h.procs[pKey{c: container, p: p.Id}]; existed { h.Unlock() @@ -669,7 +671,7 @@ func (h *serialBasedHyperstart) AddProcess(container string, p *hyperstartapi.Pr return &ps.stdinPipe, ps.stdoutPipe, ps.stderrPipe, err } -func (h *serialBasedHyperstart) SignalProcess(container, process string, signal syscall.Signal) error { +func (h *jsonBasedHyperstart) SignalProcess(container, process string, signal syscall.Signal) error { if h.vmAPIVersion <= 4242 { if process == "init" { return h.hyperstartCommand(hyperstartapi.INIT_KILLCONTAINER, hyperstartapi.KillCommand{ @@ -686,10 +688,10 @@ func (h *serialBasedHyperstart) SignalProcess(container, process string, signal }) } -func (h *serialBasedHyperstart) StartSandbox(pod *hyperstartapi.Pod) error { +func (h *jsonBasedHyperstart) StartSandbox(pod *hyperstartapi.Pod) error { return h.hyperstartCommand(hyperstartapi.INIT_STARTPOD, pod) } -func (h *serialBasedHyperstart) DestroySandbox() error { +func (h *jsonBasedHyperstart) DestroySandbox() error { return h.hyperstartCommand(hyperstartapi.INIT_DESTROYPOD, nil) } diff --git a/hypervisor/context.go b/hypervisor/context.go index b2c98238..3013e1df 100644 --- a/hypervisor/context.go +++ b/hypervisor/context.go @@ -3,6 +3,7 @@ package hypervisor import ( "fmt" "os" + "strconv" "sync" "time" @@ -10,12 +11,14 @@ import ( hyperstartapi "github.com/hyperhq/runv/hyperstart/api/json" "github.com/hyperhq/runv/hyperstart/libhyperstart" "github.com/hyperhq/runv/hypervisor/types" + "github.com/hyperhq/runv/lib/utils" ) type VmHwStatus struct { PciAddr int //next available pci addr for pci hotplug ScsiId int //next available scsi id for scsi hotplug AttachId uint64 //next available attachId for attached tty + GuestCid uint32 //vsock guest cid } const ( @@ -42,6 +45,7 @@ type VmContext struct { TtySockName string ConsoleSockName string ShareDir string + GuestCid uint32 pciAddr int //next available pci addr for pci hotplug scsiId int //next available scsi id for scsi hotplug @@ -87,6 +91,7 @@ func InitContext(id string, hub chan VmEvent, client chan *types.VmResponse, dc consoleSockName = homeDir + ConsoleSockName shareDir = homeDir + ShareDirTag ctx *VmContext + cid uint32 ) err := os.MkdirAll(shareDir, 0755) @@ -104,12 +109,26 @@ func InitContext(id string, hub chan VmEvent, client chan *types.VmResponse, dc } } + if boot.EnableVsock { + if !HDriver.SupportVmSocket() { + err := fmt.Errorf("vsock feature requested but not supported") + ctx.Log(ERROR, "%v", err) + return nil, err + } + cid, err = VsockCidManager.GetCid() + if err != nil { + ctx.Log(ERROR, "failed to get vsock guest cid: %v", err) + return nil, err + } + } + ctx = &VmContext{ Id: id, Boot: boot, PauseState: PauseStateUnpaused, pciAddr: PciAddrFrom, scsiId: 0, + GuestCid: cid, Hub: hub, client: client, DCtx: dc, @@ -172,7 +191,7 @@ func (ctx *VmContext) nextScsiId() int { return id } -func (ctx *VmContext) nextPciAddr() int { +func (ctx *VmContext) NextPciAddr() int { ctx.idLock.Lock() addr := ctx.pciAddr ctx.pciAddr++ @@ -243,6 +262,10 @@ func (ctx *VmContext) Close() { os.Remove(ctx.ShareDir) ctx.handler = nil ctx.current = "None" + if ctx.Boot.EnableVsock && ctx.GuestCid > 0 { + VsockCidManager.ReleaseCid(ctx.GuestCid) + ctx.GuestCid = 0 + } } func (ctx *VmContext) Become(handler stateHandler, desc string) { @@ -410,3 +433,19 @@ func (ctx *VmContext) RemoveVolume(name string, result chan<- api.Result) { delete(ctx.volumes, name) disk.remove(result) } + +func (ctx *VmContext) ctlSockAddr() string { + if ctx.Boot.EnableVsock { + return utils.VSOCK_SOCKET_PREFIX + strconv.FormatUint(uint64(ctx.GuestCid), 10) + ":" + strconv.FormatInt(hyperstartapi.HYPER_VSOCK_CTL_PORT, 10) + } else { + return utils.UNIX_SOCKET_PREFIX + ctx.HyperSockName + } +} + +func (ctx *VmContext) ttySockAddr() string { + if ctx.Boot.EnableVsock { + return utils.VSOCK_SOCKET_PREFIX + strconv.FormatUint(uint64(ctx.GuestCid), 10) + ":" + strconv.FormatInt(hyperstartapi.HYPER_VSOCK_MSG_PORT, 10) + } else { + return utils.UNIX_SOCKET_PREFIX + ctx.TtySockName + } +} diff --git a/hypervisor/driver.go b/hypervisor/driver.go index 60df33a1..c4e01eaa 100644 --- a/hypervisor/driver.go +++ b/hypervisor/driver.go @@ -7,6 +7,7 @@ import ( "github.com/hyperhq/runv/api" "github.com/hyperhq/runv/hypervisor/network" "github.com/hyperhq/runv/hypervisor/types" + "github.com/hyperhq/runv/lib/vsock" ) type BootConfig struct { @@ -15,6 +16,7 @@ type BootConfig struct { HotAddCpuMem bool BootToBeTemplate bool BootFromTemplate bool + EnableVsock bool MemoryPath string DevicesStatePath string Kernel string @@ -57,9 +59,11 @@ type HypervisorDriver interface { InitNetwork(bIface, bIP string, disableIptables bool) error SupportLazyMode() bool + SupportVmSocket() bool } var HDriver HypervisorDriver +var VsockCidManager vsock.VsockCidAllocator type DriverContext interface { Launch(ctx *VmContext) @@ -127,6 +131,10 @@ func (ed *EmptyDriver) SupportLazyMode() bool { return false } +func (ed *EmptyDriver) SupportVmSocket() bool { + return false +} + func (ec *EmptyContext) Launch(ctx *VmContext) {} func (ec *EmptyContext) Associate(ctx *VmContext) {} diff --git a/hypervisor/hypervisor.go b/hypervisor/hypervisor.go index eecf8f49..eaf92019 100644 --- a/hypervisor/hypervisor.go +++ b/hypervisor/hypervisor.go @@ -78,10 +78,10 @@ func (ctx *VmContext) Launch() { if ctx.Boot.BootFromTemplate { glog.Info("boot from template") ctx.PauseState = PauseStatePaused - ctx.hyperstart = libhyperstart.NewSerialJsonBasedHyperstart(ctx.HyperSockName, ctx.TtySockName, 1, false) + ctx.hyperstart = libhyperstart.NewJsonBasedHyperstart(ctx.ctlSockAddr(), ctx.ttySockAddr(), 1, false) ctx.Hub <- &InitConnectedEvent{} } else { - ctx.hyperstart = libhyperstart.NewSerialJsonBasedHyperstart(ctx.HyperSockName, ctx.TtySockName, 1, true) + ctx.hyperstart = libhyperstart.NewJsonBasedHyperstart(ctx.ctlSockAddr(), ctx.ttySockAddr(), 1, true) go ctx.watchHyperstart(true) } if glog.V(1) { @@ -112,7 +112,7 @@ func VmAssociate(vmId string, hub chan VmEvent, client chan *types.VmResponse, p return nil, err } - context.hyperstart = libhyperstart.NewSerialJsonBasedHyperstart(context.HyperSockName, context.TtySockName, pinfo.HwStat.AttachId, false) + context.hyperstart = libhyperstart.NewJsonBasedHyperstart(context.ctlSockAddr(), context.ttySockAddr(), pinfo.HwStat.AttachId, false) context.DCtx.Associate(context) if glog.V(1) { diff --git a/hypervisor/libvirt/libvirt.go b/hypervisor/libvirt/libvirt.go index 455a69b9..dee3e35d 100644 --- a/hypervisor/libvirt/libvirt.go +++ b/hypervisor/libvirt/libvirt.go @@ -23,7 +23,8 @@ var LibvirtdAddress = "qemu:///system" type LibvirtDriver struct { sync.Mutex - conn libvirtgo.VirConnection + conn libvirtgo.VirConnection + hasVsock bool } type LibvirtContext struct { @@ -40,8 +41,15 @@ func InitDriver() *LibvirtDriver { return nil } + var hasVsock bool + _, err = exec.Command("/sbin/modprobe", "vhost_vsock").Output() + if err == nil { + hasVsock = true + } + return &LibvirtDriver{ - conn: conn, + conn: conn, + hasVsock: hasVsock, } } @@ -80,6 +88,10 @@ func (ld *LibvirtDriver) SupportLazyMode() bool { return false } +func (ld *LibvirtDriver) SupportVmSocket() bool { + return ld.hasVsock +} + func (ld *LibvirtDriver) checkConnection() error { if alive, _ := ld.conn.IsAlive(); !alive { glog.V(1).Info("libvirt disconnected, reconnect") @@ -356,21 +368,31 @@ type seclab struct { Type string `xml:"type,attr"` } +type qemucmd struct { + Value string `xml:"value,attr"` +} + +type commandlines struct { + Cmds []qemucmd `xml:"qemu:arg"` +} + type domain struct { - XMLName xml.Name `xml:"domain"` - Type string `xml:"type,attr"` - Name string `xml:"name"` - Memory memory `xml:"memory"` - MaxMem *maxmem `xml:"maxMemory,omitempty"` - VCpu vcpu `xml:"vcpu"` - OS domainos `xml:"os"` - Features features `xml:"features"` - CPU cpu `xml:"cpu"` - OnPowerOff string `xml:"on_poweroff"` - OnReboot string `xml:"on_reboot"` - OnCrash string `xml:"on_crash"` - Devices device `xml:"devices"` - SecLabel seclab `xml:"seclabel"` + XMLName xml.Name `xml:"domain"` + Type string `xml:"type,attr"` + XmlnsQemu string `xml:"xmlns:qemu,attr,omitempty"` + Name string `xml:"name"` + Memory memory `xml:"memory"` + MaxMem *maxmem `xml:"maxMemory,omitempty"` + VCpu vcpu `xml:"vcpu"` + OS domainos `xml:"os"` + Features features `xml:"features"` + CPU cpu `xml:"cpu"` + OnPowerOff string `xml:"on_poweroff"` + OnReboot string `xml:"on_reboot"` + OnCrash string `xml:"on_crash"` + Devices device `xml:"devices"` + SecLabel seclab `xml:"seclabel"` + CommandLine commandlines `xml:"qemu:commandline"` } func (lc *LibvirtContext) domainXml(ctx *hypervisor.VmContext) (string, error) { @@ -428,6 +450,14 @@ func (lc *LibvirtContext) domainXml(ctx *hypervisor.VmContext) (string, error) { dom.CPU.Numa = &numa{Cell: cells} } + if ctx.Boot.EnableVsock { + dom.XmlnsQemu = "http://libvirt.org/schemas/domain/qemu/1.0" + dom.CommandLine.Cmds = append(dom.CommandLine.Cmds, qemucmd{Value: "-device"}) + vsockDev := fmt.Sprintf("vhost-vsock-pci,id=vsock0,bus=pci.0,addr=%x,guest-cid=%d", ctx.NextPciAddr(), ctx.GuestCid) + dom.CommandLine.Cmds = append(dom.CommandLine.Cmds, qemucmd{Value: vsockDev}) + glog.Infof("vsock cmds xml: %v", dom.CommandLine) + } + cmd, err := exec.LookPath("qemu-system-x86_64") if err != nil { return "", fmt.Errorf("cannot find qemu-system-x86_64 binary") diff --git a/hypervisor/network.go b/hypervisor/network.go index b80a3016..f59d71ec 100644 --- a/hypervisor/network.go +++ b/hypervisor/network.go @@ -118,7 +118,7 @@ func (nc *NetworkContext) addInterface(inf *api.InterfaceDescription, result cha return } - nc.configureInterface(idx, nc.sandbox.nextPciAddr(), fmt.Sprintf("eth%d", idx), inf, devChan) + nc.configureInterface(idx, nc.sandbox.NextPciAddr(), fmt.Sprintf("eth%d", idx), inf, devChan) }() go func() { diff --git a/hypervisor/persistence.go b/hypervisor/persistence.go index 6aa4e60c..691e8f6c 100644 --- a/hypervisor/persistence.go +++ b/hypervisor/persistence.go @@ -2,6 +2,7 @@ package hypervisor import ( "encoding/json" + "fmt" "github.com/golang/glog" hyperstartapi "github.com/hyperhq/runv/hyperstart/api/json" @@ -92,12 +93,21 @@ func (ctx *VmContext) dumpHwInfo() *VmHwStatus { PciAddr: ctx.pciAddr, ScsiId: ctx.scsiId, AttachId: ctx.hyperstart.LastStreamSeq(), + GuestCid: ctx.GuestCid, } } -func (ctx *VmContext) loadHwStatus(pinfo *PersistInfo) { +func (ctx *VmContext) loadHwStatus(pinfo *PersistInfo) error { ctx.pciAddr = pinfo.HwStat.PciAddr ctx.scsiId = pinfo.HwStat.ScsiId + ctx.GuestCid = pinfo.HwStat.GuestCid + if ctx.GuestCid != 0 { + if !VsockCidManager.MarkCidInuse(ctx.GuestCid) { + return fmt.Errorf("conflicting vsock guest cid %d: already in use", ctx.GuestCid) + } + ctx.Boot.EnableVsock = true + } + return nil } func (blk *DiskDescriptor) dump() *PersistVolumeInfo { @@ -149,7 +159,10 @@ func (pinfo *PersistInfo) vmContext(hub chan VmEvent, client chan *types.VmRespo //ctx.userSpec = pinfo.UserSpec //ctx.wg = wg - ctx.loadHwStatus(pinfo) + err = ctx.loadHwStatus(pinfo) + if err != nil { + return nil, err + } //for _, vol := range pinfo.VolumeList { // binfo := vol.blockInfo() diff --git a/hypervisor/qemu/qemu.go b/hypervisor/qemu/qemu.go index 79e2e95e..8ec38583 100644 --- a/hypervisor/qemu/qemu.go +++ b/hypervisor/qemu/qemu.go @@ -17,6 +17,7 @@ import ( //implement the hypervisor.HypervisorDriver interface type QemuDriver struct { executable string + hasVsock bool } //implement the hypervisor.DriverContext interface @@ -42,8 +43,15 @@ func InitDriver() *QemuDriver { return nil } + var hasVsock bool + _, err = exec.Command("/sbin/modprobe", "vhost_vsock").Output() + if err == nil { + hasVsock = true + } + return &QemuDriver{ executable: cmd, + hasVsock: hasVsock, } } @@ -345,3 +353,7 @@ func (qc *QemuContext) Save(ctx *hypervisor.VmContext, path string) error { func (qc *QemuDriver) SupportLazyMode() bool { return false } + +func (qc *QemuDriver) SupportVmSocket() bool { + return qc.hasVsock +} diff --git a/hypervisor/qemu/qemu_process.go b/hypervisor/qemu/qemu_process.go index 31f75627..9b4a1dda 100644 --- a/hypervisor/qemu/qemu_process.go +++ b/hypervisor/qemu/qemu_process.go @@ -126,6 +126,11 @@ func launchQemu(qc *QemuContext, ctx *hypervisor.VmContext) { args := qc.arguments(ctx) args = append(args, "-daemonize", "-pidfile", qc.qemuPidFile, "-D", qc.qemuLogFile.Name) + if ctx.Boot.EnableVsock && qc.driver.hasVsock && ctx.GuestCid > 0 { + addr := ctx.NextPciAddr() + vsockDev := fmt.Sprintf("vhost-vsock-pci,id=vsock0,bus=pci.0,addr=%x,guest-cid=%d", addr, ctx.GuestCid) + args = append(args, "-device", vsockDev) + } if glog.V(1) { glog.Info("cmdline arguments: ", strings.Join(args, " ")) diff --git a/hypervisor/vbox/vbox.go b/hypervisor/vbox/vbox.go index 633f36b5..c6a3c7bc 100644 --- a/hypervisor/vbox/vbox.go +++ b/hypervisor/vbox/vbox.go @@ -443,6 +443,10 @@ func (vc *VBoxDriver) SupportLazyMode() bool { return true } +func (vc *VBoxDriver) SupportVmSocket() bool { + return false +} + func scsiId2Name(id int) string { return "sd" + utils.DiskId2Name(id) } diff --git a/hypervisor/xen/xen.go b/hypervisor/xen/xen.go index 000b89fc..fe18cf3c 100644 --- a/hypervisor/xen/xen.go +++ b/hypervisor/xen/xen.go @@ -300,6 +300,10 @@ func (xd *XenDriver) SupportLazyMode() bool { return false } +func (xd *XenDriver) SupportVmSocket() bool { + return false +} + func diskRoutine(add bool, xc *XenContext, ctx *hypervisor.VmContext, name, sourceType, filename, format string, id int, callback hypervisor.VmEvent, result chan<- hypervisor.VmEvent) { backend := LIBXL_DISK_BACKEND_TAP diff --git a/lib/utils/utils.go b/lib/utils/utils.go index 1c2347eb..12074408 100644 --- a/lib/utils/utils.go +++ b/lib/utils/utils.go @@ -2,8 +2,18 @@ package utils import ( "crypto/rand" + "fmt" "net" + "strconv" + "strings" "time" + + "github.com/hyperhq/runv/lib/vsock" +) + +const ( + UNIX_SOCKET_PREFIX = "unix://" + VSOCK_SOCKET_PREFIX = "vsock://" ) func DiskId2Name(id int) string { @@ -14,10 +24,46 @@ func DiskId2Name(id int) string { return DiskId2Name(id/26-1) + string(ch) } -func UnixSocketConnect(name string) (conn net.Conn, err error) { +func SocketConnect(addr string) (net.Conn, error) { + switch { + case strings.HasPrefix(addr, UNIX_SOCKET_PREFIX): + return UnixSocketConnect(addr[len(UNIX_SOCKET_PREFIX):]) + case strings.HasPrefix(addr, VSOCK_SOCKET_PREFIX): + return vmSocketConnect(addr[len(VSOCK_SOCKET_PREFIX):]) + default: + return nil, fmt.Errorf("unsupported destination: %s", addr) + } +} + +func UnixSocketConnect(addr string) (conn net.Conn, err error) { + for i := 0; i < 500; i++ { + time.Sleep(20 * time.Millisecond) + conn, err = net.Dial("unix", addr) + if err == nil { + return + } + } + + return +} + +func vmSocketConnect(addr string) (conn net.Conn, err error) { + seq := strings.Split(addr, ":") + if len(seq) != 2 { + return nil, fmt.Errorf("invalid vsock destination: %v", VSOCK_SOCKET_PREFIX+addr) + } + cid, err := strconv.ParseUint(seq[0], 10, 32) + if err != nil { + return nil, fmt.Errorf("invalid vsock destination: %v", VSOCK_SOCKET_PREFIX+addr) + } + port, err := strconv.ParseUint(seq[1], 10, 32) + if err != nil { + return nil, fmt.Errorf("invalid vsock destination: %v", VSOCK_SOCKET_PREFIX+addr) + } + for i := 0; i < 500; i++ { time.Sleep(20 * time.Millisecond) - conn, err = net.Dial("unix", name) + conn, err = vsock.Dial(uint32(cid), uint32(port)) if err == nil { return } diff --git a/lib/vsock/vsock.go b/lib/vsock/vsock.go new file mode 100644 index 00000000..9a2d4e64 --- /dev/null +++ b/lib/vsock/vsock.go @@ -0,0 +1,68 @@ +// +build linux + +package vsock + +import ( + "fmt" + "sync" + + "github.com/RoaringBitmap/roaring" +) + +const hyperDefaultVsockCid = 1024 +const hyperDefaultVsockBitmapSize = 16384 + +type VsockCidAllocator interface { + sync.Locker + GetCid() (uint32, error) + MarkCidInuse(uint32) bool + ReleaseCid(uint32) +} + +type DefaultVsockCidAllocator struct { + sync.Mutex + bitmap *roaring.Bitmap + start uint32 + size uint32 + pivot uint32 +} + +func NewDefaultVsockCidAllocator() VsockCidAllocator { + return &DefaultVsockCidAllocator{ + bitmap: roaring.NewBitmap(), + start: hyperDefaultVsockCid, + size: hyperDefaultVsockBitmapSize, + pivot: hyperDefaultVsockCid, + } +} + +func (vc *DefaultVsockCidAllocator) GetCid() (uint32, error) { + var cid uint32 + vc.Lock() + defer vc.Unlock() + for i := uint32(0); i < vc.size; i++ { + cid = vc.pivot + i + if cid >= vc.start+vc.size { + cid -= vc.size + } + if vc.bitmap.CheckedAdd(cid) { + vc.pivot = cid + 1 + return cid, nil + } + } + + return cid, fmt.Errorf("No more available cid") +} + +func (vc *DefaultVsockCidAllocator) MarkCidInuse(cid uint32) bool { + vc.Lock() + defer vc.Unlock() + success := vc.bitmap.CheckedAdd(cid) + return success +} + +func (vc *DefaultVsockCidAllocator) ReleaseCid(cid uint32) { + vc.Lock() + defer vc.Unlock() + vc.bitmap.Remove(cid) +} diff --git a/lib/vsock/vsock_conn.go b/lib/vsock/vsock_conn.go new file mode 100644 index 00000000..ff47fd85 --- /dev/null +++ b/lib/vsock/vsock_conn.go @@ -0,0 +1,232 @@ +// Implementation of the net.Conn interface for VsockConn +// +// +build linux + +package vsock + +import ( + "fmt" + "io" + "net" + "os" + "sync" + "syscall" + "time" + + "golang.org/x/sys/unix" +) + +const VsockNetwork = "vsock" + +type VsockNetAddr struct { + cid uint32 + port uint32 +} + +func (va *VsockNetAddr) Network() string { + return VsockNetwork +} + +func (va *VsockNetAddr) String() string { + return fmt.Sprintf("vsock://%d:%d", va.cid, va.port) +} + +type VsockConn struct { + sysFd int + readLock sync.Mutex + writeLock sync.Mutex + + net string + laddr, raddr net.Addr +} + +func newVsockConn(fd int, src, dst *unix.SockaddrVsock) *VsockConn { + return &VsockConn{ + sysFd: fd, + net: VsockNetwork, + laddr: &VsockNetAddr{src.Cid, src.Port}, + raddr: &VsockNetAddr{dst.Cid, dst.Port}, + } +} + +func (c *VsockConn) ready() bool { return c != nil && c.sysFd != 0 } + +func (c *VsockConn) Read(b []byte) (int, error) { + var ( + err error + n int + ) + + if !c.ready() { + return 0, syscall.EINVAL + } + + c.readLock.Lock() + defer c.readLock.Unlock() + for { + n, err = syscall.Read(c.sysFd, b) + if err != nil { + n = 0 + if err != syscall.EAGAIN { + break + } + } + err = io.EOF + break + } + if err != nil && err != io.EOF { + return 0, &net.OpError{Op: "read", Net: c.net, Source: c.laddr, Addr: c.raddr, Err: err} + } + return n, nil +} + +func (c *VsockConn) Write(b []byte) (int, error) { + var ( + count, n int + err error + ) + + if !c.ready() { + return 0, syscall.EINVAL + } + + c.writeLock.Lock() + defer c.writeLock.Unlock() + for { + n, err = syscall.Write(c.sysFd, b) + if n > 0 { + count += n + } + if count == len(b) { + break + } + + if err == syscall.EAGAIN { + continue + } else if err != nil { + break + } + if n == 0 { + err = io.ErrUnexpectedEOF + break + } + } + if err != nil { + err = &net.OpError{Op: "write", Net: c.net, Source: c.laddr, Addr: c.raddr, Err: err} + } + return count, err +} + +func (c *VsockConn) Close() error { + if !c.ready() { + return syscall.EINVAL + } + err := syscall.Close(c.sysFd) + if err != nil { + err = &net.OpError{Op: "close", Net: c.net, Source: c.laddr, Addr: c.raddr, Err: err} + } + return err +} + +func (c *VsockConn) LocalAddr() net.Addr { + if !c.ready() { + return nil + } + return c.laddr +} + +func (c *VsockConn) RemoteAddr() net.Addr { + if !c.ready() { + return nil + } + return c.raddr +} + +func (c *VsockConn) SetDeadline(t time.Time) error { + if !c.ready() { + return syscall.EINVAL + } + return c.setDeadlineImpl(t, 'r'+'w') +} + +func (c *VsockConn) SetReadDeadline(t time.Time) error { + if !c.ready() { + return syscall.EINVAL + } + return c.setDeadlineImpl(t, 'r') +} + +func (c *VsockConn) SetWriteDeadline(t time.Time) error { + if !c.ready() { + return syscall.EINVAL + } + return c.setDeadlineImpl(t, 'w') +} + +func (c *VsockConn) File() (f *os.File, err error) { + defer func() { + if err != nil { + err = &net.OpError{Op: "file", Net: c.net, Source: c.laddr, Addr: c.laddr, Err: err} + } + }() + + fd, err := syscall.Dup(c.sysFd) + if err != nil { + return + } + syscall.CloseOnExec(fd) + if err = syscall.SetNonblock(fd, false); err != nil { + return + } + + return os.NewFile(uintptr(fd), c.name()), nil +} + +func (c *VsockConn) name() string { + var l, r string + if c.laddr != nil { + l = c.laddr.String() + } + if c.raddr != nil { + r = c.raddr.String() + } + return c.net + ":" + l + "->" + r +} + +func (c *VsockConn) setDeadlineImpl(t time.Time, mode int) error { + switch mode { + case 'r': + case 'w': + case 'r' + 'w': + } + return syscall.EINVAL +} + +func Dial(cid uint32, port uint32) (net.Conn, error) { + fd, err := unix.Socket(unix.AF_VSOCK, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0) + if err != nil { + return nil, err + } + unix.SetNonblock(fd, false) + + dst := &unix.SockaddrVsock{Cid: cid, Port: port} + err = unix.Connect(fd, dst) + if err != nil { + unix.Close(fd) + return nil, err + } + + sa, err := unix.Getsockname(fd) + if err != nil { + unix.Close(fd) + return nil, err + } + + src, ok := sa.(*unix.SockaddrVsock) + if !ok { + unix.Close(fd) + return nil, fmt.Errorf("failed to make vsock connection") + } + + return newVsockConn(fd, src, dst), nil +} diff --git a/lib/vsock/vsock_unsupported.go b/lib/vsock/vsock_unsupported.go new file mode 100644 index 00000000..12e77b76 --- /dev/null +++ b/lib/vsock/vsock_unsupported.go @@ -0,0 +1,12 @@ +// +build darwin dragonfly freebsd netbsd openbsd solaris + +package vsock + +import ( + "fmt" + "net" +) + +func Dial(cid uint32, port uint32) (net.Conn, error) { + return nil, fmt.Errorf("vsock is not supported") +} diff --git a/manage.go b/manage.go index b5810f7c..87bec72e 100644 --- a/manage.go +++ b/manage.go @@ -80,7 +80,7 @@ var createTemplateCommand = cli.Command{ os.Exit(-1) } - if _, err := templatecore.CreateTemplateVM(template, "", context.Int("cpu"), context.Int("mem"), kernel, initrd); err != nil { + if _, err := templatecore.CreateTemplateVM(template, "", context.Int("cpu"), context.Int("mem"), kernel, initrd, context.GlobalBool("vsock")); err != nil { fmt.Printf("Failed to create the template: %v\n", err) os.Exit(-1) } diff --git a/template/template.go b/template/template.go index 7de27284..6d7e5388 100644 --- a/template/template.go +++ b/template/template.go @@ -32,7 +32,7 @@ type TemplateVmConfig struct { Initrd string `json:"initrd"` } -func CreateTemplateVM(statePath, vmName string, cpu, mem int, kernel, initrd string) (t *TemplateVmConfig, err error) { +func CreateTemplateVM(statePath, vmName string, cpu, mem int, kernel, initrd string, vsock bool) (t *TemplateVmConfig, err error) { defer func() { if err != nil { (&TemplateVmConfig{StatePath: statePath}).Destroy() @@ -64,6 +64,7 @@ func CreateTemplateVM(statePath, vmName string, cpu, mem int, kernel, initrd str HotAddCpuMem: true, BootToBeTemplate: true, BootFromTemplate: false, + EnableVsock: vsock, MemoryPath: statePath + "/memory", DevicesStatePath: statePath + "/state", Kernel: kernel, diff --git a/vendor/github.com/RoaringBitmap/roaring/.gitignore b/vendor/github.com/RoaringBitmap/roaring/.gitignore new file mode 100644 index 00000000..f2e6367e --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/.gitignore @@ -0,0 +1,3 @@ +*~ +roaring-fuzz.zip +workdir diff --git a/vendor/github.com/RoaringBitmap/roaring/.travis.yml b/vendor/github.com/RoaringBitmap/roaring/.travis.yml new file mode 100644 index 00000000..1d018fc2 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/.travis.yml @@ -0,0 +1,20 @@ +language: go +sudo: false +install: +- go get github.com/smartystreets/goconvey/convey +- go get github.com/willf/bitset +- go get golang.org/x/tools/cmd/cover +- go get github.com/mattn/goveralls +- go get github.com/mschoch/smat +notifications: + email: false +go: +- 1.6 +- 1.7 +- tip +script: +- go test -v -covermode=count -coverprofile=coverage.out +- $HOME/gopath/bin/goveralls -coverprofile=coverage.out -service=travis-ci -repotoken KOlKyOXz0qSjAqvfTF28RzNlr3unxjrLh +matrix: + allow_failures: + - go: tip diff --git a/vendor/github.com/RoaringBitmap/roaring/AUTHORS b/vendor/github.com/RoaringBitmap/roaring/AUTHORS new file mode 100644 index 00000000..6c67f925 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/AUTHORS @@ -0,0 +1,9 @@ +# This is the official list of roaring authors for copyright purposes. + +Todd Gruben (@tgruben), +Daniel Lemire (@lemire), +Elliot Murphy (@statik), +Bob Potter (@bpot), +Tyson Maly (@tvmaly), +Will Glynn (@willglynn), +Brent Pedersen (@brentp) diff --git a/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS b/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS new file mode 100644 index 00000000..3d3e7e74 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS @@ -0,0 +1,9 @@ +# This is the official list of roaring contributors + +Todd Gruben (@tgruben), +Daniel Lemire (@lemire), +Elliot Murphy (@statik), +Bob Potter (@bpot), +Tyson Maly (@tvmaly), +Will Glynn (@willglynn), +Brent Pedersen (@brentp) diff --git a/vendor/github.com/RoaringBitmap/roaring/LICENSE b/vendor/github.com/RoaringBitmap/roaring/LICENSE new file mode 100644 index 00000000..aff5f999 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016 by the authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/RoaringBitmap/roaring/LICENSE-2.0.txt b/vendor/github.com/RoaringBitmap/roaring/LICENSE-2.0.txt new file mode 100644 index 00000000..aff5f999 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/LICENSE-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016 by the authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/RoaringBitmap/roaring/README.md b/vendor/github.com/RoaringBitmap/roaring/README.md new file mode 100644 index 00000000..9a238879 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/README.md @@ -0,0 +1,199 @@ +roaring [![Build Status](https://travis-ci.org/RoaringBitmap/roaring.png)](https://travis-ci.org/RoaringBitmap/roaring) [![Coverage Status](https://coveralls.io/repos/github/RoaringBitmap/roaring/badge.svg?branch=master)](https://coveralls.io/github/RoaringBitmap/roaring?branch=master) [![GoDoc](https://godoc.org/github.com/RoaringBitmap/roaring?status.svg)](https://godoc.org/github.com/RoaringBitmap/roaring) [![Go Report Card](https://goreportcard.com/badge/RoaringBitmap/roaring)](https://goreportcard.com/report/github.com/RoaringBitmap/roaring) +============= + +This is a go port of the Roaring bitmap data structure. + +Roaring is used by Apache Spark (https://spark.apache.org/), Apache Kylin (http://kylin.io), +Druid.io (http://druid.io/), Whoosh (https://pypi.python.org/pypi/Whoosh/) +and Apache Lucene (http://lucene.apache.org/) (as well as supporting systems +such as Solr and Elastic). + +The original java version can be found at https://github.com/RoaringBitmap/RoaringBitmap + +The Java and Go version are meant to be binary compatible: you can save bitmaps +from a Java program and load them back in Go, and vice versa. + + +This code is licensed under Apache License, Version 2.0 (ASL2.0). + +Copyright 2016 by the authors. + + +### References + +- Samy Chambi, Daniel Lemire, Owen Kaser, Robert Godin, +Better bitmap performance with Roaring bitmaps, +Software: Practice and Experience Volume 46, Issue 5, pages 709–719, May 2016 +http://arxiv.org/abs/1402.6407 This paper used data from http://lemire.me/data/realroaring2014.html +- Daniel Lemire, Gregory Ssi-Yan-Kai, Owen Kaser, Consistently faster and smaller compressed bitmaps with Roaring, Software: Practice and Experience (accepted in 2016, to appear) http://arxiv.org/abs/1603.06549 + + + +### Dependencies + + - go get github.com/smartystreets/goconvey/convey + - go get github.com/willf/bitset + - go get github.com/mschoch/smat + +Note that the smat library requires Go 1.6 or better. + +Naturally, you also need to grab the roaring code itself: + - go get github.com/RoaringBitmap/roaring + + +### Example + +Here is a simplified but complete example: + +```go +package main + +import ( + "fmt" + "github.com/RoaringBitmap/roaring" + "bytes" +) + + +func main() { + // example inspired by https://github.com/fzandona/goroar + fmt.Println("==roaring==") + rb1 := roaring.BitmapOf(1, 2, 3, 4, 5, 100, 1000) + fmt.Println(rb1.String()) + + rb2 := roaring.BitmapOf(3, 4, 1000) + fmt.Println(rb2.String()) + + rb3 := roaring.NewBitmap() + fmt.Println(rb3.String()) + + fmt.Println("Cardinality: ", rb1.GetCardinality()) + + fmt.Println("Contains 3? ", rb1.Contains(3)) + + rb1.And(rb2) + + rb3.Add(1) + rb3.Add(5) + + rb3.Or(rb1) + + // prints 1, 3, 4, 5, 1000 + i := rb3.Iterator() + for i.HasNext() { + fmt.Println(i.Next()) + } + fmt.Println() + + // next we include an example of serialization + buf := new(bytes.Buffer) + rb1.WriteTo(buf) // we omit error handling + newrb:= roaring.NewBitmap() + newrb.ReadFrom(buf) + if rb1.Equals(newrb) { + fmt.Println("I wrote the content to a byte stream and read it back.") + } +} +``` + +If you wish to use serialization and handle errors, you might want to +consider the following sample of code: + +```go + rb := BitmapOf(1, 2, 3, 4, 5, 100, 1000) + buf := new(bytes.Buffer) + size,err:=rb.WriteTo(buf) + if err != nil { + t.Errorf("Failed writing") + } + newrb:= NewBitmap() + size,err=newrb.ReadFrom(buf) + if err != nil { + t.Errorf("Failed reading") + } + if ! rb.Equals(newrb) { + t.Errorf("Cannot retrieve serialized version") + } +``` + +Given N integers in [0,x), then the serialized size in bytes of +a Roaring bitmap should never exceed this bound: + +`` 8 + 9 * ((long)x+65535)/65536 + 2 * N `` + +That is, given a fixed overhead for the universe size (x), Roaring +bitmaps never use more than 2 bytes per integer. You can call +``BoundSerializedSizeInBytes`` for a more precise estimate. + + +### Documentation + +Current documentation is available at http://godoc.org/github.com/RoaringBitmap/roaring + +### Thread-safety + +In general, it should generally be considered safe to access +the same bitmaps using different threads as +long as they are not being modified. However, if some of your +bitmaps use copy-on-write, then more care is needed: pass +to your threads a (shallow) copy of your bitmaps. + +### Coverage + +We test our software. For a report on our test coverage, see + +https://coveralls.io/github/RoaringBitmap/roaring?branch=master + +### Benchmark + +Type + + go test -bench Benchmark -run - + +### Iterative use + +You can use roaring with gore: + +- go get -u github.com/motemen/gore +- Make sure that ``$GOPATH/bin`` is in your ``$PATH``. +- go get github/RoaringBitmap/roaring + +```go +$ gore +gore version 0.2.6 :help for help +gore> :import github.com/RoaringBitmap/roaring +gore> x:=roaring.New() +gore> x.Add(1) +gore> x.String() +"{1}" +``` + + +### Fuzzy testing + +You can help us test further the library with fuzzy testing: + + go get github.com/dvyukov/go-fuzz/go-fuzz + go get github.com/dvyukov/go-fuzz/go-fuzz-build + go test -tags=gofuzz -run=TestGenerateSmatCorpus + go-fuzz-build github.com/RoaringBitmap/roaring + go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200 + +Let it run, and if the # of crashers is > 0, check out the reports in +the workdir where you should be able to find the panic goroutine stack +traces. + +### Compatibility with Java RoaringBitmap library + +You can read bitmaps in Go (resp. Java) that have been serialized in Java (resp. Go) +with the caveat that the Go library does not yet support run containers. So if you plan +to read bitmaps serialized from Java in Go, you might want to call ``removeRunCompression`` +prior to serializing your Java instances. This is a temporary limitation: we plan to +add support for run containers to the Go library. + +### Alternative in Go + +There is a Go version wrapping the C/C++ implementation https://github.com/RoaringBitmap/gocroaring + +For an alternative implementation in Go, see https://github.com/fzandona/goroar +The two versions were written independently. diff --git a/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go b/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go new file mode 100644 index 00000000..2b77b8e5 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go @@ -0,0 +1,658 @@ +package roaring + +import ( + "unsafe" +) + +type arrayContainer struct { + content []uint16 +} + +func (ac *arrayContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) { + for k := 0; k < len(ac.content); k++ { + x[k+i] = uint32(ac.content[k]) | mask + } +} + +func (ac *arrayContainer) getShortIterator() shortIterable { + return &shortIterator{ac.content, 0} +} + +func (ac *arrayContainer) getSizeInBytes() int { + // unsafe.Sizeof calculates the memory used by the top level of the slice + // descriptor - not including the size of the memory referenced by the slice. + // http://golang.org/pkg/unsafe/#Sizeof + return ac.getCardinality()*2 + int(unsafe.Sizeof(ac.content)) +} + +func (ac *arrayContainer) serializedSizeInBytes() int { + // based on https://golang.org/src/pkg/encoding/binary/binary.go#265 + // there is no serialization overhead for writing an array of fixed size vals + return ac.getCardinality() * 2 +} + +// add the values in the range [firstOfRange,lastofRange) +func (ac *arrayContainer) iaddRange(firstOfRange, lastOfRange int) container { + if firstOfRange >= lastOfRange { + return ac + } + indexstart := binarySearch(ac.content, uint16(firstOfRange)) + if indexstart < 0 { + indexstart = -indexstart - 1 + } + indexend := binarySearch(ac.content, uint16(lastOfRange-1)) + if indexend < 0 { + indexend = -indexend - 1 + } else { + indexend++ + } + rangelength := lastOfRange - firstOfRange + newcardinality := indexstart + (ac.getCardinality() - indexend) + rangelength + if newcardinality > arrayDefaultMaxSize { + a := ac.toBitmapContainer() + return a.iaddRange(firstOfRange, lastOfRange) + } + if cap(ac.content) < newcardinality { + tmp := make([]uint16, newcardinality, newcardinality) + copy(tmp[:indexstart], ac.content[:indexstart]) + copy(tmp[indexstart+rangelength:], ac.content[indexend:]) + + ac.content = tmp + } else { + ac.content = ac.content[:newcardinality] + copy(ac.content[indexstart+rangelength:], ac.content[indexend:]) + + } + for k := 0; k < rangelength; k++ { + ac.content[k+indexstart] = uint16(firstOfRange + k) + } + return ac +} + +// remove the values in the range [firstOfRange,lastOfRange) +func (ac *arrayContainer) iremoveRange(firstOfRange, lastOfRange int) container { + if firstOfRange >= lastOfRange { + return ac + } + indexstart := binarySearch(ac.content, uint16(firstOfRange)) + if indexstart < 0 { + indexstart = -indexstart - 1 + } + indexend := binarySearch(ac.content, uint16(lastOfRange-1)) + if indexend < 0 { + indexend = -indexend - 1 + } else { + indexend++ + } + rangelength := indexend - indexstart + answer := ac + copy(answer.content[indexstart:], ac.content[indexstart+rangelength:]) + answer.content = answer.content[:ac.getCardinality()-rangelength] + return answer +} + +// flip the values in the range [firstOfRange,lastOfRange) +func (ac *arrayContainer) not(firstOfRange, lastOfRange int) container { + if firstOfRange >= lastOfRange { + return ac.clone() + } + return ac.notClose(firstOfRange, lastOfRange-1) // remove everything in [firstOfRange,lastOfRange-1] +} + +// flip the values in the range [firstOfRange,lastOfRange] +func (ac *arrayContainer) notClose(firstOfRange, lastOfRange int) container { + if firstOfRange > lastOfRange { // unlike add and remove, not uses an inclusive range [firstOfRange,lastOfRange] + return ac.clone() + } + + // determine the span of array indices to be affected^M + startIndex := binarySearch(ac.content, uint16(firstOfRange)) + if startIndex < 0 { + startIndex = -startIndex - 1 + } + lastIndex := binarySearch(ac.content, uint16(lastOfRange)) + if lastIndex < 0 { + lastIndex = -lastIndex - 2 + } + currentValuesInRange := lastIndex - startIndex + 1 + spanToBeFlipped := lastOfRange - firstOfRange + 1 + newValuesInRange := spanToBeFlipped - currentValuesInRange + cardinalityChange := newValuesInRange - currentValuesInRange + newCardinality := len(ac.content) + cardinalityChange + + if newCardinality > arrayDefaultMaxSize { + return ac.toBitmapContainer().not(firstOfRange, lastOfRange+1) + } + answer := newArrayContainer() + answer.content = make([]uint16, newCardinality, newCardinality) //a hack for sure + + copy(answer.content, ac.content[:startIndex]) + outPos := startIndex + inPos := startIndex + valInRange := firstOfRange + for ; valInRange <= lastOfRange && inPos <= lastIndex; valInRange++ { + if uint16(valInRange) != ac.content[inPos] { + answer.content[outPos] = uint16(valInRange) + outPos++ + } else { + inPos++ + } + } + + for ; valInRange <= lastOfRange; valInRange++ { + answer.content[outPos] = uint16(valInRange) + outPos++ + } + + for i := lastIndex + 1; i < len(ac.content); i++ { + answer.content[outPos] = ac.content[i] + outPos++ + } + answer.content = answer.content[:newCardinality] + return answer + +} + +func (ac *arrayContainer) equals(o interface{}) bool { + srb, ok := o.(*arrayContainer) + if ok { + // Check if the containers are the same object. + if ac == srb { + return true + } + + if len(srb.content) != len(ac.content) { + return false + } + + for i, v := range ac.content { + if v != srb.content[i] { + return false + } + } + return true + } + return false +} + +func (ac *arrayContainer) toBitmapContainer() *bitmapContainer { + bc := newBitmapContainer() + bc.loadData(ac) + return bc + +} +func (ac *arrayContainer) add(x uint16) container { + // Special case adding to the end of the container. + l := len(ac.content) + if l > 0 && l < arrayDefaultMaxSize && ac.content[l-1] < x { + ac.content = append(ac.content, x) + return ac + } + + loc := binarySearch(ac.content, x) + + if loc < 0 { + if len(ac.content) >= arrayDefaultMaxSize { + a := ac.toBitmapContainer() + a.add(x) + return a + } + s := ac.content + i := -loc - 1 + s = append(s, 0) + copy(s[i+1:], s[i:]) + s[i] = x + ac.content = s + } + return ac +} + +func (ac *arrayContainer) remove(x uint16) container { + loc := binarySearch(ac.content, x) + if loc >= 0 { + s := ac.content + s = append(s[:loc], s[loc+1:]...) + ac.content = s + } + return ac +} + +func (ac *arrayContainer) or(a container) container { + switch a.(type) { + case *arrayContainer: + return ac.orArray(a.(*arrayContainer)) + case *bitmapContainer: + return a.or(ac) + } + panic("unsupported container type") +} + +func (ac *arrayContainer) ior(a container) container { + switch a.(type) { + case *arrayContainer: + return ac.orArray(a.(*arrayContainer)) + case *bitmapContainer: + return a.ior(ac) + } + panic("unsupported container type") +} + +func (ac *arrayContainer) lazyIOR(a container) container { + switch a.(type) { + case *arrayContainer: + return ac.lazyorArray(a.(*arrayContainer)) + case *bitmapContainer: + return a.lazyOR(ac) + } + panic("unsupported container type") +} + +func (ac *arrayContainer) lazyOR(a container) container { + switch a.(type) { + case *arrayContainer: + return ac.lazyorArray(a.(*arrayContainer)) + case *bitmapContainer: + return a.lazyOR(ac) + } + panic("unsupported container type") +} + +func (ac *arrayContainer) orArray(value2 *arrayContainer) container { + value1 := ac + maxPossibleCardinality := value1.getCardinality() + value2.getCardinality() + if maxPossibleCardinality > arrayDefaultMaxSize { // it could be a bitmap!^M + bc := newBitmapContainer() + for k := 0; k < len(value2.content); k++ { + v := value2.content[k] + i := uint(v) >> 6 + mask := uint64(1) << (v % 64) + bc.bitmap[i] |= mask + } + for k := 0; k < len(ac.content); k++ { + v := ac.content[k] + i := uint(v) >> 6 + mask := uint64(1) << (v % 64) + bc.bitmap[i] |= mask + } + bc.cardinality = int(popcntSlice(bc.bitmap)) + if bc.cardinality <= arrayDefaultMaxSize { + return bc.toArrayContainer() + } + return bc + } + answer := newArrayContainerCapacity(maxPossibleCardinality) + nl := union2by2(value1.content, value2.content, answer.content) + answer.content = answer.content[:nl] // reslice to match actual used capacity + return answer +} + +func (ac *arrayContainer) lazyorArray(value2 *arrayContainer) container { + value1 := ac + maxPossibleCardinality := value1.getCardinality() + value2.getCardinality() + if maxPossibleCardinality > arrayLazyLowerBound { // it could be a bitmap!^M + bc := newBitmapContainer() + for k := 0; k < len(value2.content); k++ { + v := value2.content[k] + i := uint(v) >> 6 + mask := uint64(1) << (v % 64) + bc.bitmap[i] |= mask + } + for k := 0; k < len(ac.content); k++ { + v := ac.content[k] + i := uint(v) >> 6 + mask := uint64(1) << (v % 64) + bc.bitmap[i] |= mask + } + bc.cardinality = invalidCardinality + return bc + } + answer := newArrayContainerCapacity(maxPossibleCardinality) + nl := union2by2(value1.content, value2.content, answer.content) + answer.content = answer.content[:nl] // reslice to match actual used capacity + return answer +} + +func (ac *arrayContainer) and(a container) container { + switch a.(type) { + case *arrayContainer: + return ac.andArray(a.(*arrayContainer)) + case *bitmapContainer: + return a.and(ac) + } + panic("unsupported container type") +} + +func (ac *arrayContainer) intersects(a container) bool { + switch a.(type) { + case *arrayContainer: + return ac.intersectsArray(a.(*arrayContainer)) + case *bitmapContainer: + return a.intersects(ac) + } + panic("unsupported container type") +} + +func (ac *arrayContainer) iand(a container) container { + switch a.(type) { + case *arrayContainer: + return ac.iandArray(a.(*arrayContainer)) + case *bitmapContainer: + return ac.iandBitmap(a.(*bitmapContainer)) + } + panic("unsupported container type") +} + +func (ac *arrayContainer) iandBitmap(bc *bitmapContainer) *arrayContainer { + pos := 0 + c := ac.getCardinality() + for k := 0; k < c; k++ { + if bc.contains(ac.content[k]) { + ac.content[pos] = ac.content[k] + pos++ + } + } + ac.content = ac.content[:pos] + return ac + +} + +func (ac *arrayContainer) xor(a container) container { + switch a.(type) { + case *arrayContainer: + return ac.xorArray(a.(*arrayContainer)) + case *bitmapContainer: + return a.xor(ac) + } + panic("unsupported container type") +} + +func (ac *arrayContainer) xorArray(value2 *arrayContainer) container { + value1 := ac + totalCardinality := value1.getCardinality() + value2.getCardinality() + if totalCardinality > arrayDefaultMaxSize { // it could be a bitmap! + bc := newBitmapContainer() + for k := 0; k < len(value2.content); k++ { + v := value2.content[k] + i := uint(v) >> 6 + bc.bitmap[i] ^= (uint64(1) << (v % 64)) + } + for k := 0; k < len(ac.content); k++ { + v := ac.content[k] + i := uint(v) >> 6 + bc.bitmap[i] ^= (uint64(1) << (v % 64)) + } + bc.computeCardinality() + if bc.cardinality <= arrayDefaultMaxSize { + return bc.toArrayContainer() + } + return bc + } + desiredCapacity := totalCardinality + answer := newArrayContainerCapacity(desiredCapacity) + length := exclusiveUnion2by2(value1.content, value2.content, answer.content) + answer.content = answer.content[:length] + return answer + +} + +func (ac *arrayContainer) andNot(a container) container { + switch a.(type) { + case *arrayContainer: + return ac.andNotArray(a.(*arrayContainer)) + case *bitmapContainer: + return ac.andNotBitmap(a.(*bitmapContainer)) + } + panic("unsupported container type") +} + +func (ac *arrayContainer) iandNot(a container) container { + switch a.(type) { + case *arrayContainer: + return ac.iandNotArray(a.(*arrayContainer)) + case *bitmapContainer: + return ac.iandNotBitmap(a.(*bitmapContainer)) + } + panic("unsupported container type") +} + +func (ac *arrayContainer) andNotArray(value2 *arrayContainer) container { + value1 := ac + desiredcapacity := value1.getCardinality() + answer := newArrayContainerCapacity(desiredcapacity) + length := difference(value1.content, value2.content, answer.content) + answer.content = answer.content[:length] + return answer +} + +func (ac *arrayContainer) iandNotArray(value2 *arrayContainer) container { + length := difference(ac.content, value2.content, ac.content) + ac.content = ac.content[:length] + return ac +} + +func (ac *arrayContainer) andNotBitmap(value2 *bitmapContainer) container { + desiredcapacity := ac.getCardinality() + answer := newArrayContainerCapacity(desiredcapacity) + answer.content = answer.content[:desiredcapacity] + pos := 0 + for _, v := range ac.content { + if !value2.contains(v) { + answer.content[pos] = v + pos++ + } + } + answer.content = answer.content[:pos] + return answer +} + +func (ac *arrayContainer) andBitmap(value2 *bitmapContainer) container { + desiredcapacity := ac.getCardinality() + answer := newArrayContainerCapacity(desiredcapacity) + answer.content = answer.content[:desiredcapacity] + pos := 0 + for _, v := range ac.content { + if value2.contains(v) { + answer.content[pos] = v + pos++ + } + } + answer.content = answer.content[:pos] + return answer +} + +func (ac *arrayContainer) iandNotBitmap(value2 *bitmapContainer) container { + pos := 0 + for _, v := range ac.content { + if !value2.contains(v) { + ac.content[pos] = v + pos++ + } + } + ac.content = ac.content[:pos] + return ac +} + +func copyOf(array []uint16, size int) []uint16 { + result := make([]uint16, size) + for i, x := range array { + if i == size { + break + } + result[i] = x + } + return result +} + +// flip the values in the range [firstOfRange,lastOfRange) +func (ac *arrayContainer) inot(firstOfRange, lastOfRange int) container { + if firstOfRange >= lastOfRange { + return ac + } + return ac.inotClose(firstOfRange, lastOfRange-1) // remove everything in [firstOfRange,lastOfRange-1] +} + +// flip the values in the range [firstOfRange,lastOfRange] +func (ac *arrayContainer) inotClose(firstOfRange, lastOfRange int) container { + if firstOfRange > lastOfRange { // unlike add and remove, not uses an inclusive range [firstOfRange,lastOfRange] + return ac + } + // determine the span of array indices to be affected + startIndex := binarySearch(ac.content, uint16(firstOfRange)) + if startIndex < 0 { + startIndex = -startIndex - 1 + } + lastIndex := binarySearch(ac.content, uint16(lastOfRange)) + if lastIndex < 0 { + lastIndex = -lastIndex - 1 - 1 + } + currentValuesInRange := lastIndex - startIndex + 1 + spanToBeFlipped := lastOfRange - firstOfRange + 1 + + newValuesInRange := spanToBeFlipped - currentValuesInRange + buffer := make([]uint16, newValuesInRange) + cardinalityChange := newValuesInRange - currentValuesInRange + newCardinality := len(ac.content) + cardinalityChange + if cardinalityChange > 0 { + if newCardinality > len(ac.content) { + if newCardinality > arrayDefaultMaxSize { + return ac.toBitmapContainer().inot(firstOfRange, lastOfRange+1) + } + ac.content = copyOf(ac.content, newCardinality) + } + base := lastIndex + 1 + copy(ac.content[lastIndex+1+cardinalityChange:], ac.content[base:base+len(ac.content)-1-lastIndex]) + ac.negateRange(buffer, startIndex, lastIndex, firstOfRange, lastOfRange+1) + } else { // no expansion needed + ac.negateRange(buffer, startIndex, lastIndex, firstOfRange, lastOfRange+1) + if cardinalityChange < 0 { + + for i := startIndex + newValuesInRange; i < newCardinality; i++ { + ac.content[i] = ac.content[i-cardinalityChange] + } + } + } + ac.content = ac.content[:newCardinality] + return ac +} + +func (ac *arrayContainer) negateRange(buffer []uint16, startIndex, lastIndex, startRange, lastRange int) { + // compute the negation into buffer + outPos := 0 + inPos := startIndex // value here always >= valInRange, + // until it is exhausted + // n.b., we can start initially exhausted. + + valInRange := startRange + for ; valInRange < lastRange && inPos <= lastIndex; valInRange++ { + if uint16(valInRange) != ac.content[inPos] { + buffer[outPos] = uint16(valInRange) + outPos++ + } else { + inPos++ + } + } + + // if there are extra items (greater than the biggest + // pre-existing one in range), buffer them + for ; valInRange < lastRange; valInRange++ { + buffer[outPos] = uint16(valInRange) + outPos++ + } + + if outPos != len(buffer) { + panic("negateRange: internal bug") + } + + for i, item := range buffer { + ac.content[i+startIndex] = item + } +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} +func (ac *arrayContainer) andArray(value2 *arrayContainer) *arrayContainer { + desiredcapacity := min(ac.getCardinality(), value2.getCardinality()) + answer := newArrayContainerCapacity(desiredcapacity) + length := intersection2by2( + ac.content, + value2.content, + answer.content) + answer.content = answer.content[:length] + return answer +} + +func (ac *arrayContainer) intersectsArray(value2 *arrayContainer) bool { + return intersects2by2( + ac.content, + value2.content) +} + +func (ac *arrayContainer) iandArray(value2 *arrayContainer) *arrayContainer { + length := intersection2by2( + ac.content, + value2.content, + ac.content) + ac.content = ac.content[:length] + return ac +} + +func (ac *arrayContainer) getCardinality() int { + return len(ac.content) +} + +func (ac *arrayContainer) rank(x uint16) int { + answer := binarySearch(ac.content, x) + if answer >= 0 { + return answer + 1 + } + return -answer - 1 + +} + +func (ac *arrayContainer) selectInt(x uint16) int { + return int(ac.content[x]) +} + +func (ac *arrayContainer) clone() container { + ptr := arrayContainer{make([]uint16, len(ac.content))} + copy(ptr.content, ac.content[:]) + return &ptr +} + +func (ac *arrayContainer) contains(x uint16) bool { + return binarySearch(ac.content, x) >= 0 +} + +func (ac *arrayContainer) loadData(bitmapContainer *bitmapContainer) { + ac.content = make([]uint16, bitmapContainer.cardinality, bitmapContainer.cardinality) + bitmapContainer.fillArray(ac.content) +} +func newArrayContainer() *arrayContainer { + p := new(arrayContainer) + return p +} + +func newArrayContainerCapacity(size int) *arrayContainer { + p := new(arrayContainer) + p.content = make([]uint16, 0, size) + return p +} + +func newArrayContainerSize(size int) *arrayContainer { + p := new(arrayContainer) + p.content = make([]uint16, size, size) + return p +} + +func newArrayContainerRange(firstOfRun, lastOfRun int) *arrayContainer { + valuesInRange := lastOfRun - firstOfRun + 1 + this := newArrayContainerCapacity(valuesInRange) + for i := 0; i < valuesInRange; i++ { + this.content = append(this.content, uint16(firstOfRun+i)) + } + return this +} diff --git a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go new file mode 100644 index 00000000..8542aaff --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go @@ -0,0 +1,596 @@ +package roaring + +type bitmapContainer struct { + cardinality int + bitmap []uint64 +} + +func newBitmapContainer() *bitmapContainer { + p := new(bitmapContainer) + size := (1 << 16) / 64 + p.bitmap = make([]uint64, size, size) + return p +} + +func newBitmapContainerwithRange(firstOfRun, lastOfRun int) *bitmapContainer { + this := newBitmapContainer() + this.cardinality = lastOfRun - firstOfRun + 1 + if this.cardinality == maxCapacity { + fill(this.bitmap, uint64(0xffffffffffffffff)) + } else { + firstWord := firstOfRun / 64 + lastWord := lastOfRun / 64 + zeroPrefixLength := uint64(firstOfRun & 63) + zeroSuffixLength := uint64(63 - (lastOfRun & 63)) + + fillRange(this.bitmap, firstWord, lastWord+1, uint64(0xffffffffffffffff)) + this.bitmap[firstWord] ^= ((uint64(1) << zeroPrefixLength) - 1) + blockOfOnes := (uint64(1) << zeroSuffixLength) - 1 + maskOnLeft := blockOfOnes << (uint64(64) - zeroSuffixLength) + this.bitmap[lastWord] ^= maskOnLeft + } + return this +} + +type bitmapContainerShortIterator struct { + ptr *bitmapContainer + i int +} + +func (bcsi *bitmapContainerShortIterator) next() uint16 { + j := bcsi.i + bcsi.i = bcsi.ptr.NextSetBit(bcsi.i + 1) + return uint16(j) +} +func (bcsi *bitmapContainerShortIterator) hasNext() bool { + return bcsi.i >= 0 +} +func newBitmapContainerShortIterator(a *bitmapContainer) *bitmapContainerShortIterator { + return &bitmapContainerShortIterator{a, a.NextSetBit(0)} +} +func (bc *bitmapContainer) getShortIterator() shortIterable { + return newBitmapContainerShortIterator(bc) +} + +func (bc *bitmapContainer) getSizeInBytes() int { + return len(bc.bitmap) * 8 +} + +func (bc *bitmapContainer) serializedSizeInBytes() int { + return len(bc.bitmap) * 8 +} + +func bitmapEquals(a, b []uint64) bool { + if len(a) != len(b) { + return false + } + for i, v := range a { + if v != b[i] { + return false + } + } + return true +} + +func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) { + // TODO: should be written as optimized assembly + pos := i + base := mask + for k := 0; k < len(bc.bitmap); k++ { + bitset := bc.bitmap[k] + for bitset != 0 { + t := bitset & -bitset + x[pos] = base + uint32(popcount(t-1)) + pos++ + bitset ^= t + } + base += 64 + } +} + +func (bc *bitmapContainer) equals(o interface{}) bool { + srb, ok := o.(*bitmapContainer) + if ok { + if srb.cardinality != bc.cardinality { + return false + } + return bitmapEquals(bc.bitmap, srb.bitmap) + } + return false +} + +func (bc *bitmapContainer) add(i uint16) container { + x := int(i) + previous := bc.bitmap[x/64] + mask := uint64(1) << (uint(x) % 64) + newb := previous | mask + bc.bitmap[x/64] = newb + bc.cardinality += int(uint64(previous^newb) >> (uint(x) % 64)) + return bc +} + +func (bc *bitmapContainer) remove(i uint16) container { + if bc.contains(i) { + bc.cardinality-- + bc.bitmap[i/64] &^= (uint64(1) << (i % 64)) + if bc.cardinality == arrayDefaultMaxSize { + return bc.toArrayContainer() + } + } + return bc +} + +func (bc *bitmapContainer) getCardinality() int { + return bc.cardinality +} + +func (bc *bitmapContainer) clone() container { + ptr := bitmapContainer{bc.cardinality, make([]uint64, len(bc.bitmap))} + copy(ptr.bitmap, bc.bitmap[:]) + return &ptr +} + +// add all values in range [firstOfRange,lastOfRange) +func (bc *bitmapContainer) iaddRange(firstOfRange, lastOfRange int) container { + bc.cardinality += setBitmapRangeAndCardinalityChange(bc.bitmap, firstOfRange, lastOfRange) + return bc +} + +// add all values in range [firstOfRange,lastOfRange) +// unused code +/*func (bc *bitmapContainer) addRange(firstOfRange, lastOfRange int) container { + answer := &bitmapContainer{bc.cardinality, make([]uint64, len(bc.bitmap))} + copy(answer.bitmap, bc.bitmap[:]) + answer.cardinality += setBitmapRangeAndCardinalityChange(answer.bitmap, firstOfRange, lastOfRange) + return answer +}*/ + +// remove all values in range [firstOfRange,lastOfRange) +// unused code +/*func (bc *bitmapContainer) removeRange(firstOfRange, lastOfRange int) container { + answer := &bitmapContainer{bc.cardinality, make([]uint64, len(bc.bitmap))} + copy(answer.bitmap, bc.bitmap[:]) + answer.cardinality += resetBitmapRangeAndCardinalityChange(answer.bitmap, firstOfRange, lastOfRange) + if answer.getCardinality() <= arrayDefaultMaxSize { + return answer.toArrayContainer() + } + return answer +}*/ + +// remove all values in range [firstOfRange,lastOfRange) +func (bc *bitmapContainer) iremoveRange(firstOfRange, lastOfRange int) container { + bc.cardinality += resetBitmapRangeAndCardinalityChange(bc.bitmap, firstOfRange, lastOfRange) + if bc.getCardinality() <= arrayDefaultMaxSize { + return bc.toArrayContainer() + } + return bc +} + +// flip all values in range [firstOfRange,lastOfRange) +func (bc *bitmapContainer) inot(firstOfRange, lastOfRange int) container { + if lastOfRange-firstOfRange == maxCapacity { + flipBitmapRange(bc.bitmap, firstOfRange, lastOfRange) + bc.cardinality = maxCapacity - bc.cardinality + } else if lastOfRange-firstOfRange > maxCapacity/2 { + flipBitmapRange(bc.bitmap, firstOfRange, lastOfRange) + bc.computeCardinality() + } else { + bc.cardinality += flipBitmapRangeAndCardinalityChange(bc.bitmap, firstOfRange, lastOfRange) + } + if bc.getCardinality() <= arrayDefaultMaxSize { + return bc.toArrayContainer() + } + return bc +} + +// flip all values in range [firstOfRange,lastOfRange) +func (bc *bitmapContainer) not(firstOfRange, lastOfRange int) container { + answer := bc.clone() + return answer.inot(firstOfRange, lastOfRange) +} + +func (bc *bitmapContainer) or(a container) container { + switch a.(type) { + case *arrayContainer: + return bc.orArray(a.(*arrayContainer)) + case *bitmapContainer: + return bc.orBitmap(a.(*bitmapContainer)) + } + panic("unsupported container type") +} + +func (bc *bitmapContainer) ior(a container) container { + switch a.(type) { + case *arrayContainer: + return bc.iorArray(a.(*arrayContainer)) + case *bitmapContainer: + return bc.iorBitmap(a.(*bitmapContainer)) + } + panic("unsupported container type") +} + +func (bc *bitmapContainer) lazyIOR(a container) container { + switch a.(type) { + case *arrayContainer: + return bc.lazyIORArray(a.(*arrayContainer)) + case *bitmapContainer: + return bc.lazyIORBitmap(a.(*bitmapContainer)) + } + panic("unsupported container type") +} + +func (bc *bitmapContainer) lazyOR(a container) container { + switch a.(type) { + case *arrayContainer: + return bc.lazyORArray(a.(*arrayContainer)) + case *bitmapContainer: + return bc.lazyORBitmap(a.(*bitmapContainer)) + } + panic("unsupported container type") +} + +func (bc *bitmapContainer) orArray(value2 *arrayContainer) container { + answer := bc.clone().(*bitmapContainer) + c := value2.getCardinality() + for k := 0; k < c; k++ { + v := value2.content[k] + i := uint(v) >> 6 + bef := answer.bitmap[i] + aft := bef | (uint64(1) << (v % 64)) + answer.bitmap[i] = aft + answer.cardinality += int((bef - aft) >> 63) + } + return answer +} + +func (bc *bitmapContainer) orBitmap(value2 *bitmapContainer) container { + answer := newBitmapContainer() + for k := 0; k < len(answer.bitmap); k++ { + answer.bitmap[k] = bc.bitmap[k] | value2.bitmap[k] + } + answer.computeCardinality() + return answer +} + +func (bc *bitmapContainer) computeCardinality() { + bc.cardinality = int(popcntSlice(bc.bitmap)) +} + +func (bc *bitmapContainer) iorArray(value2 *arrayContainer) container { + answer := bc + c := value2.getCardinality() + for k := 0; k < c; k++ { + vc := value2.content[k] + i := uint(vc) >> 6 + bef := answer.bitmap[i] + aft := bef | (uint64(1) << (vc % 64)) + answer.bitmap[i] = aft + answer.cardinality += int((bef - aft) >> 63) + } + return answer +} + +func (bc *bitmapContainer) iorBitmap(value2 *bitmapContainer) container { + answer := bc + answer.cardinality = 0 + for k := 0; k < len(answer.bitmap); k++ { + answer.bitmap[k] = bc.bitmap[k] | value2.bitmap[k] + } + answer.computeCardinality() + return answer +} + +func (bc *bitmapContainer) lazyIORArray(value2 *arrayContainer) container { + answer := bc + c := value2.getCardinality() + for k := 0; k < c; k++ { + vc := value2.content[k] + i := uint(vc) >> 6 + answer.bitmap[i] = answer.bitmap[i] | (uint64(1) << (vc % 64)) + } + answer.cardinality = invalidCardinality + return answer +} + +func (bc *bitmapContainer) lazyORArray(value2 *arrayContainer) container { + answer := bc.clone().(*bitmapContainer) + return answer.lazyIORArray(value2) +} + +func (bc *bitmapContainer) lazyIORBitmap(value2 *bitmapContainer) container { + answer := bc + for k := 0; k < len(answer.bitmap); k++ { + answer.bitmap[k] = bc.bitmap[k] | value2.bitmap[k] + } + bc.cardinality = invalidCardinality + return answer +} + +func (bc *bitmapContainer) lazyORBitmap(value2 *bitmapContainer) container { + answer := bc.clone().(*bitmapContainer) + return answer.lazyIORBitmap(value2) +} + +func (bc *bitmapContainer) xor(a container) container { + switch a.(type) { + case *arrayContainer: + return bc.xorArray(a.(*arrayContainer)) + case *bitmapContainer: + return bc.xorBitmap(a.(*bitmapContainer)) + } + panic("unsupported container type") +} + +func (bc *bitmapContainer) xorArray(value2 *arrayContainer) container { + answer := bc.clone().(*bitmapContainer) + c := value2.getCardinality() + for k := 0; k < c; k++ { + vc := value2.content[k] + index := uint(vc) >> 6 + abi := answer.bitmap[index] + mask := uint64(1) << (vc % 64) + answer.cardinality += 1 - 2*int((abi&mask)>>(vc%64)) + answer.bitmap[index] = abi ^ mask + } + if answer.cardinality <= arrayDefaultMaxSize { + return answer.toArrayContainer() + } + return answer +} + +func (bc *bitmapContainer) rank(x uint16) int { + // TODO: rewrite in assembly + leftover := (uint(x) + 1) & 63 + if leftover == 0 { + return int(popcntSlice(bc.bitmap[:(uint(x)+1)/64])) + } + return int(popcntSlice(bc.bitmap[:(uint(x)+1)/64]) + popcount(bc.bitmap[(uint(x)+1)/64]<<(64-leftover))) +} + +func (bc *bitmapContainer) selectInt(x uint16) int { + remaining := x + for k := 0; k < len(bc.bitmap); k++ { + w := popcount(bc.bitmap[k]) + if uint16(w) > remaining { + return int(k*64 + selectBitPosition(bc.bitmap[k], int(remaining))) + } + remaining -= uint16(w) + } + return -1 +} + +func (bc *bitmapContainer) xorBitmap(value2 *bitmapContainer) container { + newCardinality := int(popcntXorSlice(bc.bitmap, value2.bitmap)) + + if newCardinality > arrayDefaultMaxSize { + answer := newBitmapContainer() + for k := 0; k < len(answer.bitmap); k++ { + answer.bitmap[k] = bc.bitmap[k] ^ value2.bitmap[k] + } + answer.cardinality = newCardinality + return answer + } + ac := newArrayContainerSize(newCardinality) + fillArrayXOR(ac.content, bc.bitmap, value2.bitmap) + ac.content = ac.content[:newCardinality] + return ac +} + +func (bc *bitmapContainer) and(a container) container { + switch a.(type) { + case *arrayContainer: + return bc.andArray(a.(*arrayContainer)) + case *bitmapContainer: + return bc.andBitmap(a.(*bitmapContainer)) + } + panic("unsupported container type") +} + +func (bc *bitmapContainer) intersects(a container) bool { + switch a.(type) { + case *arrayContainer: + return bc.intersectsArray(a.(*arrayContainer)) + case *bitmapContainer: + return bc.intersectsBitmap(a.(*bitmapContainer)) + } + panic("unsupported container type") +} + +func (bc *bitmapContainer) iand(a container) container { + switch a.(type) { + case *arrayContainer: + return bc.andArray(a.(*arrayContainer)) + case *bitmapContainer: + return bc.iandBitmap(a.(*bitmapContainer)) + } + panic("unsupported container type") +} + +func (bc *bitmapContainer) andArray(value2 *arrayContainer) *arrayContainer { + answer := newArrayContainerCapacity(len(value2.content)) + c := value2.getCardinality() + for k := 0; k < c; k++ { + v := value2.content[k] + if bc.contains(v) { + answer.content = append(answer.content, v) + } + } + return answer +} + +func (bc *bitmapContainer) andBitmap(value2 *bitmapContainer) container { + newcardinality := int(popcntAndSlice(bc.bitmap, value2.bitmap)) + if newcardinality > arrayDefaultMaxSize { + answer := newBitmapContainer() + for k := 0; k < len(answer.bitmap); k++ { + answer.bitmap[k] = bc.bitmap[k] & value2.bitmap[k] + } + answer.cardinality = newcardinality + return answer + } + ac := newArrayContainerSize(newcardinality) + fillArrayAND(ac.content, bc.bitmap, value2.bitmap) + ac.content = ac.content[:newcardinality] //not sure why i need this + return ac + +} + +func (bc *bitmapContainer) intersectsArray(value2 *arrayContainer) bool { + c := value2.getCardinality() + for k := 0; k < c; k++ { + v := value2.content[k] + if bc.contains(v) { + return true + } + } + return false +} + +func (bc *bitmapContainer) intersectsBitmap(value2 *bitmapContainer) bool { + for k := 0; k < len(bc.bitmap); k++ { + if (bc.bitmap[k] & value2.bitmap[k]) != 0 { + return true + } + } + return false + +} + +func (bc *bitmapContainer) iandBitmap(value2 *bitmapContainer) container { + newcardinality := int(popcntAndSlice(bc.bitmap, value2.bitmap)) + if newcardinality > arrayDefaultMaxSize { + for k := 0; k < len(bc.bitmap); k++ { + bc.bitmap[k] = bc.bitmap[k] & value2.bitmap[k] + } + bc.cardinality = newcardinality + return bc + } + ac := newArrayContainerSize(newcardinality) + fillArrayAND(ac.content, bc.bitmap, value2.bitmap) + ac.content = ac.content[:newcardinality] //not sure why i need this + return ac + +} + +func (bc *bitmapContainer) andNot(a container) container { + switch a.(type) { + case *arrayContainer: + return bc.andNotArray(a.(*arrayContainer)) + case *bitmapContainer: + return bc.andNotBitmap(a.(*bitmapContainer)) + } + panic("unsupported container type") +} + +func (bc *bitmapContainer) iandNot(a container) container { + switch a.(type) { + case *arrayContainer: + return bc.andNotArray(a.(*arrayContainer)) + case *bitmapContainer: + return bc.iandNotBitmap(a.(*bitmapContainer)) + } + panic("unsupported container type") +} + +func (bc *bitmapContainer) andNotArray(value2 *arrayContainer) container { + answer := bc.clone().(*bitmapContainer) + c := value2.getCardinality() + for k := 0; k < c; k++ { + vc := value2.content[k] + i := uint(vc) >> 6 + oldv := answer.bitmap[i] + newv := oldv &^ (uint64(1) << (vc % 64)) + answer.bitmap[i] = newv + answer.cardinality -= int(uint64(oldv^newv) >> (vc % 64)) + } + if answer.cardinality <= arrayDefaultMaxSize { + return answer.toArrayContainer() + } + return answer +} + +func (bc *bitmapContainer) andNotBitmap(value2 *bitmapContainer) container { + newCardinality := int(popcntMaskSlice(bc.bitmap, value2.bitmap)) + if newCardinality > arrayDefaultMaxSize { + answer := newBitmapContainer() + for k := 0; k < len(answer.bitmap); k++ { + answer.bitmap[k] = bc.bitmap[k] &^ value2.bitmap[k] + } + answer.cardinality = newCardinality + return answer + } + ac := newArrayContainerSize(newCardinality) + fillArrayANDNOT(ac.content, bc.bitmap, value2.bitmap) + return ac +} + +func (bc *bitmapContainer) iandNotBitmap(value2 *bitmapContainer) container { + newCardinality := int(popcntMaskSlice(bc.bitmap, value2.bitmap)) + if newCardinality > arrayDefaultMaxSize { + for k := 0; k < len(bc.bitmap); k++ { + bc.bitmap[k] = bc.bitmap[k] &^ value2.bitmap[k] + } + bc.cardinality = newCardinality + return bc + } + ac := newArrayContainerSize(newCardinality) + fillArrayANDNOT(ac.content, bc.bitmap, value2.bitmap) + return ac +} + +func (bc *bitmapContainer) contains(i uint16) bool { //testbit + x := int(i) + mask := uint64(1) << uint(x%64) + return (bc.bitmap[x/64] & mask) != 0 +} + +func (bc *bitmapContainer) loadData(arrayContainer *arrayContainer) { + bc.cardinality = arrayContainer.getCardinality() + c := arrayContainer.getCardinality() + for k := 0; k < c; k++ { + x := arrayContainer.content[k] + i := int(x) / 64 + bc.bitmap[i] |= (uint64(1) << uint(x%64)) + } +} + +func (bc *bitmapContainer) toArrayContainer() *arrayContainer { + ac := newArrayContainerCapacity(bc.cardinality) + ac.loadData(bc) + return ac +} + +func (bc *bitmapContainer) fillArray(container []uint16) { + //TODO: rewrite in assembly + pos := 0 + base := 0 + for k := 0; k < len(bc.bitmap); k++ { + bitset := bc.bitmap[k] + for bitset != 0 { + t := bitset & -bitset + container[pos] = uint16((base + int(popcount(t-1)))) + pos = pos + 1 + bitset ^= t + } + base += 64 + } +} + +func (bc *bitmapContainer) NextSetBit(i int) int { + x := i / 64 + if x >= len(bc.bitmap) { + return -1 + } + w := bc.bitmap[x] + w = w >> uint(i%64) + if w != 0 { + return i + numberOfTrailingZeros(w) + } + x++ + for ; x < len(bc.bitmap); x++ { + if bc.bitmap[x] != 0 { + return (x * 64) + numberOfTrailingZeros(bc.bitmap[x]) + } + } + return -1 +} diff --git a/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go b/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go new file mode 100644 index 00000000..35ebfc33 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go @@ -0,0 +1,193 @@ +package roaring + +import ( + "container/heap" +) + +// Or function that requires repairAfterLazy +func lazyOR(x1, x2 *Bitmap) *Bitmap { + answer := NewBitmap() + pos1 := 0 + pos2 := 0 + length1 := x1.highlowcontainer.size() + length2 := x2.highlowcontainer.size() +main: + for (pos1 < length1) && (pos2 < length2) { + s1 := x1.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + + for { + if s1 < s2 { + answer.highlowcontainer.appendCopy(x1.highlowcontainer, pos1) + pos1++ + if pos1 == length1 { + break main + } + s1 = x1.highlowcontainer.getKeyAtIndex(pos1) + } else if s1 > s2 { + answer.highlowcontainer.appendCopy(x2.highlowcontainer, pos2) + pos2++ + if pos2 == length2 { + break main + } + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } else { + + answer.highlowcontainer.appendContainer(s1, x1.highlowcontainer.getContainerAtIndex(pos1).lazyOR(x2.highlowcontainer.getContainerAtIndex(pos2)), false) + pos1++ + pos2++ + if (pos1 == length1) || (pos2 == length2) { + break main + } + s1 = x1.highlowcontainer.getKeyAtIndex(pos1) + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } + } + } + if pos1 == length1 { + answer.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2) + } else if pos2 == length2 { + answer.highlowcontainer.appendCopyMany(x1.highlowcontainer, pos1, length1) + } + return answer +} + +// In-place Or function that requires repairAfterLazy +func (x1 *Bitmap) lazyOR(x2 *Bitmap) *Bitmap { + answer := NewBitmap() // TODO: we return a new bitmap... could be optimized + pos1 := 0 + pos2 := 0 + length1 := x1.highlowcontainer.size() + length2 := x2.highlowcontainer.size() +main: + for (pos1 < length1) && (pos2 < length2) { + s1 := x1.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + + for { + if s1 < s2 { + answer.highlowcontainer.appendWithoutCopy(x1.highlowcontainer, pos1) + pos1++ + if pos1 == length1 { + break main + } + s1 = x1.highlowcontainer.getKeyAtIndex(pos1) + } else if s1 > s2 { + answer.highlowcontainer.appendCopy(x2.highlowcontainer, pos2) + pos2++ + if pos2 == length2 { + break main + } + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } else { + + answer.highlowcontainer.appendContainer(s1, x1.highlowcontainer.getWritableContainerAtIndex(pos1).lazyIOR(x2.highlowcontainer.getContainerAtIndex(pos2)), false) + pos1++ + pos2++ + if (pos1 == length1) || (pos2 == length2) { + break main + } + s1 = x1.highlowcontainer.getKeyAtIndex(pos1) + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } + } + } + if pos1 == length1 { + answer.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2) + } else if pos2 == length2 { + answer.highlowcontainer.appendWithoutCopyMany(x1.highlowcontainer, pos1, length1) + } + return answer +} + +// to be called after lazy aggregates +func (x1 *Bitmap) repairAfterLazy() { + for pos := 0; pos < x1.highlowcontainer.size(); pos++ { + c := x1.highlowcontainer.getContainerAtIndex(pos) + switch c.(type) { + case *bitmapContainer: + if c.(*bitmapContainer).cardinality == invalidCardinality { + c = x1.highlowcontainer.getWritableContainerAtIndex(pos) + c.(*bitmapContainer).computeCardinality() + if c.(*bitmapContainer).getCardinality() <= arrayDefaultMaxSize { + x1.highlowcontainer.setContainerAtIndex(pos, c.(*bitmapContainer).toArrayContainer()) + } + } + } + } +} + +// FastAnd computes the intersection between many bitmaps quickly +// Compared to the And function, it can take many bitmaps as input, thus saving the trouble +// of manually calling "And" many times. +func FastAnd(bitmaps ...*Bitmap) *Bitmap { + if len(bitmaps) == 0 { + return NewBitmap() + } else if len(bitmaps) == 1 { + return bitmaps[0].Clone() + } + answer := And(bitmaps[0], bitmaps[1]) + for _, bm := range bitmaps[2:] { + answer.And(bm) + } + return answer +} + +// FastOr computes the union between many bitmaps quickly, as opposed to having to call Or repeatedly. +// It might also be faster than calling Or repeatedly. +func FastOr(bitmaps ...*Bitmap) *Bitmap { + if len(bitmaps) == 0 { + return NewBitmap() + } else if len(bitmaps) == 1 { + return bitmaps[0].Clone() + } + answer := lazyOR(bitmaps[0], bitmaps[1]) + for _, bm := range bitmaps[2:] { + answer = answer.lazyOR(bm) + } + answer.repairAfterLazy() + return answer +} + +// HeapOr computes the union between many bitmaps quickly using a heap. +// It might be faster than calling Or repeatedly. +func HeapOr(bitmaps ...*Bitmap) *Bitmap { + if len(bitmaps) == 0 { + return NewBitmap() + } + // TODO: for better speed, we could do the operation lazily, see Java implementation + pq := make(priorityQueue, len(bitmaps)) + for i, bm := range bitmaps { + pq[i] = &item{bm, i} + } + heap.Init(&pq) + + for pq.Len() > 1 { + x1 := heap.Pop(&pq).(*item) + x2 := heap.Pop(&pq).(*item) + heap.Push(&pq, &item{Or(x1.value, x2.value), 0}) + } + return heap.Pop(&pq).(*item).value +} + +// HeapXor computes the symmetric difference between many bitmaps quickly (as opposed to calling Xor repeated). +// Internally, this function uses a heap. +// It might be faster than calling Xor repeatedly. +func HeapXor(bitmaps ...*Bitmap) *Bitmap { + if len(bitmaps) == 0 { + return NewBitmap() + } + + pq := make(priorityQueue, len(bitmaps)) + for i, bm := range bitmaps { + pq[i] = &item{bm, i} + } + heap.Init(&pq) + + for pq.Len() > 1 { + x1 := heap.Pop(&pq).(*item) + x2 := heap.Pop(&pq).(*item) + heap.Push(&pq, &item{Xor(x1.value, x2.value), 0}) + } + return heap.Pop(&pq).(*item).value +} diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt.go b/vendor/github.com/RoaringBitmap/roaring/popcnt.go new file mode 100644 index 00000000..e35ca670 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/popcnt.go @@ -0,0 +1,53 @@ +package roaring + +// bit population count, take from +// https://code.google.com/p/go/issues/detail?id=4988#c11 +// credit: https://code.google.com/u/arnehormann/ +func popcount(x uint64) uint64 { + x -= (x >> 1) & 0x5555555555555555 + x = (x>>2)&0x3333333333333333 + x&0x3333333333333333 + x += x >> 4 + x &= 0x0f0f0f0f0f0f0f0f + x *= 0x0101010101010101 + return x >> 56 +} + +func popcntSliceGo(s []uint64) uint64 { + cnt := uint64(0) + for _, x := range s { + cnt += popcount(x) + } + return cnt +} + +func popcntMaskSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] &^ m[i]) + } + return cnt +} + +func popcntAndSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] & m[i]) + } + return cnt +} + +func popcntOrSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] | m[i]) + } + return cnt +} + +func popcntXorSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] ^ m[i]) + } + return cnt +} diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt_amd64.s b/vendor/github.com/RoaringBitmap/roaring/popcnt_amd64.s new file mode 100644 index 00000000..18f58784 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/popcnt_amd64.s @@ -0,0 +1,103 @@ +// +build amd64,!appengine + +TEXT ·hasAsm(SB),4,$0-1 +MOVQ $1, AX +CPUID +SHRQ $23, CX +ANDQ $1, CX +MOVB CX, ret+0(FP) +RET + +#define POPCNTQ_DX_DX BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0xd2 + +TEXT ·popcntSliceAsm(SB),4,$0-32 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntSliceEnd +popcntSliceLoop: +BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0x16 // POPCNTQ (SI), DX +ADDQ DX, AX +ADDQ $8, SI +LOOP popcntSliceLoop +popcntSliceEnd: +MOVQ AX, ret+24(FP) +RET + +TEXT ·popcntMaskSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntMaskSliceEnd +MOVQ m+24(FP), DI +popcntMaskSliceLoop: +MOVQ (DI), DX +NOTQ DX +ANDQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntMaskSliceLoop +popcntMaskSliceEnd: +MOVQ AX, ret+48(FP) +RET + +TEXT ·popcntAndSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntAndSliceEnd +MOVQ m+24(FP), DI +popcntAndSliceLoop: +MOVQ (DI), DX +ANDQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntAndSliceLoop +popcntAndSliceEnd: +MOVQ AX, ret+48(FP) +RET + +TEXT ·popcntOrSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntOrSliceEnd +MOVQ m+24(FP), DI +popcntOrSliceLoop: +MOVQ (DI), DX +ORQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntOrSliceLoop +popcntOrSliceEnd: +MOVQ AX, ret+48(FP) +RET + +TEXT ·popcntXorSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntXorSliceEnd +MOVQ m+24(FP), DI +popcntXorSliceLoop: +MOVQ (DI), DX +XORQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntXorSliceLoop +popcntXorSliceEnd: +MOVQ AX, ret+48(FP) +RET diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go b/vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go new file mode 100644 index 00000000..35bfc440 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go @@ -0,0 +1,67 @@ +// +build amd64,!appengine + +package roaring + +// *** the following functions are defined in popcnt_amd64.s + +//go:noescape + +func hasAsm() bool + +// useAsm is a flag used to select the GO or ASM implementation of the popcnt function +var useAsm = hasAsm() + +//go:noescape + +func popcntSliceAsm(s []uint64) uint64 + +//go:noescape + +func popcntMaskSliceAsm(s, m []uint64) uint64 + +//go:noescape + +func popcntAndSliceAsm(s, m []uint64) uint64 + +//go:noescape + +func popcntOrSliceAsm(s, m []uint64) uint64 + +//go:noescape + +func popcntXorSliceAsm(s, m []uint64) uint64 + +func popcntSlice(s []uint64) uint64 { + if useAsm { + return popcntSliceAsm(s) + } + return popcntSliceGo(s) +} + +func popcntMaskSlice(s, m []uint64) uint64 { + if useAsm { + return popcntMaskSliceAsm(s, m) + } + return popcntMaskSliceGo(s, m) +} + +func popcntAndSlice(s, m []uint64) uint64 { + if useAsm { + return popcntAndSliceAsm(s, m) + } + return popcntAndSliceGo(s, m) +} + +func popcntOrSlice(s, m []uint64) uint64 { + if useAsm { + return popcntOrSliceAsm(s, m) + } + return popcntOrSliceGo(s, m) +} + +func popcntXorSlice(s, m []uint64) uint64 { + if useAsm { + return popcntXorSliceAsm(s, m) + } + return popcntXorSliceGo(s, m) +} diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go b/vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go new file mode 100644 index 00000000..3c05fb45 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go @@ -0,0 +1,23 @@ +// +build !amd64 appengine + +package roaring + +func popcntSlice(s []uint64) uint64 { + return popcntSliceGo(s) +} + +func popcntMaskSlice(s, m []uint64) uint64 { + return popcntMaskSliceGo(s, m) +} + +func popcntAndSlice(s, m []uint64) uint64 { + return popcntAndSliceGo(s, m) +} + +func popcntOrSlice(s, m []uint64) uint64 { + return popcntOrSliceGo(s, m) +} + +func popcntXorSlice(s, m []uint64) uint64 { + return popcntXorSliceGo(s, m) +} diff --git a/vendor/github.com/RoaringBitmap/roaring/priorityqueue.go b/vendor/github.com/RoaringBitmap/roaring/priorityqueue.go new file mode 100644 index 00000000..bed593ec --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/priorityqueue.go @@ -0,0 +1,101 @@ +package roaring + +import "container/heap" + +///////////// +// The priorityQueue is used to keep Bitmaps sorted. +//////////// + +type item struct { + value *Bitmap + index int +} + +type priorityQueue []*item + +func (pq priorityQueue) Len() int { return len(pq) } + +func (pq priorityQueue) Less(i, j int) bool { + return pq[i].value.GetSizeInBytes() < pq[j].value.GetSizeInBytes() +} + +func (pq priorityQueue) Swap(i, j int) { + pq[i], pq[j] = pq[j], pq[i] + pq[i].index = i + pq[j].index = j +} + +func (pq *priorityQueue) Push(x interface{}) { + n := len(*pq) + item := x.(*item) + item.index = n + *pq = append(*pq, item) +} + +func (pq *priorityQueue) Pop() interface{} { + old := *pq + n := len(old) + item := old[n-1] + item.index = -1 // for safety + *pq = old[0 : n-1] + return item +} + +func (pq *priorityQueue) update(item *item, value *Bitmap) { + item.value = value + heap.Fix(pq, item.index) +} + +///////////// +// The containerPriorityQueue is used to keep the containers of various Bitmaps sorted. +//////////// + +type containeritem struct { + value *Bitmap + keyindex int + index int +} + +type containerPriorityQueue []*containeritem + +func (pq containerPriorityQueue) Len() int { return len(pq) } + +func (pq containerPriorityQueue) Less(i, j int) bool { + k1 := pq[i].value.highlowcontainer.getKeyAtIndex(pq[i].keyindex) + k2 := pq[j].value.highlowcontainer.getKeyAtIndex(pq[j].keyindex) + if k1 != k2 { + return k1 < k2 + } + c1 := pq[i].value.highlowcontainer.getContainerAtIndex(pq[i].keyindex) + c2 := pq[j].value.highlowcontainer.getContainerAtIndex(pq[j].keyindex) + + return c1.getCardinality() > c2.getCardinality() +} + +func (pq containerPriorityQueue) Swap(i, j int) { + pq[i], pq[j] = pq[j], pq[i] + pq[i].index = i + pq[j].index = j +} + +func (pq *containerPriorityQueue) Push(x interface{}) { + n := len(*pq) + item := x.(*containeritem) + item.index = n + *pq = append(*pq, item) +} + +func (pq *containerPriorityQueue) Pop() interface{} { + old := *pq + n := len(old) + item := old[n-1] + item.index = -1 // for safety + *pq = old[0 : n-1] + return item +} + +func (pq *containerPriorityQueue) update(item *containeritem, value *Bitmap, keyindex int) { + item.value = value + item.keyindex = keyindex + heap.Fix(pq, item.index) +} diff --git a/vendor/github.com/RoaringBitmap/roaring/roaring.go b/vendor/github.com/RoaringBitmap/roaring/roaring.go new file mode 100644 index 00000000..ed4fe516 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/roaring.go @@ -0,0 +1,1113 @@ +// Package roaring is an implementation of Roaring Bitmaps in Go. +// They provide fast compressed bitmap data structures (also called bitset). +// They are ideally suited to represent sets of integers over +// relatively small ranges. +// See http://roaringbitmap.org for details. +package roaring + +import ( + "bufio" + "bytes" + "encoding/base64" + "fmt" + "io" + "strconv" +) + +// Bitmap represents a compressed bitmap where you can add integers. +type Bitmap struct { + highlowcontainer roaringArray +} + +// ToBase64 serializes a bitmap as Base64 +func (rb *Bitmap) ToBase64() (string, error) { + buf := new(bytes.Buffer) + _, err := rb.WriteTo(buf) + return base64.StdEncoding.EncodeToString(buf.Bytes()), err + +} + +// FromBase64 deserializes a bitmap from Base64 +func (rb *Bitmap) FromBase64(str string) (int64, error) { + data, err := base64.StdEncoding.DecodeString(str) + if err != nil { + return 0, err + } + buf := bytes.NewBuffer(data) + + return rb.ReadFrom(buf) +} + +// WriteTo writes a serialized version of this bitmap to stream +func (rb *Bitmap) WriteTo(stream io.Writer) (int64, error) { + return rb.highlowcontainer.writeTo(stream) +} + +// ReadFrom reads a serialized version of this bitmap from stream +func (rb *Bitmap) ReadFrom(stream io.Reader) (int64, error) { + return rb.highlowcontainer.readFrom(stream) +} + +// MarshalBinary implements the encoding.BinaryMarshaler interface for the bitmap +func (rb *Bitmap) MarshalBinary() ([]byte, error) { + var buf bytes.Buffer + writer := bufio.NewWriter(&buf) + _, err := rb.WriteTo(writer) + if err != nil { + return nil, err + } + err = writer.Flush() + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface for the bitmap +func (rb *Bitmap) UnmarshalBinary(data []byte) error { + var buf bytes.Buffer + _, err := buf.Write(data) + if err != nil { + return err + } + reader := bufio.NewReader(&buf) + _, err = rb.ReadFrom(reader) + return err +} + +// NewBitmap creates a new empty Bitmap (see also New) +func NewBitmap() *Bitmap { + return &Bitmap{*newRoaringArray()} +} + +// New creates a new empty Bitmap (same as NewBitmap) +func New() *Bitmap { + return &Bitmap{*newRoaringArray()} +} + +// Clear removes all content from the Bitmap and frees the memory +func (rb *Bitmap) Clear() { + rb.highlowcontainer = *newRoaringArray() +} + +// ToArray creates a new slice containing all of the integers stored in the Bitmap in sorted order +func (rb *Bitmap) ToArray() []uint32 { + array := make([]uint32, rb.GetCardinality()) + pos := 0 + pos2 := 0 + + for pos < rb.highlowcontainer.size() { + hs := toIntUnsigned(rb.highlowcontainer.getKeyAtIndex(pos)) << 16 + c := rb.highlowcontainer.getContainerAtIndex(pos) + pos++ + c.fillLeastSignificant16bits(array, pos2, hs) + pos2 += c.getCardinality() + } + return array +} + +// GetSizeInBytes estimates the memory usage of the Bitmap. Note that this +// might differ slightly from the amount of bytes required for persistent storage +func (rb *Bitmap) GetSizeInBytes() uint64 { + size := uint64(8) + for _, c := range rb.highlowcontainer.containers { + size += uint64(2) + uint64(c.getSizeInBytes()) + } + return size +} + +// GetSerializedSizeInBytes computes the serialized size in bytes the Bitmap. It should correspond to the +// number of bytes written when invoking WriteTo +func (rb *Bitmap) GetSerializedSizeInBytes() uint64 { + return rb.highlowcontainer.serializedSizeInBytes() +} + +// BoundSerializedSizeInBytes returns an upper bound on the serialized size in bytes +// assuming that one wants to store "cardinality" integers in [0, universe_size) +func BoundSerializedSizeInBytes(cardinality uint64, universeSize uint64) uint64 { + contnbr := (universeSize + uint64(65535)) / uint64(65536) + if contnbr > cardinality { + contnbr = cardinality + // we can't have more containers than we have values + } + headermax := 8*contnbr + 4 + if 4 > (contnbr+7)/8 { + headermax += 4 + } else { + headermax += (contnbr + 7) / 8 + } + valsarray := 2 * cardinality + valsbitmap := contnbr * 8192 + valsbest := valsarray + if valsbest > valsbitmap { + valsbest = valsbitmap + } + return valsbest + headermax +} + +// IntIterable allows you to iterate over the values in a Bitmap +type IntIterable interface { + HasNext() bool + Next() uint32 +} + +type intIterator struct { + pos int + hs uint32 + iter shortIterable + highlowcontainer *roaringArray +} + +// HasNext returns true if there are more integers to iterate over +func (ii *intIterator) HasNext() bool { + return ii.pos < ii.highlowcontainer.size() +} + +func (ii *intIterator) init() { + if ii.highlowcontainer.size() > ii.pos { + ii.iter = ii.highlowcontainer.getContainerAtIndex(ii.pos).getShortIterator() + ii.hs = toIntUnsigned(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16 + } +} + +// Next returns the next integer +func (ii *intIterator) Next() uint32 { + x := toIntUnsigned(ii.iter.next()) | ii.hs + if !ii.iter.hasNext() { + ii.pos = ii.pos + 1 + ii.init() + } + return x +} + +func newIntIterator(a *Bitmap) *intIterator { + p := new(intIterator) + p.pos = 0 + p.highlowcontainer = &a.highlowcontainer + p.init() + return p +} + +// String creates a string representation of the Bitmap +func (rb *Bitmap) String() string { + // inspired by https://github.com/fzandona/goroar/ + var buffer bytes.Buffer + start := []byte("{") + buffer.Write(start) + i := rb.Iterator() + counter := 0 + if i.HasNext() { + counter = counter + 1 + buffer.WriteString(strconv.FormatInt(int64(i.Next()), 10)) + } + for i.HasNext() { + buffer.WriteString(",") + counter = counter + 1 + // to avoid exhausting the memory + if counter > 0x40000 { + buffer.WriteString("...") + break + } + buffer.WriteString(strconv.FormatInt(int64(i.Next()), 10)) + } + buffer.WriteString("}") + return buffer.String() +} + +// Iterator creates a new IntIterable to iterate over the integers contained in the bitmap, in sorted order +func (rb *Bitmap) Iterator() IntIterable { + return newIntIterator(rb) +} + +// Clone creates a copy of the Bitmap +func (rb *Bitmap) Clone() *Bitmap { + ptr := new(Bitmap) + ptr.highlowcontainer = *rb.highlowcontainer.clone() + return ptr +} + +// Contains returns true if the integer is contained in the bitmap +func (rb *Bitmap) Contains(x uint32) bool { + hb := highbits(x) + c := rb.highlowcontainer.getContainer(hb) + return c != nil && c.contains(lowbits(x)) +} + +// ContainsInt returns true if the integer is contained in the bitmap (this is a convenience method, the parameter is casted to uint32 and Contains is called) +func (rb *Bitmap) ContainsInt(x int) bool { + return rb.Contains(uint32(x)) +} + +// Equals returns true if the two bitmaps contain the same integers +func (rb *Bitmap) Equals(o interface{}) bool { + srb, ok := o.(*Bitmap) + if ok { + return srb.highlowcontainer.equals(rb.highlowcontainer) + } + return false +} + +// Add the integer x to the bitmap +func (rb *Bitmap) Add(x uint32) { + hb := highbits(x) + ra := &rb.highlowcontainer + i := ra.getIndex(hb) + if i >= 0 { + var c container + c = ra.getWritableContainerAtIndex(i) + c = c.add(lowbits(x)) + rb.highlowcontainer.setContainerAtIndex(i, c) + } else { + newac := newArrayContainer() + rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, newac.add(lowbits(x))) + } +} + +// add the integer x to the bitmap, return the container and its index +func (rb *Bitmap) addwithptr(x uint32) (int, container) { + hb := highbits(x) + ra := &rb.highlowcontainer + i := ra.getIndex(hb) + var c container + if i >= 0 { + c = ra.getWritableContainerAtIndex(i) + c = c.add(lowbits(x)) + rb.highlowcontainer.setContainerAtIndex(i, c) + return i, c + } + newac := newArrayContainer() + c = newac.add(lowbits(x)) + rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, c) + return -i - 1, c +} + +// CheckedAdd adds the integer x to the bitmap and return true if it was added (false if the integer was already present) +func (rb *Bitmap) CheckedAdd(x uint32) bool { + // TODO: add unit tests for this method + hb := highbits(x) + i := rb.highlowcontainer.getIndex(hb) + if i >= 0 { + C := rb.highlowcontainer.getWritableContainerAtIndex(i) + oldcard := C.getCardinality() + C = C.add(lowbits(x)) + rb.highlowcontainer.setContainerAtIndex(i, C) + return C.getCardinality() > oldcard + } + newac := newArrayContainer() + rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, newac.add(lowbits(x))) + return true + +} + +// AddInt adds the integer x to the bitmap (convenience method: the parameter is casted to uint32 and we call Add) +func (rb *Bitmap) AddInt(x int) { + rb.Add(uint32(x)) +} + +// Remove the integer x from the bitmap +func (rb *Bitmap) Remove(x uint32) { + hb := highbits(x) + i := rb.highlowcontainer.getIndex(hb) + if i >= 0 { + c := rb.highlowcontainer.getWritableContainerAtIndex(i).remove(lowbits(x)) + rb.highlowcontainer.setContainerAtIndex(i, c.remove(lowbits(x))) + if rb.highlowcontainer.getContainerAtIndex(i).getCardinality() == 0 { + rb.highlowcontainer.removeAtIndex(i) + } + } +} + +// CheckedRemove removes the integer x from the bitmap and return true if the integer was effectively remove (and false if the integer was not present) +func (rb *Bitmap) CheckedRemove(x uint32) bool { + // TODO: add unit tests for this method + hb := highbits(x) + i := rb.highlowcontainer.getIndex(hb) + if i >= 0 { + C := rb.highlowcontainer.getWritableContainerAtIndex(i) + oldcard := C.getCardinality() + C = C.remove(lowbits(x)) + rb.highlowcontainer.setContainerAtIndex(i, C) + if rb.highlowcontainer.getContainerAtIndex(i).getCardinality() == 0 { + rb.highlowcontainer.removeAtIndex(i) + return true + } + return C.getCardinality() < oldcard + } + return false + +} + +// IsEmpty returns true if the Bitmap is empty (it is faster than doing (GetCardinality() == 0)) +func (rb *Bitmap) IsEmpty() bool { + return rb.highlowcontainer.size() == 0 +} + +// GetCardinality returns the number of integers contained in the bitmap +func (rb *Bitmap) GetCardinality() uint64 { + size := uint64(0) + for _, c := range rb.highlowcontainer.containers { + size += uint64(c.getCardinality()) + } + return size +} + +// Rank returns the number of integers that are smaller or equal to x (Rank(infinity) would be GetCardinality()) +func (rb *Bitmap) Rank(x uint32) uint64 { + size := uint64(0) + for i := 0; i < rb.highlowcontainer.size(); i++ { + key := rb.highlowcontainer.getKeyAtIndex(i) + if key > highbits(x) { + return size + } + if key < highbits(x) { + size += uint64(rb.highlowcontainer.getContainerAtIndex(i).getCardinality()) + } else { + return size + uint64(rb.highlowcontainer.getContainerAtIndex(i).rank(lowbits(x))) + } + } + return size +} + +// Select returns the xth integer in the bitmap +func (rb *Bitmap) Select(x uint32) (uint32, error) { + if rb.GetCardinality() <= uint64(x) { + return 0, fmt.Errorf("can't find %dth integer in a bitmap with only %d items", x, rb.GetCardinality()) + } + + remaining := x + for i := 0; i < rb.highlowcontainer.size(); i++ { + c := rb.highlowcontainer.getContainerAtIndex(i) + if remaining >= uint32(c.getCardinality()) { + remaining -= uint32(c.getCardinality()) + } else { + key := rb.highlowcontainer.getKeyAtIndex(i) + return uint32(key)<<16 + uint32(c.selectInt(uint16(remaining))), nil + } + } + return 0, fmt.Errorf("can't find %dth integer in a bitmap with only %d items", x, rb.GetCardinality()) +} + +// And computes the intersection between two bitmaps and stores the result in the current bitmap +func (rb *Bitmap) And(x2 *Bitmap) { + pos1 := 0 + pos2 := 0 + intersectionsize := 0 + length1 := rb.highlowcontainer.size() + length2 := x2.highlowcontainer.size() + +main: + for { + if pos1 < length1 && pos2 < length2 { + s1 := rb.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + for { + if s1 == s2 { + c1 := rb.highlowcontainer.getWritableContainerAtIndex(pos1) + c2 := x2.highlowcontainer.getContainerAtIndex(pos2) + diff := c1.iand(c2) + if diff.getCardinality() > 0 { + rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, diff, false) + intersectionsize++ + } + pos1++ + pos2++ + if (pos1 == length1) || (pos2 == length2) { + break main + } + s1 = rb.highlowcontainer.getKeyAtIndex(pos1) + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } else if s1 < s2 { + pos1 = rb.highlowcontainer.advanceUntil(s2, pos1) + if pos1 == length1 { + break main + } + s1 = rb.highlowcontainer.getKeyAtIndex(pos1) + } else { //s1 > s2 + pos2 = x2.highlowcontainer.advanceUntil(s1, pos2) + if pos2 == length2 { + break main + } + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } + } + } else { + break + } + } + rb.highlowcontainer.resize(intersectionsize) +} + +// OrCardinality returns the cardinality of the union between two bitmaps, bitmaps are not modified +func (rb *Bitmap) OrCardinality(x2 *Bitmap) uint64 { + pos1 := 0 + pos2 := 0 + length1 := rb.highlowcontainer.size() + length2 := x2.highlowcontainer.size() + answer := uint64(0) +main: + for { + if (pos1 < length1) && (pos2 < length2) { + s1 := rb.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + + for { + if s1 < s2 { + answer += uint64(rb.highlowcontainer.getContainerAtIndex(pos1).getCardinality()) + pos1++ + if pos1 == length1 { + break main + } + s1 = rb.highlowcontainer.getKeyAtIndex(pos1) + } else if s1 > s2 { + answer += uint64(x2.highlowcontainer.getContainerAtIndex(pos2).getCardinality()) + pos2++ + if pos2 == length2 { + break main + } + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } else { + // TODO: could be faster if we did not have to materialize the container + answer += uint64(rb.highlowcontainer.getContainerAtIndex(pos1).or(x2.highlowcontainer.getContainerAtIndex(pos2)).getCardinality()) + pos1++ + pos2++ + if (pos1 == length1) || (pos2 == length2) { + break main + } + s1 = rb.highlowcontainer.getKeyAtIndex(pos1) + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } + } + } else { + break + } + } + for ; pos1 < length1; pos1++ { + answer += uint64(rb.highlowcontainer.getContainerAtIndex(pos1).getCardinality()) + } + for ; pos2 < length2; pos2++ { + answer += uint64(x2.highlowcontainer.getContainerAtIndex(pos2).getCardinality()) + } + return answer +} + +// AndCardinality returns the cardinality of the intersection between two bitmaps, bitmaps are not modified +func (rb *Bitmap) AndCardinality(x2 *Bitmap) uint64 { + pos1 := 0 + pos2 := 0 + answer := uint64(0) + length1 := rb.highlowcontainer.size() + length2 := x2.highlowcontainer.size() + +main: + for { + if pos1 < length1 && pos2 < length2 { + s1 := rb.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + for { + if s1 == s2 { + c1 := rb.highlowcontainer.getContainerAtIndex(pos1) + c2 := x2.highlowcontainer.getContainerAtIndex(pos2) + diff := c1.and(c2) + answer += uint64(diff.getCardinality()) // TODO: could be faster if we did not have to compute diff + pos1++ + pos2++ + if (pos1 == length1) || (pos2 == length2) { + break main + } + s1 = rb.highlowcontainer.getKeyAtIndex(pos1) + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } else if s1 < s2 { + pos1 = rb.highlowcontainer.advanceUntil(s2, pos1) + if pos1 == length1 { + break main + } + s1 = rb.highlowcontainer.getKeyAtIndex(pos1) + } else { //s1 > s2 + pos2 = x2.highlowcontainer.advanceUntil(s1, pos2) + if pos2 == length2 { + break main + } + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } + } + } else { + break + } + } + return answer +} + +// Intersects checks whether two bitmap intersects, bitmaps are not modified +func (rb *Bitmap) Intersects(x2 *Bitmap) bool { + pos1 := 0 + pos2 := 0 + length1 := rb.highlowcontainer.size() + length2 := x2.highlowcontainer.size() + +main: + for { + if pos1 < length1 && pos2 < length2 { + s1 := rb.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + for { + if s1 == s2 { + c1 := rb.highlowcontainer.getContainerAtIndex(pos1) + c2 := x2.highlowcontainer.getContainerAtIndex(pos2) + if c1.intersects(c2) { + return true + } + pos1++ + pos2++ + if (pos1 == length1) || (pos2 == length2) { + break main + } + s1 = rb.highlowcontainer.getKeyAtIndex(pos1) + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } else if s1 < s2 { + pos1 = rb.highlowcontainer.advanceUntil(s2, pos1) + if pos1 == length1 { + break main + } + s1 = rb.highlowcontainer.getKeyAtIndex(pos1) + } else { //s1 > s2 + pos2 = x2.highlowcontainer.advanceUntil(s1, pos2) + if pos2 == length2 { + break main + } + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } + } + } else { + break + } + } + return false +} + +// Xor computes the symmetric difference between two bitmaps and stores the result in the current bitmap +func (rb *Bitmap) Xor(x2 *Bitmap) { + pos1 := 0 + pos2 := 0 + length1 := rb.highlowcontainer.size() + length2 := x2.highlowcontainer.size() + for { + if (pos1 < length1) && (pos2 < length2) { + s1 := rb.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + if s1 < s2 { + pos1 = rb.highlowcontainer.advanceUntil(s2, pos1) + if pos1 == length1 { + break + } + } else if s1 > s2 { + c := x2.highlowcontainer.getWritableContainerAtIndex(pos2) + rb.highlowcontainer.insertNewKeyValueAt(pos1, x2.highlowcontainer.getKeyAtIndex(pos2), c) + length1++ + pos1++ + pos2++ + } else { + // TODO: couple be computed in-place for reduced memory usage + c := rb.highlowcontainer.getContainerAtIndex(pos1).xor(x2.highlowcontainer.getContainerAtIndex(pos2)) + if c.getCardinality() > 0 { + rb.highlowcontainer.setContainerAtIndex(pos1, c) + pos1++ + } else { + rb.highlowcontainer.removeAtIndex(pos1) + length1-- + } + pos2++ + } + } else { + break + } + } + if pos1 == length1 { + rb.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2) + } +} + +// Or computes the union between two bitmaps and stores the result in the current bitmap +func (rb *Bitmap) Or(x2 *Bitmap) { + results := Or(rb, x2) // Todo: could be computed in-place for reduced memory usage + rb.highlowcontainer = results.highlowcontainer +} + +// AndNot computes the difference between two bitmaps and stores the result in the current bitmap +func (rb *Bitmap) AndNot(x2 *Bitmap) { + pos1 := 0 + pos2 := 0 + intersectionsize := 0 + length1 := rb.highlowcontainer.size() + length2 := x2.highlowcontainer.size() + +main: + for { + if pos1 < length1 && pos2 < length2 { + s1 := rb.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + for { + if s1 == s2 { + c1 := rb.highlowcontainer.getWritableContainerAtIndex(pos1) + c2 := x2.highlowcontainer.getContainerAtIndex(pos2) + diff := c1.iandNot(c2) + if diff.getCardinality() > 0 { + rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, diff, false) + intersectionsize++ + } + pos1++ + pos2++ + if (pos1 == length1) || (pos2 == length2) { + break main + } + s1 = rb.highlowcontainer.getKeyAtIndex(pos1) + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } else if s1 < s2 { + c1 := rb.highlowcontainer.getContainerAtIndex(pos1) + mustCopyOnWrite := rb.highlowcontainer.needsCopyOnWrite(pos1) + rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, c1, mustCopyOnWrite) + intersectionsize++ + pos1++ + if pos1 == length1 { + break main + } + s1 = rb.highlowcontainer.getKeyAtIndex(pos1) + } else { //s1 > s2 + pos2 = x2.highlowcontainer.advanceUntil(s1, pos2) + if pos2 == length2 { + break main + } + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } + } + } else { + break + } + } + // TODO:implement as a copy + for pos1 < length1 { + c1 := rb.highlowcontainer.getContainerAtIndex(pos1) + s1 := rb.highlowcontainer.getKeyAtIndex(pos1) + mustCopyOnWrite := rb.highlowcontainer.needsCopyOnWrite(pos1) + rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, c1, mustCopyOnWrite) + intersectionsize++ + pos1++ + } + rb.highlowcontainer.resize(intersectionsize) +} + +// Or computes the union between two bitmaps and returns the result +func Or(x1, x2 *Bitmap) *Bitmap { + answer := NewBitmap() + pos1 := 0 + pos2 := 0 + length1 := x1.highlowcontainer.size() + length2 := x2.highlowcontainer.size() +main: + for (pos1 < length1) && (pos2 < length2) { + s1 := x1.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + + for { + if s1 < s2 { + answer.highlowcontainer.appendCopy(x1.highlowcontainer, pos1) + pos1++ + if pos1 == length1 { + break main + } + s1 = x1.highlowcontainer.getKeyAtIndex(pos1) + } else if s1 > s2 { + answer.highlowcontainer.appendCopy(x2.highlowcontainer, pos2) + pos2++ + if pos2 == length2 { + break main + } + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } else { + + answer.highlowcontainer.appendContainer(s1, x1.highlowcontainer.getContainerAtIndex(pos1).or(x2.highlowcontainer.getContainerAtIndex(pos2)), false) + pos1++ + pos2++ + if (pos1 == length1) || (pos2 == length2) { + break main + } + s1 = x1.highlowcontainer.getKeyAtIndex(pos1) + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } + } + } + if pos1 == length1 { + answer.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2) + } else if pos2 == length2 { + answer.highlowcontainer.appendCopyMany(x1.highlowcontainer, pos1, length1) + } + return answer +} + +// And computes the intersection between two bitmaps and returns the result +func And(x1, x2 *Bitmap) *Bitmap { + answer := NewBitmap() + pos1 := 0 + pos2 := 0 + length1 := x1.highlowcontainer.size() + length2 := x2.highlowcontainer.size() +main: + for pos1 < length1 && pos2 < length2 { + s1 := x1.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + for { + if s1 == s2 { + C := x1.highlowcontainer.getContainerAtIndex(pos1) + C = C.and(x2.highlowcontainer.getContainerAtIndex(pos2)) + + if C.getCardinality() > 0 { + answer.highlowcontainer.appendContainer(s1, C, false) + } + pos1++ + pos2++ + if (pos1 == length1) || (pos2 == length2) { + break main + } + s1 = x1.highlowcontainer.getKeyAtIndex(pos1) + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } else if s1 < s2 { + pos1 = x1.highlowcontainer.advanceUntil(s2, pos1) + if pos1 == length1 { + break main + } + s1 = x1.highlowcontainer.getKeyAtIndex(pos1) + } else { // s1 > s2 + pos2 = x2.highlowcontainer.advanceUntil(s1, pos2) + if pos2 == length2 { + break main + } + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } + } + } + return answer +} + +// Xor computes the symmetric difference between two bitmaps and returns the result +func Xor(x1, x2 *Bitmap) *Bitmap { + answer := NewBitmap() + pos1 := 0 + pos2 := 0 + length1 := x1.highlowcontainer.size() + length2 := x2.highlowcontainer.size() + for { + if (pos1 < length1) && (pos2 < length2) { + s1 := x1.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + if s1 < s2 { + answer.highlowcontainer.appendCopy(x1.highlowcontainer, pos1) + pos1++ + } else if s1 > s2 { + answer.highlowcontainer.appendCopy(x2.highlowcontainer, pos2) + pos2++ + } else { + c := x1.highlowcontainer.getContainerAtIndex(pos1).xor(x2.highlowcontainer.getContainerAtIndex(pos2)) + if c.getCardinality() > 0 { + answer.highlowcontainer.appendContainer(s1, c, false) + } + pos1++ + pos2++ + } + } else { + break + } + } + if pos1 == length1 { + answer.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2) + } else if pos2 == length2 { + answer.highlowcontainer.appendCopyMany(x1.highlowcontainer, pos1, length1) + } + return answer +} + +// AndNot computes the difference between two bitmaps and returns the result +func AndNot(x1, x2 *Bitmap) *Bitmap { + answer := NewBitmap() + pos1 := 0 + pos2 := 0 + length1 := x1.highlowcontainer.size() + length2 := x2.highlowcontainer.size() + +main: + for { + if pos1 < length1 && pos2 < length2 { + s1 := x1.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + for { + if s1 < s2 { + answer.highlowcontainer.appendCopy(x1.highlowcontainer, pos1) + pos1++ + if pos1 == length1 { + break main + } + s1 = x1.highlowcontainer.getKeyAtIndex(pos1) + } else if s1 == s2 { + c1 := x1.highlowcontainer.getContainerAtIndex(pos1) + c2 := x2.highlowcontainer.getContainerAtIndex(pos2) + diff := c1.andNot(c2) + if diff.getCardinality() > 0 { + answer.highlowcontainer.appendContainer(s1, diff, false) + } + pos1++ + pos2++ + if (pos1 == length1) || (pos2 == length2) { + break main + } + s1 = x1.highlowcontainer.getKeyAtIndex(pos1) + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } else { //s1 > s2 + pos2 = x2.highlowcontainer.advanceUntil(s1, pos2) + if pos2 == length2 { + break main + } + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } + } + } else { + break + } + } + if pos2 == length2 { + answer.highlowcontainer.appendCopyMany(x1.highlowcontainer, pos1, length1) + } + return answer +} + +// AddMany add all of the values in dat +func (rb *Bitmap) AddMany(dat []uint32) { + if len(dat) == 0 { + return + } + prev := dat[0] + idx, c := rb.addwithptr(prev) + for _, i := range dat[1:] { + if highbits(prev) == highbits(i) { + c = c.add(lowbits(i)) + rb.highlowcontainer.setContainerAtIndex(idx, c) + } else { + idx, c = rb.addwithptr(prev) + } + prev = i + } +} + +// BitmapOf generates a new bitmap filled with the specified integers +func BitmapOf(dat ...uint32) *Bitmap { + ans := NewBitmap() + ans.AddMany(dat) + return ans +} + +// Flip negates the bits in the given range (i.e., [rangeStart,rangeEnd)), any integer present in this range and in the bitmap is removed, +// and any integer present in the range and not in the bitmap is added. +// The function uses 64-bit parameters even though a Bitmap stores 32-bit values because it is allowed and meaningful to use [0,uint64(0x100000000)) as a range +// while uint64(0x100000000) cannot be represented as a 32-bit value. +func (rb *Bitmap) Flip(rangeStart, rangeEnd uint64) { + + if rangeStart >= rangeEnd { + return + } + + hbStart := highbits(uint32(rangeStart)) + lbStart := lowbits(uint32(rangeStart)) + hbLast := highbits(uint32(rangeEnd - 1)) + lbLast := lowbits(uint32(rangeEnd - 1)) + + max := toIntUnsigned(maxLowBit()) + for hb := hbStart; hb <= hbLast; hb++ { + containerStart := uint32(0) + if hb == hbStart { + containerStart = toIntUnsigned(lbStart) + } + containerLast := max + if hb == hbLast { + containerLast = toIntUnsigned(lbLast) + } + + i := rb.highlowcontainer.getIndex(hb) + + if i >= 0 { + c := rb.highlowcontainer.getWritableContainerAtIndex(i).inot(int(containerStart), int(containerLast)+1) + if c.getCardinality() > 0 { + rb.highlowcontainer.setContainerAtIndex(i, c) + } else { + rb.highlowcontainer.removeAtIndex(i) + } + } else { // *think* the range of ones must never be + // empty. + rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, rangeOfOnes(int(containerStart), int(containerLast))) + } + } +} + +// FlipInt calls Flip after casting the parameters (convenience method) +func (rb *Bitmap) FlipInt(rangeStart, rangeEnd int) { + rb.Flip(uint64(rangeStart), uint64(rangeEnd)) +} + +// AddRange adds the integers in [rangeStart, rangeEnd) to the bitmap. +// The function uses 64-bit parameters even though a Bitmap stores 32-bit values because it is allowed and meaningful to use [0,uint64(0x100000000)) as a range +// while uint64(0x100000000) cannot be represented as a 32-bit value. +func (rb *Bitmap) AddRange(rangeStart, rangeEnd uint64) { + if rangeStart >= rangeEnd { + return + } + + hbStart := toIntUnsigned(highbits(uint32(rangeStart))) + lbStart := toIntUnsigned(lowbits(uint32(rangeStart))) + hbLast := toIntUnsigned(highbits(uint32(rangeEnd - 1))) + lbLast := toIntUnsigned(lowbits(uint32(rangeEnd - 1))) + + max := toIntUnsigned(maxLowBit()) + for hb := uint16(hbStart); hb <= uint16(hbLast); hb++ { + containerStart := uint32(0) + if hb == uint16(hbStart) { + containerStart = lbStart + } + containerLast := max + if hb == uint16(hbLast) { + containerLast = lbLast + } + + i := rb.highlowcontainer.getIndex(hb) + + if i >= 0 { + c := rb.highlowcontainer.getWritableContainerAtIndex(i).iaddRange(int(containerStart), int(containerLast+1)) + rb.highlowcontainer.setContainerAtIndex(i, c) + } else { // *think* the range of ones must never be + // empty. + rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, rangeOfOnes(int(containerStart), int(containerLast))) + } + } +} + +// RemoveRange removes the integers in [rangeStart, rangeEnd) from the bitmap. +// The function uses 64-bit parameters even though a Bitmap stores 32-bit values because it is allowed and meaningful to use [0,uint64(0x100000000)) as a range +// while uint64(0x100000000) cannot be represented as a 32-bit value. +func (rb *Bitmap) RemoveRange(rangeStart, rangeEnd uint64) { + if rangeStart >= rangeEnd { + return + } + + hbStart := toIntUnsigned(highbits(uint32(rangeStart))) + lbStart := toIntUnsigned(lowbits(uint32(rangeStart))) + hbLast := toIntUnsigned(highbits(uint32(rangeEnd - 1))) + lbLast := toIntUnsigned(lowbits(uint32(rangeEnd - 1))) + + max := toIntUnsigned(maxLowBit()) + + if hbStart == hbLast { + i := rb.highlowcontainer.getIndex(uint16(hbStart)) + if i < 0 { + return + } + c := rb.highlowcontainer.getWritableContainerAtIndex(i).iremoveRange(int(lbStart), int(lbLast+1)) + if c.getCardinality() > 0 { + rb.highlowcontainer.setContainerAtIndex(i, c) + } else { + rb.highlowcontainer.removeAtIndex(i) + } + return + } + ifirst := rb.highlowcontainer.getIndex(uint16(hbStart)) + ilast := rb.highlowcontainer.getIndex(uint16(hbLast)) + + if ifirst >= 0 { + if lbStart != 0 { + c := rb.highlowcontainer.getWritableContainerAtIndex(ifirst).iremoveRange(int(lbStart), int(max+1)) + if c.getCardinality() > 0 { + rb.highlowcontainer.setContainerAtIndex(ifirst, c) + ifirst++ + } + } + } else { + ifirst = -ifirst - 1 + } + if ilast >= 0 { + if lbLast != max { + c := rb.highlowcontainer.getWritableContainerAtIndex(ilast).iremoveRange(int(0), int(lbLast+1)) + if c.getCardinality() > 0 { + rb.highlowcontainer.setContainerAtIndex(ilast, c) + } else { + ilast++ + } + } else { + ilast++ + } + } else { + ilast = -ilast - 1 + } + rb.highlowcontainer.removeIndexRange(ifirst, ilast) +} + +// Flip negates the bits in the given range (i.e., [rangeStart,rangeEnd)), any integer present in this range and in the bitmap is removed, +// and any integer present in the range and not in the bitmap is added, a new bitmap is returned leaving +// the current bitmap unchanged. +// The function uses 64-bit parameters even though a Bitmap stores 32-bit values because it is allowed and meaningful to use [0,uint64(0x100000000)) as a range +// while uint64(0x100000000) cannot be represented as a 32-bit value. +func Flip(bm *Bitmap, rangeStart, rangeEnd uint64) *Bitmap { + if rangeStart >= rangeEnd { + return bm.Clone() + } + + answer := NewBitmap() + hbStart := highbits(uint32(rangeStart)) + lbStart := lowbits(uint32(rangeStart)) + hbLast := highbits(uint32(rangeEnd - 1)) + lbLast := lowbits(uint32(rangeEnd - 1)) + + // copy the containers before the active area + answer.highlowcontainer.appendCopiesUntil(bm.highlowcontainer, hbStart) + + max := toIntUnsigned(maxLowBit()) + for hb := hbStart; hb <= hbLast; hb++ { + containerStart := uint32(0) + if hb == hbStart { + containerStart = toIntUnsigned(lbStart) + } + containerLast := max + if hb == hbLast { + containerLast = toIntUnsigned(lbLast) + } + + i := bm.highlowcontainer.getIndex(hb) + j := answer.highlowcontainer.getIndex(hb) + + if i >= 0 { + c := bm.highlowcontainer.getContainerAtIndex(i).not(int(containerStart), int(containerLast)+1) + if c.getCardinality() > 0 { + answer.highlowcontainer.insertNewKeyValueAt(-j-1, hb, c) + } + + } else { // *think* the range of ones must never be + // empty. + answer.highlowcontainer.insertNewKeyValueAt(-j-1, hb, + rangeOfOnes(int(containerStart), int(containerLast))) + } + } + // copy the containers after the active area. + answer.highlowcontainer.appendCopiesAfter(bm.highlowcontainer, hbLast) + + return answer +} + +// SetCopyOnWrite sets this bitmap to use copy-on-write so that copies are fast and memory conscious +// if the parameter is true, otherwise we leave the default where hard copies are made +// (copy-on-write requires extra care in a threaded context). +func (rb *Bitmap) SetCopyOnWrite(val bool) { + rb.highlowcontainer.copyOnWrite = val +} + +// GetCopyOnWrite gets this bitmap's copy-on-write property +func (rb *Bitmap) GetCopyOnWrite() (val bool) { + return rb.highlowcontainer.copyOnWrite +} + +// FlipInt calls Flip after casting the parameters (convenience method) +func FlipInt(bm *Bitmap, rangeStart, rangeEnd int) *Bitmap { + return Flip(bm, uint64(rangeStart), uint64(rangeEnd)) +} diff --git a/vendor/github.com/RoaringBitmap/roaring/roaringarray.go b/vendor/github.com/RoaringBitmap/roaring/roaringarray.go new file mode 100644 index 00000000..e5bf6cb8 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/roaringarray.go @@ -0,0 +1,463 @@ +package roaring + +import ( + "encoding/binary" + "io" +) + +type container interface { + clone() container + and(container) container + iand(container) container // i stands for inplace + andNot(container) container + iandNot(container) container // i stands for inplace + getCardinality() int + rank(uint16) int + add(uint16) container + //addRange(start, final int) container // range is [firstOfRange,lastOfRange) (unused) + iaddRange(start, final int) container // i stands for inplace, range is [firstOfRange,lastOfRange) + remove(uint16) container + not(start, final int) container // range is [firstOfRange,lastOfRange) + inot(firstOfRange, lastOfRange int) container // i stands for inplace, range is [firstOfRange,lastOfRange) + xor(r container) container + getShortIterator() shortIterable + contains(i uint16) bool + equals(i interface{}) bool + fillLeastSignificant16bits(array []uint32, i int, mask uint32) + or(r container) container + ior(r container) container // i stands for inplace + intersects(r container) bool // whether the two containers intersect + lazyOR(r container) container + lazyIOR(r container) container + getSizeInBytes() int + //removeRange(start, final int) container // range is [firstOfRange,lastOfRange) (unused) + iremoveRange(start, final int) container // i stands for inplace, range is [firstOfRange,lastOfRange) + selectInt(uint16) int + serializedSizeInBytes() int + readFrom(io.Reader) (int, error) + writeTo(io.Writer) (int, error) +} + +// careful: range is [firstOfRange,lastOfRange] +func rangeOfOnes(start, last int) container { + if (last - start + 1) > arrayDefaultMaxSize { + return newBitmapContainerwithRange(start, last) + } + + return newArrayContainerRange(start, last) +} + +type roaringArray struct { + keys []uint16 + containers []container + needCopyOnWrite []bool + copyOnWrite bool +} + +func newRoaringArray() *roaringArray { + ra := &roaringArray{} + ra.clear() + ra.copyOnWrite = false + return ra +} + +func (ra *roaringArray) appendContainer(key uint16, value container, mustCopyOnWrite bool) { + ra.keys = append(ra.keys, key) + ra.containers = append(ra.containers, value) + ra.needCopyOnWrite = append(ra.needCopyOnWrite, mustCopyOnWrite) +} + +func (ra *roaringArray) appendWithoutCopy(sa roaringArray, startingindex int) { + ra.appendContainer(sa.keys[startingindex], sa.containers[startingindex], false) +} + +func (ra *roaringArray) appendCopy(sa roaringArray, startingindex int) { + ra.appendContainer(sa.keys[startingindex], sa.containers[startingindex], true) + sa.setNeedsCopyOnWrite(startingindex) +} + +func (ra *roaringArray) appendWithoutCopyMany(sa roaringArray, startingindex, end int) { + for i := startingindex; i < end; i++ { + ra.appendWithoutCopy(sa, i) + } +} + +func (ra *roaringArray) appendCopyMany(sa roaringArray, startingindex, end int) { + for i := startingindex; i < end; i++ { + ra.appendCopy(sa, i) + } +} + +func (ra *roaringArray) appendCopiesUntil(sa roaringArray, stoppingKey uint16) { + for i := 0; i < sa.size(); i++ { + if sa.keys[i] >= stoppingKey { + break + } + ra.appendContainer(sa.keys[i], sa.containers[i], true) + sa.setNeedsCopyOnWrite(i) + } +} + +func (ra *roaringArray) appendCopiesAfter(sa roaringArray, beforeStart uint16) { + startLocation := sa.getIndex(beforeStart) + if startLocation >= 0 { + startLocation++ + } else { + startLocation = -startLocation - 1 + } + + for i := startLocation; i < sa.size(); i++ { + ra.appendContainer(sa.keys[i], sa.containers[i], true) + sa.setNeedsCopyOnWrite(i) + } +} + +func (ra *roaringArray) removeIndexRange(begin, end int) { + if end <= begin { + return + } + + r := end - begin + + copy(ra.keys[begin:], ra.keys[end:]) + copy(ra.containers[begin:], ra.containers[end:]) + copy(ra.needCopyOnWrite[begin:], ra.needCopyOnWrite[end:]) + + ra.resize(len(ra.keys) - r) +} + +func (ra *roaringArray) resize(newsize int) { + for k := newsize; k < len(ra.containers); k++ { + ra.containers[k] = nil + } + + ra.keys = ra.keys[:newsize] + ra.containers = ra.containers[:newsize] + ra.needCopyOnWrite = ra.needCopyOnWrite[:newsize] +} + +func (ra *roaringArray) clear() { + ra.keys = make([]uint16, 0) + ra.containers = make([]container, 0) + ra.needCopyOnWrite = make([]bool, 0) +} + +func (ra *roaringArray) clone() *roaringArray { + sa := new(roaringArray) + sa.keys = make([]uint16, len(ra.keys)) + sa.containers = make([]container, len(ra.containers)) + sa.needCopyOnWrite = make([]bool, len(ra.needCopyOnWrite)) + sa.copyOnWrite = ra.copyOnWrite + copy(sa.keys, ra.keys) + if sa.copyOnWrite { + copy(sa.keys, ra.keys) + copy(sa.containers, ra.containers) + sa.markAllAsNeedingCopyOnWrite() + ra.markAllAsNeedingCopyOnWrite() + } else { + for i := range sa.needCopyOnWrite { + sa.needCopyOnWrite[i] = false + } + for i := range sa.containers { + sa.containers[i] = ra.containers[i].clone() + } + } + return sa +} + +func (ra *roaringArray) containsKey(x uint16) bool { + return (ra.binarySearch(0, len(ra.keys), x) >= 0) +} + +func (ra *roaringArray) getContainer(x uint16) container { + i := ra.binarySearch(0, len(ra.keys), x) + if i < 0 { + return nil + } + return ra.containers[i] +} + +func (ra *roaringArray) getWritableContainerContainer(x uint16) container { + i := ra.binarySearch(0, len(ra.keys), x) + if i < 0 { + return nil + } + if ra.needCopyOnWrite[i] { + ra.containers[i] = ra.containers[i].clone() + ra.needCopyOnWrite[i] = false + } + return ra.containers[i] +} + +func (ra *roaringArray) getContainerAtIndex(i int) container { + return ra.containers[i] +} + +func (ra *roaringArray) getWritableContainerAtIndex(i int) container { + if ra.needCopyOnWrite[i] { + ra.containers[i] = ra.containers[i].clone() + ra.needCopyOnWrite[i] = false + } + return ra.containers[i] +} + +func (ra *roaringArray) getIndex(x uint16) int { + // before the binary search, we optimize for frequent cases + size := len(ra.keys) + if (size == 0) || (ra.keys[size-1] == x) { + return size - 1 + } + return ra.binarySearch(0, size, x) +} + +func (ra *roaringArray) getKeyAtIndex(i int) uint16 { + return ra.keys[i] +} + +func (ra *roaringArray) insertNewKeyValueAt(i int, key uint16, value container) { + ra.keys = append(ra.keys, 0) + ra.containers = append(ra.containers, nil) + + copy(ra.keys[i+1:], ra.keys[i:]) + copy(ra.containers[i+1:], ra.containers[i:]) + + ra.keys[i] = key + ra.containers[i] = value + + ra.needCopyOnWrite = append(ra.needCopyOnWrite, false) + copy(ra.needCopyOnWrite[i+1:], ra.needCopyOnWrite[i:]) + ra.needCopyOnWrite[i] = false +} + +func (ra *roaringArray) remove(key uint16) bool { + i := ra.binarySearch(0, len(ra.keys), key) + if i >= 0 { // if a new key + ra.removeAtIndex(i) + return true + } + return false +} + +func (ra *roaringArray) removeAtIndex(i int) { + copy(ra.keys[i:], ra.keys[i+1:]) + copy(ra.containers[i:], ra.containers[i+1:]) + + copy(ra.needCopyOnWrite[i:], ra.needCopyOnWrite[i+1:]) + + ra.resize(len(ra.keys) - 1) +} + +func (ra *roaringArray) setContainerAtIndex(i int, c container) { + ra.containers[i] = c +} + +func (ra *roaringArray) replaceKeyAndContainerAtIndex(i int, key uint16, c container, mustCopyOnWrite bool) { + ra.keys[i] = key + ra.containers[i] = c + ra.needCopyOnWrite[i] = mustCopyOnWrite +} + +func (ra *roaringArray) size() int { + return len(ra.keys) +} + +func (ra *roaringArray) binarySearch(begin, end int, ikey uint16) int { + low := begin + high := end - 1 + for low+16 <= high { + middleIndex := int(uint((low + high)) >> 1) + middleValue := ra.keys[middleIndex] + + if middleValue < ikey { + low = middleIndex + 1 + } else if middleValue > ikey { + high = middleIndex - 1 + } else { + return middleIndex + } + } + for ; low <= high; low++ { + val := ra.keys[low] + if val >= ikey { + if val == ikey { + return low + } + break + } + } + return -(low + 1) +} + +func (ra *roaringArray) equals(o interface{}) bool { + srb, ok := o.(roaringArray) + if ok { + + if srb.size() != ra.size() { + return false + } + for i, k := range ra.keys { + if k != srb.keys[i] { + return false + } + } + + for i, c := range ra.containers { + if !c.equals(srb.containers[i]) { + return false + } + } + return true + } + return false +} + +func (ra *roaringArray) serializedSizeInBytes() uint64 { + count := uint64(4 + 4) + for _, c := range ra.containers { + count = count + 4 + 4 + count = count + uint64(c.serializedSizeInBytes()) + } + return count +} + +func (ra *roaringArray) writeTo(stream io.Writer) (int64, error) { + preambleSize := 4 + 4 + 4*len(ra.keys) + buf := make([]byte, preambleSize+4*len(ra.keys)) + binary.LittleEndian.PutUint32(buf[0:], uint32(serialCookie)) + binary.LittleEndian.PutUint32(buf[4:], uint32(len(ra.keys))) + + for i, key := range ra.keys { + off := 8 + i*4 + binary.LittleEndian.PutUint16(buf[off:], uint16(key)) + + c := ra.containers[i] + binary.LittleEndian.PutUint16(buf[off+2:], uint16(c.getCardinality()-1)) + } + + startOffset := int64(preambleSize + 4*len(ra.keys)) + for i, c := range ra.containers { + binary.LittleEndian.PutUint32(buf[preambleSize+i*4:], uint32(startOffset)) + startOffset += int64(getSizeInBytesFromCardinality(c.getCardinality())) + } + + _, err := stream.Write(buf) + if err != nil { + return 0, err + } + + for _, c := range ra.containers { + _, err := c.writeTo(stream) + if err != nil { + return 0, err + } + } + return startOffset, nil +} + +func (ra *roaringArray) readFrom(stream io.Reader) (int64, error) { + var cookie uint32 + err := binary.Read(stream, binary.LittleEndian, &cookie) + if err != nil { + return 0, err + } + if cookie != serialCookie { + return 0, err + } + var size uint32 + err = binary.Read(stream, binary.LittleEndian, &size) + if err != nil { + return 0, err + } + keycard := make([]uint16, 2*size, 2*size) + err = binary.Read(stream, binary.LittleEndian, keycard) + if err != nil { + return 0, err + } + offsets := make([]uint32, size, size) + err = binary.Read(stream, binary.LittleEndian, offsets) + if err != nil { + return 0, err + } + offset := int64(4 + 4 + 8*size) + for i := uint32(0); i < size; i++ { + c := int(keycard[2*i+1]) + 1 + offset += int64(getSizeInBytesFromCardinality(c)) + if c > arrayDefaultMaxSize { + nb := newBitmapContainer() + nb.readFrom(stream) + nb.cardinality = int(c) + ra.appendContainer(keycard[2*i], nb, false) + } else { + nb := newArrayContainerSize(int(c)) + nb.readFrom(stream) + ra.appendContainer(keycard[2*i], nb, false) + } + } + return offset, nil +} + +func (ra *roaringArray) advanceUntil(min uint16, pos int) int { + lower := pos + 1 + + if lower >= len(ra.keys) || ra.keys[lower] >= min { + return lower + } + + spansize := 1 + + for lower+spansize < len(ra.keys) && ra.keys[lower+spansize] < min { + spansize *= 2 + } + var upper int + if lower+spansize < len(ra.keys) { + upper = lower + spansize + } else { + upper = len(ra.keys) - 1 + } + + if ra.keys[upper] == min { + return upper + } + + if ra.keys[upper] < min { + // means + // array + // has no + // item + // >= min + // pos = array.length; + return len(ra.keys) + } + + // we know that the next-smallest span was too small + lower += (spansize / 2) + + mid := 0 + for lower+1 != upper { + mid = (lower + upper) / 2 + if ra.keys[mid] == min { + return mid + } else if ra.keys[mid] < min { + lower = mid + } else { + upper = mid + } + } + return upper +} + +func (ra *roaringArray) markAllAsNeedingCopyOnWrite() { + needCopyOnWrite := make([]bool, len(ra.keys)) + for i := range needCopyOnWrite { + needCopyOnWrite[i] = true + } + ra.needCopyOnWrite = needCopyOnWrite +} + +func (ra *roaringArray) needsCopyOnWrite(i int) bool { + return ra.needCopyOnWrite[i] +} + +func (ra *roaringArray) setNeedsCopyOnWrite(i int) { + ra.needCopyOnWrite[i] = true +} diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go b/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go new file mode 100644 index 00000000..185e15c1 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go @@ -0,0 +1,51 @@ +// +build !amd64,!386 appengine + +package roaring + +import ( + "encoding/binary" + "io" +) + +func (b *arrayContainer) writeTo(stream io.Writer) (int, error) { + buf := make([]byte, 2*len(b.content)) + for i, v := range b.content { + base := i * 2 + buf[base] = byte(v) + buf[base+1] = byte(v >> 8) + } + return stream.Write(buf) +} + +func (b *arrayContainer) readFrom(stream io.Reader) (int, error) { + err := binary.Read(stream, binary.LittleEndian, b.content) + if err != nil { + return 0, err + } + return 2 * len(b.content), nil +} + +func (b *bitmapContainer) writeTo(stream io.Writer) (int, error) { + // Write set + buf := make([]byte, 8*len(b.bitmap)) + for i, v := range b.bitmap { + base := i * 8 + buf[base] = byte(v) + buf[base+1] = byte(v >> 8) + buf[base+2] = byte(v >> 16) + buf[base+3] = byte(v >> 24) + buf[base+4] = byte(v >> 32) + buf[base+5] = byte(v >> 40) + buf[base+6] = byte(v >> 48) + buf[base+7] = byte(v >> 56) + } + return stream.Write(buf) +} + +func (b *bitmapContainer) readFrom(stream io.Reader) (int, error) { + err := binary.Read(stream, binary.LittleEndian, b.bitmap) + if err != nil { + return 0, err + } + return 8 * len(b.bitmap), nil +} diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go b/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go new file mode 100644 index 00000000..e099ed79 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go @@ -0,0 +1,53 @@ +// +build 386 amd64,!appengine + +package roaring + +import ( + "io" + "reflect" + "unsafe" +) + +func (b *arrayContainer) writeTo(stream io.Writer) (int, error) { + buf := uint16SliceAsByteSlice(b.content) + return stream.Write(buf) +} + +func (b *bitmapContainer) writeTo(stream io.Writer) (int, error) { + buf := uint64SliceAsByteSlice(b.bitmap) + return stream.Write(buf) +} + +func (b *arrayContainer) readFrom(stream io.Reader) (int, error) { + buf := uint16SliceAsByteSlice(b.content) + return io.ReadFull(stream, buf) +} + +func (b *bitmapContainer) readFrom(stream io.Reader) (int, error) { + buf := uint64SliceAsByteSlice(b.bitmap) + return io.ReadFull(stream, buf) +} + +func uint64SliceAsByteSlice(slice []uint64) []byte { + // make a new slice header + header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice)) + + // update its capacity and length + header.Len *= 8 + header.Cap *= 8 + + // return it + return *(*[]byte)(unsafe.Pointer(&header)) +} + +func uint16SliceAsByteSlice(slice []uint16) []byte { + // make a new slice header + header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice)) + + // update its capacity and length + header.Len *= 2 + header.Cap *= 2 + + // return it + return *(*[]byte)(unsafe.Pointer(&header)) +} diff --git a/vendor/github.com/RoaringBitmap/roaring/setutil.go b/vendor/github.com/RoaringBitmap/roaring/setutil.go new file mode 100644 index 00000000..42c0a757 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/setutil.go @@ -0,0 +1,446 @@ +package roaring + +func equal(a, b []uint16) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func difference(set1 []uint16, set2 []uint16, buffer []uint16) int { + if 0 == len(set2) { + for k := 0; k < len(set1); k++ { + buffer[k] = set1[k] + } + return len(set1) + } + if 0 == len(set1) { + return 0 + } + pos := 0 + k1 := 0 + k2 := 0 + buffer = buffer[:cap(buffer)] + s1 := set1[k1] + s2 := set2[k2] + for { + if s1 < s2 { + buffer[pos] = s1 + pos++ + k1++ + if k1 >= len(set1) { + break + } + s1 = set1[k1] + } else if s1 == s2 { + k1++ + k2++ + if k1 >= len(set1) { + break + } + s1 = set1[k1] + if k2 >= len(set2) { + for ; k1 < len(set1); k1++ { + buffer[pos] = set1[k1] + pos++ + } + break + } + s2 = set2[k2] + } else { // if (val1>val2) + k2++ + if k2 >= len(set2) { + for ; k1 < len(set1); k1++ { + buffer[pos] = set1[k1] + pos++ + } + break + } + s2 = set2[k2] + } + } + return pos + +} + +func exclusiveUnion2by2(set1 []uint16, set2 []uint16, buffer []uint16) int { + if 0 == len(set2) { + buffer = buffer[:len(set1)] + copy(buffer, set1[:]) + return len(set1) + } + if 0 == len(set1) { + buffer = buffer[:len(set2)] + copy(buffer, set2[:]) + return len(set2) + } + pos := 0 + k1 := 0 + k2 := 0 + s1 := set1[k1] + s2 := set2[k2] + buffer = buffer[:cap(buffer)] + for { + if s1 < s2 { + buffer[pos] = s1 + pos++ + k1++ + if k1 >= len(set1) { + for ; k2 < len(set2); k2++ { + buffer[pos] = set2[k2] + pos++ + } + break + } + s1 = set1[k1] + } else if s1 == s2 { + k1++ + k2++ + if k1 >= len(set1) { + for ; k2 < len(set2); k2++ { + buffer[pos] = set2[k2] + pos++ + } + break + } + if k2 >= len(set2) { + for ; k1 < len(set1); k1++ { + buffer[pos] = set1[k1] + pos++ + } + break + } + s1 = set1[k1] + s2 = set2[k2] + } else { // if (val1>val2) + buffer[pos] = s2 + pos++ + k2++ + if k2 >= len(set2) { + for ; k1 < len(set1); k1++ { + buffer[pos] = set1[k1] + pos++ + } + break + } + s2 = set2[k2] + } + } + return pos +} + +func union2by2(set1 []uint16, set2 []uint16, buffer []uint16) int { + pos := 0 + k1 := 0 + k2 := 0 + if 0 == len(set2) { + buffer = buffer[:len(set1)] + copy(buffer, set1[:]) + return len(set1) + } + if 0 == len(set1) { + buffer = buffer[:len(set2)] + copy(buffer, set2[:]) + return len(set2) + } + s1 := set1[k1] + s2 := set2[k2] + buffer = buffer[:cap(buffer)] + for { + if s1 < s2 { + buffer[pos] = s1 + pos++ + k1++ + if k1 >= len(set1) { + copy(buffer[pos:], set2[k2:]) + pos += len(set2) - k2 + break + } + s1 = set1[k1] + } else if s1 == s2 { + buffer[pos] = s1 + pos++ + k1++ + k2++ + if k1 >= len(set1) { + copy(buffer[pos:], set2[k2:]) + pos += len(set2) - k2 + break + } + if k2 >= len(set2) { + copy(buffer[pos:], set1[k1:]) + pos += len(set1) - k1 + break + } + s1 = set1[k1] + s2 = set2[k2] + } else { // if (set1[k1]>set2[k2]) + buffer[pos] = s2 + pos++ + k2++ + if k2 >= len(set2) { + copy(buffer[pos:], set1[k1:]) + pos += len(set1) - k1 + break + } + s2 = set2[k2] + } + } + return pos +} + +func intersection2by2( + set1 []uint16, + set2 []uint16, + buffer []uint16) int { + + if len(set1)*64 < len(set2) { + return onesidedgallopingintersect2by2(set1, set2, buffer) + } else if len(set2)*64 < len(set1) { + return onesidedgallopingintersect2by2(set2, set1, buffer) + } else { + return localintersect2by2(set1, set2, buffer) + } +} + +func intersects2by2( + set1 []uint16, + set2 []uint16) bool { + // could be optimized if one set is much larger than the other one + if (0 == len(set1)) || (0 == len(set2)) { + return false + } + k1 := 0 + k2 := 0 + s1 := set1[k1] + s2 := set2[k2] +mainwhile: + for { + + if s2 < s1 { + for { + k2++ + if k2 == len(set2) { + break mainwhile + } + s2 = set2[k2] + if s2 >= s1 { + break + } + } + } + if s1 < s2 { + for { + k1++ + if k1 == len(set1) { + break mainwhile + } + s1 = set1[k1] + if s1 >= s2 { + break + } + } + + } else { + // (set2[k2] == set1[k1]) + return true + } + } + return false +} + +func localintersect2by2( + set1 []uint16, + set2 []uint16, + buffer []uint16) int { + + if (0 == len(set1)) || (0 == len(set2)) { + return 0 + } + k1 := 0 + k2 := 0 + pos := 0 + buffer = buffer[:cap(buffer)] + s1 := set1[k1] + s2 := set2[k2] +mainwhile: + for { + + if s2 < s1 { + for { + k2++ + if k2 == len(set2) { + break mainwhile + } + s2 = set2[k2] + if s2 >= s1 { + break + } + } + } + if s1 < s2 { + for { + k1++ + if k1 == len(set1) { + break mainwhile + } + s1 = set1[k1] + if s1 >= s2 { + break + } + } + + } else { + // (set2[k2] == set1[k1]) + buffer[pos] = s1 + pos++ + k1++ + if k1 == len(set1) { + break + } + s1 = set1[k1] + k2++ + if k2 == len(set2) { + break + } + s2 = set2[k2] + } + } + return pos +} + +func advanceUntil( + array []uint16, + pos int, + length int, + min uint16) int { + lower := pos + 1 + + if lower >= length || array[lower] >= min { + return lower + } + + spansize := 1 + + for lower+spansize < length && array[lower+spansize] < min { + spansize *= 2 + } + var upper int + if lower+spansize < length { + upper = lower + spansize + } else { + upper = length - 1 + } + + if array[upper] == min { + return upper + } + + if array[upper] < min { + // means + // array + // has no + // item + // >= min + // pos = array.length; + return length + } + + // we know that the next-smallest span was too small + lower += (spansize / 2) + + mid := 0 + for lower+1 != upper { + mid = (lower + upper) / 2 + if array[mid] == min { + return mid + } else if array[mid] < min { + lower = mid + } else { + upper = mid + } + } + return upper + +} + +func onesidedgallopingintersect2by2( + smallset []uint16, + largeset []uint16, + buffer []uint16) int { + + if 0 == len(smallset) { + return 0 + } + buffer = buffer[:cap(buffer)] + k1 := 0 + k2 := 0 + pos := 0 + s1 := largeset[k1] + s2 := smallset[k2] +mainwhile: + + for { + if s1 < s2 { + k1 = advanceUntil(largeset, k1, len(largeset), s2) + if k1 == len(largeset) { + break mainwhile + } + s1 = largeset[k1] + } + if s2 < s1 { + k2++ + if k2 == len(smallset) { + break mainwhile + } + s2 = smallset[k2] + } else { + + buffer[pos] = s2 + pos++ + k2++ + if k2 == len(smallset) { + break + } + s2 = smallset[k2] + k1 = advanceUntil(largeset, k1, len(largeset), s2) + if k1 == len(largeset) { + break mainwhile + } + s1 = largeset[k1] + } + + } + return pos +} + +func binarySearch(array []uint16, ikey uint16) int { + low := 0 + high := len(array) - 1 + for low+16 <= high { + middleIndex := int(uint32(low+high) >> 1) + middleValue := array[middleIndex] + if middleValue < ikey { + low = middleIndex + 1 + } else if middleValue > ikey { + high = middleIndex - 1 + } else { + return middleIndex + } + } + for ; low <= high; low++ { + val := array[low] + if val >= ikey { + if val == ikey { + return low + } + break + } + } + return -(low + 1) +} diff --git a/vendor/github.com/RoaringBitmap/roaring/shortiterator.go b/vendor/github.com/RoaringBitmap/roaring/shortiterator.go new file mode 100644 index 00000000..ef0acbd1 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/shortiterator.go @@ -0,0 +1,21 @@ +package roaring + +type shortIterable interface { + hasNext() bool + next() uint16 +} + +type shortIterator struct { + slice []uint16 + loc int +} + +func (si *shortIterator) hasNext() bool { + return si.loc < len(si.slice) +} + +func (si *shortIterator) next() uint16 { + a := si.slice[si.loc] + si.loc++ + return a +} diff --git a/vendor/github.com/RoaringBitmap/roaring/smat.go b/vendor/github.com/RoaringBitmap/roaring/smat.go new file mode 100644 index 00000000..5c663237 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/smat.go @@ -0,0 +1,376 @@ +// +build gofuzz + +/* +# Instructions for smat testing for roaring + +[smat](https://github.com/mschoch/smat) is a framework that provides +state machine assisted fuzz testing. + +To run the smat tests for roaring... + +## Prerequisites + + $ go get github.com/dvyukov/go-fuzz/go-fuzz + $ go get github.com/dvyukov/go-fuzz/go-fuzz-build + +## Steps + +1. Generate initial smat corpus: +``` + go test -tags=gofuzz -run=TestGenerateSmatCorpus +``` + +2. Build go-fuzz test program with instrumentation: +``` + go-fuzz-build github.com/RoaringBitmap/roaring +``` + +3. Run go-fuzz: +``` + go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200 +``` + +You should see output like... +``` +2016/09/16 13:58:35 slaves: 8, corpus: 1 (3s ago), crashers: 0, restarts: 1/0, execs: 0 (0/sec), cover: 0, uptime: 3s +2016/09/16 13:58:38 slaves: 8, corpus: 1 (6s ago), crashers: 0, restarts: 1/0, execs: 0 (0/sec), cover: 0, uptime: 6s +2016/09/16 13:58:41 slaves: 8, corpus: 1 (9s ago), crashers: 0, restarts: 1/44, execs: 44 (5/sec), cover: 0, uptime: 9s +2016/09/16 13:58:44 slaves: 8, corpus: 1 (12s ago), crashers: 0, restarts: 1/45, execs: 45 (4/sec), cover: 0, uptime: 12s +2016/09/16 13:58:47 slaves: 8, corpus: 1 (15s ago), crashers: 0, restarts: 1/46, execs: 46 (3/sec), cover: 0, uptime: 15s +2016/09/16 13:58:50 slaves: 8, corpus: 1 (18s ago), crashers: 0, restarts: 1/47, execs: 47 (3/sec), cover: 0, uptime: 18s +2016/09/16 13:58:53 slaves: 8, corpus: 1 (21s ago), crashers: 0, restarts: 1/63, execs: 63 (3/sec), cover: 0, uptime: 21s +2016/09/16 13:58:56 slaves: 8, corpus: 1 (24s ago), crashers: 0, restarts: 1/65, execs: 65 (3/sec), cover: 0, uptime: 24s +2016/09/16 13:58:59 slaves: 8, corpus: 1 (27s ago), crashers: 0, restarts: 1/66, execs: 66 (2/sec), cover: 0, uptime: 27s +2016/09/16 13:59:02 slaves: 8, corpus: 1 (30s ago), crashers: 0, restarts: 1/67, execs: 67 (2/sec), cover: 0, uptime: 30s +2016/09/16 13:59:05 slaves: 8, corpus: 1 (33s ago), crashers: 0, restarts: 1/83, execs: 83 (3/sec), cover: 0, uptime: 33s +2016/09/16 13:59:08 slaves: 8, corpus: 1 (36s ago), crashers: 0, restarts: 1/84, execs: 84 (2/sec), cover: 0, uptime: 36s +2016/09/16 13:59:11 slaves: 8, corpus: 2 (0s ago), crashers: 0, restarts: 1/85, execs: 85 (2/sec), cover: 0, uptime: 39s +2016/09/16 13:59:14 slaves: 8, corpus: 17 (2s ago), crashers: 0, restarts: 1/86, execs: 86 (2/sec), cover: 480, uptime: 42s +2016/09/16 13:59:17 slaves: 8, corpus: 17 (5s ago), crashers: 0, restarts: 1/66, execs: 132 (3/sec), cover: 487, uptime: 45s +2016/09/16 13:59:20 slaves: 8, corpus: 17 (8s ago), crashers: 0, restarts: 1/440, execs: 2645 (55/sec), cover: 487, uptime: 48s + +``` + +Let it run, and if the # of crashers is > 0, check out the reports in +the workdir where you should be able to find the panic goroutine stack +traces. +*/ + + +package roaring + +import ( + "fmt" + "sort" + + "github.com/mschoch/smat" + "github.com/willf/bitset" +) + +// fuzz test using state machine driven by byte stream. +func Fuzz(data []byte) int { + return smat.Fuzz(&smatContext{}, smat.ActionID('S'), smat.ActionID('T'), + smatActionMap, data) +} + +var smatDebug = false + +func smatLog(prefix, format string, args ...interface{}) { + if smatDebug { + fmt.Print(prefix) + fmt.Printf(format, args...) + } +} + +type smatContext struct { + pairs []*smatPair + + // Two registers, x & y. + x int + y int + + actions int +} + +type smatPair struct { + bm *Bitmap + bs *bitset.BitSet +} + +// ------------------------------------------------------------------ + +var smatActionMap = smat.ActionMap{ + smat.ActionID('X'): smatAction("x++", smatWrap(func(c *smatContext) { c.x++ })), + smat.ActionID('x'): smatAction("x--", smatWrap(func(c *smatContext) { c.x-- })), + smat.ActionID('Y'): smatAction("y++", smatWrap(func(c *smatContext) { c.y++ })), + smat.ActionID('y'): smatAction("y--", smatWrap(func(c *smatContext) { c.y-- })), + smat.ActionID('*'): smatAction("x*y", smatWrap(func(c *smatContext) { c.x = c.x * c.y })), + smat.ActionID('<'): smatAction("x<<", smatWrap(func(c *smatContext) { c.x = c.x << 1 })), + + smat.ActionID('^'): smatAction("swap", smatWrap(func(c *smatContext) { c.x, c.y = c.y, c.x })), + + smat.ActionID('['): smatAction(" pushPair", smatWrap(smatPushPair)), + smat.ActionID(']'): smatAction(" popPair", smatWrap(smatPopPair)), + + smat.ActionID('B'): smatAction(" setBit", smatWrap(smatSetBit)), + smat.ActionID('b'): smatAction(" removeBit", smatWrap(smatRemoveBit)), + + smat.ActionID('o'): smatAction(" or", smatWrap(smatOr)), + smat.ActionID('a'): smatAction(" and", smatWrap(smatAnd)), + + smat.ActionID('#'): smatAction(" cardinality", smatWrap(smatCardinality)), + + smat.ActionID('O'): smatAction(" orCardinality", smatWrap(smatOrCardinality)), + smat.ActionID('A'): smatAction(" andCardinality", smatWrap(smatAndCardinality)), + + smat.ActionID('c'): smatAction(" clear", smatWrap(smatClear)), + + smat.ActionID('e'): smatAction(" isEmpty", smatWrap(smatIsEmpty)), + + smat.ActionID('i'): smatAction(" intersects", smatWrap(smatIntersects)), + + smat.ActionID('f'): smatAction(" flip", smatWrap(smatFlip)), + + smat.ActionID('-'): smatAction(" difference", smatWrap(smatDifference)), +} + +var smatRunningPercentActions []smat.PercentAction + +func init() { + var ids []int + for actionId := range smatActionMap { + ids = append(ids, int(actionId)) + } + sort.Ints(ids) + + pct := 100 / len(smatActionMap) + for _, actionId := range ids { + smatRunningPercentActions = append(smatRunningPercentActions, + smat.PercentAction{pct, smat.ActionID(actionId)}) + } + + smatActionMap[smat.ActionID('S')] = smatAction("SETUP", smatSetupFunc) + smatActionMap[smat.ActionID('T')] = smatAction("TEARDOWN", smatTeardownFunc) +} + +// We only have one smat state: running. +func smatRunning(next byte) smat.ActionID { + return smat.PercentExecute(next, smatRunningPercentActions...) +} + +func smatAction(name string, f func(ctx smat.Context) (smat.State, error)) func(smat.Context) (smat.State, error) { + return func(ctx smat.Context) (smat.State, error) { + c := ctx.(*smatContext) + c.actions++ + + smatLog(" ", "%s\n", name) + + return f(ctx) + } +} + +// Creates an smat action func based on a simple callback. +func smatWrap(cb func(c *smatContext)) func(smat.Context) (next smat.State, err error) { + return func(ctx smat.Context) (next smat.State, err error) { + c := ctx.(*smatContext) + cb(c) + return smatRunning, nil + } +} + +// Invokes a callback function with the input v bounded to len(c.pairs). +func (c *smatContext) withPair(v int, cb func(*smatPair)) { + if len(c.pairs) > 0 { + if v < 0 { + v = -v + } + v = v % len(c.pairs) + cb(c.pairs[v]) + } +} + +// ------------------------------------------------------------------ + +func smatSetupFunc(ctx smat.Context) (next smat.State, err error) { + return smatRunning, nil +} + +func smatTeardownFunc(ctx smat.Context) (next smat.State, err error) { + return nil, err +} + +// ------------------------------------------------------------------ + +func smatPushPair(c *smatContext) { + c.pairs = append(c.pairs, &smatPair{ + bm: NewBitmap(), + bs: bitset.New(100), + }) +} + +func smatPopPair(c *smatContext) { + if len(c.pairs) > 0 { + c.pairs = c.pairs[0 : len(c.pairs)-1] + } +} + +func smatSetBit(c *smatContext) { + c.withPair(c.x, func(p *smatPair) { + y := uint32(c.y) + p.bm.AddInt(int(y)) + p.bs.Set(uint(y)) + p.checkEquals() + }) +} + +func smatRemoveBit(c *smatContext) { + c.withPair(c.x, func(p *smatPair) { + y := uint32(c.y) + p.bm.Remove(y) + p.bs.Clear(uint(y)) + p.checkEquals() + }) +} + +func smatAnd(c *smatContext) { + c.withPair(c.x, func(px *smatPair) { + c.withPair(c.y, func(py *smatPair) { + px.bm.And(py.bm) + px.bs = px.bs.Intersection(py.bs) + px.checkEquals() + py.checkEquals() + }) + }) +} + +func smatOr(c *smatContext) { + c.withPair(c.x, func(px *smatPair) { + c.withPair(c.y, func(py *smatPair) { + px.bm.Or(py.bm) + px.bs = px.bs.Union(py.bs) + px.checkEquals() + py.checkEquals() + }) + }) +} + +func smatAndCardinality(c *smatContext) { + c.withPair(c.x, func(px *smatPair) { + c.withPair(c.y, func(py *smatPair) { + c0 := px.bm.AndCardinality(py.bm) + c1 := px.bs.IntersectionCardinality(py.bs) + if c0 != uint64(c1) { + panic("expected same add cardinality") + } + px.checkEquals() + py.checkEquals() + }) + }) +} + +func smatOrCardinality(c *smatContext) { + c.withPair(c.x, func(px *smatPair) { + c.withPair(c.y, func(py *smatPair) { + c0 := px.bm.OrCardinality(py.bm) + c1 := px.bs.UnionCardinality(py.bs) + if c0 != uint64(c1) { + panic("expected same or cardinality") + } + px.checkEquals() + py.checkEquals() + }) + }) +} + +func smatClear(c *smatContext) { + c.withPair(c.x, func(px *smatPair) { + px.bm.Clear() + px.bs = px.bs.ClearAll() + px.checkEquals() + }) +} + +func smatCardinality(c *smatContext) { + c.withPair(c.x, func(px *smatPair) { + c0 := px.bm.GetCardinality() + c1 := px.bs.Count() + if c0 != uint64(c1) { + panic("expected same cardinality") + } + }) +} + +func smatIsEmpty(c *smatContext) { + c.withPair(c.x, func(px *smatPair) { + c0 := px.bm.IsEmpty() + c1 := px.bs.None() + if c0 != c1 { + panic("expected same is empty") + } + }) +} + +func smatIntersects(c *smatContext) { + c.withPair(c.x, func(px *smatPair) { + c.withPair(c.y, func(py *smatPair) { + v0 := px.bm.Intersects(py.bm) + v1 := px.bs.IntersectionCardinality(py.bs) > 0 + if v0 != v1 { + panic("intersects not equal") + } + + px.checkEquals() + py.checkEquals() + }) + }) +} + +func smatFlip(c *smatContext) { + c.withPair(c.x, func(p *smatPair) { + y := uint32(c.y) + p.bm.Flip(uint64(y), uint64(y)+1) + p.bs = p.bs.Flip(uint(y)) + p.checkEquals() + }) +} + +func smatDifference(c *smatContext) { + c.withPair(c.x, func(px *smatPair) { + c.withPair(c.y, func(py *smatPair) { + px.bm.AndNot(py.bm) + px.bs = px.bs.Difference(py.bs) + px.checkEquals() + py.checkEquals() + }) + }) +} + +func (p *smatPair) checkEquals() { + if !p.equalsBitSet(p.bs, p.bm) { + panic("bitset mismatch") + } +} + +func (p *smatPair) equalsBitSet(a *bitset.BitSet, b *Bitmap) bool { + for i, e := a.NextSet(0); e; i, e = a.NextSet(i + 1) { + if !b.ContainsInt(int(i)) { + fmt.Printf("in a bitset, not b bitmap, i: %d\n", i) + fmt.Printf(" a bitset: %s\n b bitmap: %s\n", + a.String(), b.String()) + return false + } + } + + i := b.Iterator() + for i.HasNext() { + v := i.Next() + if !a.Test(uint(v)) { + fmt.Printf("in b bitmap, not a bitset, v: %d\n", v) + fmt.Printf(" a bitset: %s\n b bitmap: %s\n", + a.String(), b.String()) + return false + } + } + + return true +} diff --git a/vendor/github.com/RoaringBitmap/roaring/util.go b/vendor/github.com/RoaringBitmap/roaring/util.go new file mode 100644 index 00000000..0f117ef9 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/util.go @@ -0,0 +1,255 @@ +package roaring + +const ( + arrayDefaultMaxSize = 4096 // containers with 4096 or fewer integers should be array containers. + arrayLazyLowerBound = 1024 + maxCapacity = 1 << 16 + serialCookie = 12346 + invalidCardinality = -1 +) + +func getSizeInBytesFromCardinality(card int) int { + if card > arrayDefaultMaxSize { + return maxCapacity / 8 + } + return 2 * int(card) +} + +// should be replaced with optimized assembly instructions +func numberOfTrailingZeros(i uint64) int { + if i == 0 { + return 64 + } + x := i + n := int64(63) + y := x << 32 + if y != 0 { + n -= 32 + x = y + } + y = x << 16 + if y != 0 { + n -= 16 + x = y + } + y = x << 8 + if y != 0 { + n -= 8 + x = y + } + y = x << 4 + if y != 0 { + n -= 4 + x = y + } + y = x << 2 + if y != 0 { + n -= 2 + x = y + } + return int(n - int64(uint64(x<<1)>>63)) +} + +func fill(arr []uint64, val uint64) { + for i := range arr { + arr[i] = val + } +} +func fillRange(arr []uint64, start, end int, val uint64) { + for i := start; i < end; i++ { + arr[i] = val + } +} + +func fillArrayAND(container []uint16, bitmap1, bitmap2 []uint64) { + if len(bitmap1) != len(bitmap2) { + panic("array lengths don't match") + } + // TODO: rewrite in assembly + pos := 0 + for k := range bitmap1 { + bitset := bitmap1[k] & bitmap2[k] + for bitset != 0 { + t := bitset & -bitset + container[pos] = uint16((k*64 + int(popcount(t-1)))) + pos = pos + 1 + bitset ^= t + } + } +} + +func fillArrayANDNOT(container []uint16, bitmap1, bitmap2 []uint64) { + if len(bitmap1) != len(bitmap2) { + panic("array lengths don't match") + } + // TODO: rewrite in assembly + pos := 0 + for k := range bitmap1 { + bitset := bitmap1[k] &^ bitmap2[k] + for bitset != 0 { + t := bitset & -bitset + container[pos] = uint16((k*64 + int(popcount(t-1)))) + pos = pos + 1 + bitset ^= t + } + } +} + +func fillArrayXOR(container []uint16, bitmap1, bitmap2 []uint64) { + if len(bitmap1) != len(bitmap2) { + panic("array lengths don't match") + } + // TODO: rewrite in assembly + pos := 0 + for k := 0; k < len(bitmap1); k++ { + bitset := bitmap1[k] ^ bitmap2[k] + for bitset != 0 { + t := bitset & -bitset + container[pos] = uint16((k*64 + int(popcount(t-1)))) + pos = pos + 1 + bitset ^= t + } + } +} + +func highbits(x uint32) uint16 { + return uint16(x >> 16) +} +func lowbits(x uint32) uint16 { + return uint16(x & 0xFFFF) +} + +func maxLowBit() uint16 { + return uint16(0xFFFF) +} + +func toIntUnsigned(x uint16) uint32 { + return uint32(x) +} + +func flipBitmapRange(bitmap []uint64, start int, end int) { + if start >= end { + return + } + firstword := start / 64 + endword := (end - 1) / 64 + bitmap[firstword] ^= ^(^uint64(0) << uint(start%64)) + for i := firstword; i < endword; i++ { + bitmap[i] = ^bitmap[i] + } + bitmap[endword] ^= ^uint64(0) >> (uint(-end) % 64) +} + +func resetBitmapRange(bitmap []uint64, start int, end int) { + if start >= end { + return + } + firstword := start / 64 + endword := (end - 1) / 64 + if firstword == endword { + bitmap[firstword] &= ^((^uint64(0) << uint(start%64)) & (^uint64(0) >> (uint(-end) % 64))) + return + } + bitmap[firstword] &= ^(^uint64(0) << uint(start%64)) + for i := firstword + 1; i < endword; i++ { + bitmap[i] = 0 + } + bitmap[endword] &= ^(^uint64(0) >> (uint(-end) % 64)) + +} + +func setBitmapRange(bitmap []uint64, start int, end int) { + if start >= end { + return + } + firstword := start / 64 + endword := (end - 1) / 64 + if firstword == endword { + bitmap[firstword] |= (^uint64(0) << uint(start%64)) & (^uint64(0) >> (uint(-end) % 64)) + return + } + bitmap[firstword] |= ^uint64(0) << uint(start%64) + for i := firstword + 1; i < endword; i++ { + bitmap[i] = ^uint64(0) + } + bitmap[endword] |= ^uint64(0) >> (uint(-end) % 64) +} + +func flipBitmapRangeAndCardinalityChange(bitmap []uint64, start int, end int) int { + before := wordCardinalityForBitmapRange(bitmap, start, end) + flipBitmapRange(bitmap, start, end) + after := wordCardinalityForBitmapRange(bitmap, start, end) + return int(after - before) +} + +func resetBitmapRangeAndCardinalityChange(bitmap []uint64, start int, end int) int { + before := wordCardinalityForBitmapRange(bitmap, start, end) + resetBitmapRange(bitmap, start, end) + after := wordCardinalityForBitmapRange(bitmap, start, end) + return int(after - before) +} + +func setBitmapRangeAndCardinalityChange(bitmap []uint64, start int, end int) int { + before := wordCardinalityForBitmapRange(bitmap, start, end) + setBitmapRange(bitmap, start, end) + after := wordCardinalityForBitmapRange(bitmap, start, end) + return int(after - before) +} + +func wordCardinalityForBitmapRange(bitmap []uint64, start int, end int) uint64 { + answer := uint64(0) + if start >= end { + return answer + } + firstword := start / 64 + endword := (end - 1) / 64 + for i := firstword; i <= endword; i++ { + answer += popcount(bitmap[i]) + } + return answer +} + +func selectBitPosition(w uint64, j int) int { + seen := 0 + + // Divide 64bit + part := w & 0xFFFFFFFF + n := popcount(part) + if n <= uint64(j) { + part = w >> 32 + seen += 32 + j -= int(n) + } + w = part + + // Divide 32bit + part = w & 0xFFFF + n = popcount(part) + if n <= uint64(j) { + part = w >> 16 + seen += 16 + j -= int(n) + } + w = part + + // Divide 16bit + part = w & 0xFF + n = popcount(part) + if n <= uint64(j) { + part = w >> 8 + seen += 8 + j -= int(n) + } + w = part + + // Lookup in final byte + var counter uint + for counter = 0; counter < 8; counter++ { + j -= int((w >> counter) & 1) + if j < 0 { + break + } + } + return seen + int(counter) + +} diff --git a/vendor/github.com/mschoch/smat/.gitignore b/vendor/github.com/mschoch/smat/.gitignore new file mode 100644 index 00000000..eee88075 --- /dev/null +++ b/vendor/github.com/mschoch/smat/.gitignore @@ -0,0 +1,14 @@ +#* +*.sublime-* +*~ +.#* +.project +.settings +**/.idea/ +**/*.iml +/examples/bolt/boltsmat-fuzz.zip +/examples/bolt/workdir/ +.DS_Store +coverage.out +*.test +tags diff --git a/vendor/github.com/mschoch/smat/.travis.yml b/vendor/github.com/mschoch/smat/.travis.yml new file mode 100644 index 00000000..3c9c3463 --- /dev/null +++ b/vendor/github.com/mschoch/smat/.travis.yml @@ -0,0 +1,16 @@ +sudo: false +language: go +go: +- 1.6 +script: +- go get golang.org/x/tools/cmd/cover +- go get github.com/mattn/goveralls +- go get github.com/kisielk/errcheck +- go test -v -race +- go vet +- errcheck ./... +- go test -coverprofile=profile.out -covermode=count +- goveralls -service=travis-ci -coverprofile=profile.out -repotoken $COVERALLS +notifications: + email: + - marty.schoch@gmail.com diff --git a/vendor/github.com/mschoch/smat/LICENSE b/vendor/github.com/mschoch/smat/LICENSE new file mode 100644 index 00000000..7a4a3ea2 --- /dev/null +++ b/vendor/github.com/mschoch/smat/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/vendor/github.com/mschoch/smat/README.md b/vendor/github.com/mschoch/smat/README.md new file mode 100644 index 00000000..f5ca1c54 --- /dev/null +++ b/vendor/github.com/mschoch/smat/README.md @@ -0,0 +1,166 @@ +# smat – State Machine Assisted Testing + +The concept is simple, describe valid uses of your library as states and actions. States describe which actions are possible, and with what probability they should occur. Actions mutate the context and transition to another state. + +By doing this, two things are possible: + +1. Use [go-fuzz](https://github.com/dvyukov/go-fuzz) to find/test interesting sequences of operations on your library. + +2. Automate longevity testing of your application by performing long sequences of valid operations. + +**NOTE**: both of these can also incorporate validation logic (not just failure detection by building validation into the state machine) + +## Status + +The API is still not stable. This is brand new and we'll probably change things we don't like... + +[![Build Status](https://travis-ci.org/mschoch/smat.svg?branch=master)](https://travis-ci.org/mschoch/smat) +[![Coverage Status](https://coveralls.io/repos/github/mschoch/smat/badge.svg?branch=master)](https://coveralls.io/github/mschoch/smat?branch=master) +[![GoDoc](https://godoc.org/github.com/mschoch/smat?status.svg)](https://godoc.org/github.com/mschoch/smat) +[![codebeat badge](https://codebeat.co/badges/c3ff6180-a241-4128-97f0-fa6bf6f48752)](https://codebeat.co/projects/github-com-mschoch-smat) +[![Go Report Card](https://goreportcard.com/badge/github.com/mschoch/smat)](https://goreportcard.com/report/github.com/mschoch/smat) + +## License + +Apache 2.0 + +## How do I use it? + +### smat.Context + +Choose a structure to keep track of any state. You pass in an instance of this when you start, and it will be passed to every action when it executes. The actions may mutate this context. + +For example, consider a database library, once you open a database handle, you need to use it inside of the other actions. So you might use a structure like: + +``` +type context struct { + db *DB +} +``` + +### smat.State + +A state represents a state that your application/library can be in, and the probabilities thats certain actions should be taken. + +For example, consider a database library, in a state where the database is open, there many things you can do. Let's consider just two right now, you can set a value, or you can delete a value. + +``` +func dbOpen(next byte) smat.ActionID { + return smat.PercentExecute(next, + smat.PercentAction{50, setValue}, + smat.PercentAction{50, deleteValue}, + ) +} +``` + +This says that in the open state, there are two valid actions, 50% of the time you should set a value and 50% of the time you should delete a value. **NOTE**: these percentages are just for characterizing the test workload. + +### smat.Action + +Actions are functions that do some work, optionally mutate the context, and indicate the next state to transition to. Below we see an example action to set value in a database. + +``` +func setValueFunc(ctx smat.Context) (next smat.State, err error) { + // type assert to our custom context type + context := ctx.(*context) + // perform the operation + err = context.db.Set("k", "v") + if err != nil { + return nil, err + } + // return the new state + return dbOpen, nil +} +``` + +### smat.ActionID and smat.ActionMap + +Actions are just functions, and since we can't compare functions in Go, we need to introduce an external identifier for them. This allows us to build a bi-directional mapping which we'll take advantage of later. + +``` +const ( + setup smat.ActionID = iota + teardown + setValue + deleteValue +) + +var actionMap = smat.ActionMap{ + setup: setupFunc, + teardown: teardownFunc, + setValue: setValueFunc, + deleteValue: deleteValueFunc, +} +``` + +### smat.ActionSeq + +A common way that many users think about a library is as a sequence of actions to be performed. Using the ActionID's that we've already seen we can build up sequences of operations. + +``` + actionSeq := smat.ActionSeq{ + open, + setValue, + setValue, + setValue, + } +``` + +Notice that we build these actions using the constants we defined above, and because of this we can have a bi-directional mapping between a stream of bytes (driving the state machine) and a sequence of actions to be performed. + +## Fuzzing + +We've built a lot of pieces, lets wire it up to go-fuzz. + +``` +func Fuzz(data []byte) int { + return smat.Fuzz(&context{}, setup, teardown, actionMap, data) +} +``` + +* The first argument is an instance of context structure. +* The second argument is the ActionID of our setup function. The setup function does not consume any of the input stream and is used to initialize the context and determine the start state. +* The third argument is the teardown function. This will be called unconditionally to clean up any resources associated with the test. +* The fourth argument is the actionMap which maps all ActionIDs to Actions. +* The fifth argument is the data passed in from the go-fuzz application. + +### Generating Initial go-fuzz Corpus + +Earlier we mentioned the bi-directional mapping between Actions and the byte stream driving the state machine. We can now leverage this to build the inital go-fuzz corpus. + +Using the `ActinSeq`s we learned about earlier we can build up a list of them as: + + var actionSeqs = []smat.ActionSeq{...} + +Then, we can write them out to disk using: + +``` +for i, actionSeq := range actionSeqs { + byteSequence, err := actionSeq.ByteEncoding(&context{}, setup, teardown, actionMap) + if err != nil { + // handle error + } + os.MkdirAll("workdir/corpus", 0700) + ioutil.WriteFile(fmt.Sprintf("workdir/corpus/%d", i), byteSequence, 0600) +} +``` + +You can then either put this into a test case or a main application depending on your needs. + +## Longevity Testing + +Fuzzing is great, but most of your corpus is likely to be shorter meaningful sequences. And go-fuzz works to find shortest sequences that cause problems, but sometimes you actually want to explore longer sequences that appear to go-fuzz as not triggering additional code coverage. + +For these cases we have another helper you can use: + +``` + Longevity(ctx, setup, teardown, actionMap, 0, closeChan) +``` + +The first four arguments are the same, the last two are: +* random seed used to ensure repeatable tests +* closeChan (chan struct{}) - close this channel if you want the function to stop and return ErrClosed, otherwise it will run forever + +## Examples + +See the examples directory for a working example that tests some BoltDB functionality. diff --git a/vendor/github.com/mschoch/smat/actionseq.go b/vendor/github.com/mschoch/smat/actionseq.go new file mode 100644 index 00000000..6c8297f8 --- /dev/null +++ b/vendor/github.com/mschoch/smat/actionseq.go @@ -0,0 +1,61 @@ +// Copyright (c) 2016 Marty Schoch + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the +// License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an "AS +// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language +// governing permissions and limitations under the License. + +package smat + +// ActionSeq represents a sequence of actions, used for populating a corpus +// of byte sequences for the corresponding fuzz tests +type ActionSeq []ActionID + +// ByteEncoding runs the FSM to produce a byte sequence to trigger the +// desired action +func (a ActionSeq) ByteEncoding(ctx Context, setup, teardown ActionID, actionMap ActionMap) ([]byte, error) { + setupFunc, teardownFunc, err := actionMap.findSetupTeardown(setup, teardown) + if err != nil { + return nil, err + } + state, err := setupFunc(ctx) + if err != nil { + return nil, err + } + defer func() { + _, _ = teardownFunc(ctx) + }() + + var rv []byte + for _, actionID := range a { + b, err := probeStateForAction(state, actionID) + if err != nil { + return nil, err + } + rv = append(rv, b) + action, ok := actionMap[actionID] + if !ok { + continue + } + state, err = action(ctx) + if err != nil { + return nil, err + } + } + return rv, nil +} + +func probeStateForAction(state State, actionID ActionID) (byte, error) { + for i := 0; i < 256; i++ { + nextActionID := state(byte(i)) + if nextActionID == actionID { + return byte(i), nil + } + } + return 0, ErrActionNotPossible +} diff --git a/vendor/github.com/mschoch/smat/smat.go b/vendor/github.com/mschoch/smat/smat.go new file mode 100644 index 00000000..f6ea4975 --- /dev/null +++ b/vendor/github.com/mschoch/smat/smat.go @@ -0,0 +1,161 @@ +// Copyright (c) 2016 Marty Schoch + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the +// License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an "AS +// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language +// governing permissions and limitations under the License. + +package smat + +import ( + "bufio" + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + "math/rand" +) + +// Logger is a configurable logger used by this package +// by default output is discarded +var Logger = log.New(ioutil.Discard, "smat ", log.LstdFlags) + +// Context is a container for any user state +type Context interface{} + +// State is a function which describes which action to perform in the event +// that a particular byte is seen +type State func(next byte) ActionID + +// PercentAction describes the frequency with which an action should occur +// for example: Action{Percent:10, Action:DonateMoney} means that 10% of +// the time you should donate money. +type PercentAction struct { + Percent int + Action ActionID +} + +// Action is any function which returns the next state to transition to +// it can optionally mutate the provided context object +// if any error occurs, it may return an error which will abort execution +type Action func(Context) (State, error) + +// ActionID is a unique identifier for an action +type ActionID int + +// NopAction does nothing and simply continues to the next input +var NopAction ActionID = -1 + +// ActionMap is a mapping form ActionID to Action +type ActionMap map[ActionID]Action + +func (a ActionMap) findSetupTeardown(setup, teardown ActionID) (Action, Action, error) { + setupFunc, ok := a[setup] + if !ok { + return nil, nil, ErrSetupMissing + } + teardownFunc, ok := a[teardown] + if !ok { + return nil, nil, ErrTeardownMissing + } + return setupFunc, teardownFunc, nil +} + +// Fuzz runs the fuzzing state machine with the provided context +// first, the setup action is executed unconditionally +// the start state is determined by this action +// actionMap is a lookup table for all actions +// the data byte slice determines all future state transitions +// finally, the teardown action is executed unconditionally for cleanup +func Fuzz(ctx Context, setup, teardown ActionID, actionMap ActionMap, data []byte) int { + reader := bytes.NewReader(data) + err := runReader(ctx, setup, teardown, actionMap, reader, nil) + if err != nil { + panic(err) + } + return 1 +} + +// Longevity runs the state machine with the provided context +// first, the setup action is executed unconditionally +// the start state is determined by this action +// actionMap is a lookup table for all actions +// random bytes are generated to determine all future state transitions +// finally, the teardown action is executed unconditionally for cleanup +func Longevity(ctx Context, setup, teardown ActionID, actionMap ActionMap, seed int64, closeChan chan struct{}) error { + source := rand.NewSource(seed) + return runReader(ctx, setup, teardown, actionMap, rand.New(source), closeChan) +} + +var ( + // ErrSetupMissing is returned when the setup action cannot be found + ErrSetupMissing = fmt.Errorf("setup action missing") + // ErrTeardownMissing is returned when the teardown action cannot be found + ErrTeardownMissing = fmt.Errorf("teardown action missing") + // ErrClosed is returned when the closeChan was closed to cancel the op + ErrClosed = fmt.Errorf("closed") + // ErrActionNotPossible is returned when an action is encountered in a + // FuzzCase that is not possible in the current state + ErrActionNotPossible = fmt.Errorf("action not possible in state") +) + +func runReader(ctx Context, setup, teardown ActionID, actionMap ActionMap, r io.Reader, closeChan chan struct{}) error { + setupFunc, teardownFunc, err := actionMap.findSetupTeardown(setup, teardown) + if err != nil { + return err + } + Logger.Printf("invoking setup action") + state, err := setupFunc(ctx) + if err != nil { + return err + } + defer func() { + Logger.Printf("invoking teardown action") + _, _ = teardownFunc(ctx) + }() + + reader := bufio.NewReader(r) + for next, err := reader.ReadByte(); err == nil; next, err = reader.ReadByte() { + select { + case <-closeChan: + return ErrClosed + default: + actionID := state(next) + action, ok := actionMap[actionID] + if !ok { + Logger.Printf("no such action defined, continuing") + continue + } + Logger.Printf("invoking action - %d", actionID) + state, err = action(ctx) + if err != nil { + Logger.Printf("it was action %d that returned err %v", actionID, err) + return err + } + } + } + return err +} + +// PercentExecute interprets the next byte as a random value and normalizes it +// to values 0-99, it then looks to see which action should be execued based +// on the action distributions +func PercentExecute(next byte, pas ...PercentAction) ActionID { + percent := int(99 * int(next) / 255) + + sofar := 0 + for _, pa := range pas { + sofar = sofar + pa.Percent + if percent < sofar { + return pa.Action + } + + } + return NopAction +} diff --git a/vendor/github.com/willf/bitset/.gitignore b/vendor/github.com/willf/bitset/.gitignore new file mode 100644 index 00000000..5c204d28 --- /dev/null +++ b/vendor/github.com/willf/bitset/.gitignore @@ -0,0 +1,26 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof + +target diff --git a/vendor/github.com/willf/bitset/.travis.yml b/vendor/github.com/willf/bitset/.travis.yml new file mode 100644 index 00000000..11a99a2b --- /dev/null +++ b/vendor/github.com/willf/bitset/.travis.yml @@ -0,0 +1,16 @@ +language: go + +sudo: false + +branches: + except: + - release + +branches: + only: + - master + - develop + +go: + - 1.5 + - tip diff --git a/vendor/github.com/willf/bitset/LICENSE b/vendor/github.com/willf/bitset/LICENSE new file mode 100644 index 00000000..59cab8a9 --- /dev/null +++ b/vendor/github.com/willf/bitset/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2014 Will Fitzgerald. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/willf/bitset/README.md b/vendor/github.com/willf/bitset/README.md new file mode 100644 index 00000000..2fe69a1a --- /dev/null +++ b/vendor/github.com/willf/bitset/README.md @@ -0,0 +1,65 @@ +# bitset + +*Go language library to map between non-negative integers and boolean values* + +[![Master Branch](https://img.shields.io/badge/-master:-gray.svg)](https://github.com/willf/bitset/tree/master) +[![Master Build Status](https://travis-ci.org/willf/bitset.svg?branch=master)](https://travis-ci.org/willf/bitset) +[![Develop Branch](https://img.shields.io/badge/-develop:-gray.svg)](https://github.com/willf/bitset/tree/develop) +[![Develop Build Status](https://secure.travis-ci.org/willf/bitset.svg?branch=develop)](https://travis-ci.org/willf/bitset?branch=develop) + + + +## Description + +Package bitset implements bitsets, a mapping between non-negative integers and boolean values. +It should be more efficient than map[uint] bool. + +It provides methods for setting, clearing, flipping, and testing individual integers. + +But it also provides set intersection, union, difference, complement, and symmetric operations, as well as tests to check whether any, all, or no bits are set, and querying a bitset's current length and number of postive bits. + +BitSets are expanded to the size of the largest set bit; the memory allocation is approximately Max bits, where Max is the largest set bit. BitSets are never shrunk. On creation, a hint can be given for the number of bits that will be used. + +Many of the methods, including Set, Clear, and Flip, return a BitSet pointer, which allows for chaining. + +### Example use: + + import "bitset" + var b BitSet + b.Set(10).Set(11) + if b.Test(1000) { + b.Clear(1000) + } + for i,e := v.NextSet(0); e; i,e = v.NextSet(i + 1) { + frmt.Println("The following bit is set:",i); + } + if B.Intersection(bitset.New(100).Set(10)).Count() > 1 { + fmt.Println("Intersection works.") + } + +As an alternative to BitSets, one should check out the 'big' package, which provides a (less set-theoretical) view of bitsets. + +Discussions golang-nuts Google Group: + +* [Revised BitSet](https://groups.google.com/forum/#!topic/golang-nuts/5i3l0CXDiBg) +* [simple bitset?](https://groups.google.com/d/topic/golang-nuts/7n1VkRTlBf4/discussion) + +Godoc documentation is at: https://godoc.org/github.com/willf/bitset + + +## Getting started + +This application is written in the go language, please refer to the guides in https://golang.org for getting started. + +This project include a Makefile that allows you to test and build the project with simple commands. +To see all available options: +```bash +make help +``` + +## Running all tests + +Before committing the code, please check if it passes all tests using +```bash +make qa +``` diff --git a/vendor/github.com/willf/bitset/RELEASE b/vendor/github.com/willf/bitset/RELEASE new file mode 100644 index 00000000..56a6051c --- /dev/null +++ b/vendor/github.com/willf/bitset/RELEASE @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/vendor/github.com/willf/bitset/VERSION b/vendor/github.com/willf/bitset/VERSION new file mode 100644 index 00000000..524cb552 --- /dev/null +++ b/vendor/github.com/willf/bitset/VERSION @@ -0,0 +1 @@ +1.1.1 diff --git a/vendor/github.com/willf/bitset/bitset.go b/vendor/github.com/willf/bitset/bitset.go new file mode 100644 index 00000000..07b4842d --- /dev/null +++ b/vendor/github.com/willf/bitset/bitset.go @@ -0,0 +1,694 @@ +/* +Package bitset implements bitsets, a mapping +between non-negative integers and boolean values. It should be more +efficient than map[uint] bool. + +It provides methods for setting, clearing, flipping, and testing +individual integers. + +But it also provides set intersection, union, difference, +complement, and symmetric operations, as well as tests to +check whether any, all, or no bits are set, and querying a +bitset's current length and number of postive bits. + +BitSets are expanded to the size of the largest set bit; the +memory allocation is approximately Max bits, where Max is +the largest set bit. BitSets are never shrunk. On creation, +a hint can be given for the number of bits that will be used. + +Many of the methods, including Set,Clear, and Flip, return +a BitSet pointer, which allows for chaining. + +Example use: + + import "bitset" + var b BitSet + b.Set(10).Set(11) + if b.Test(1000) { + b.Clear(1000) + } + if B.Intersection(bitset.New(100).Set(10)).Count() > 1 { + fmt.Println("Intersection works.") + } + +As an alternative to BitSets, one should check out the 'big' package, +which provides a (less set-theoretical) view of bitsets. + +*/ +package bitset + +import ( + "bufio" + "bytes" + "encoding/base64" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "strconv" +) + +// the wordSize of a bit set +const wordSize = uint(64) + +// log2WordSize is lg(wordSize) +const log2WordSize = uint(6) + +// A BitSet is a set of bits. The zero value of a BitSet is an empty set of length 0. +type BitSet struct { + length uint + set []uint64 +} + +// Error is used to distinguish errors (panics) generated in this package. +type Error string + +// safeSet will fixup b.set to be non-nil and return the field value +func (b *BitSet) safeSet() []uint64 { + if b.set == nil { + b.set = make([]uint64, wordsNeeded(0)) + } + return b.set +} + +// From is a constructor used to create a BitSet from an array of integers +func From(buf []uint64) *BitSet { + return &BitSet{uint(len(buf)) * 64, buf} +} + +// Bytes returns the bitset as array of integers +func (b *BitSet) Bytes() []uint64 { + return b.set +} + +// wordsNeeded calculates the number of words needed for i bits +func wordsNeeded(i uint) int { + if i > ((^uint(0)) - wordSize + 1) { + return int((^uint(0)) >> log2WordSize) + } + return int((i + (wordSize - 1)) >> log2WordSize) +} + +// New creates a new BitSet with a hint that length bits will be required +func New(length uint) (bset *BitSet) { + defer func() { + if r := recover(); r != nil { + bset = &BitSet{ + 0, + make([]uint64, 0), + } + } + }() + + bset = &BitSet{ + length, + make([]uint64, wordsNeeded(length)), + } + + return bset +} + +// Cap returns the total possible capicity, or number of bits +func Cap() uint { + return ^uint(0) +} + +// Len returns the length of the BitSet in words +func (b *BitSet) Len() uint { + return b.length +} + +// extendSetMaybe adds additional words to incorporate new bits if needed +func (b *BitSet) extendSetMaybe(i uint) { + if i >= b.length { // if we need more bits, make 'em + nsize := wordsNeeded(i + 1) + if b.set == nil { + b.set = make([]uint64, nsize) + } else if cap(b.set) >= nsize { + b.set = b.set[:nsize] // fast resize + } else if len(b.set) < nsize { + newset := make([]uint64, nsize, 2*nsize) // increase capacity 2x + copy(newset, b.set) + b.set = newset + } + b.length = i + 1 + } +} + +// Test whether bit i is set. +func (b *BitSet) Test(i uint) bool { + if i >= b.length { + return false + } + return b.set[i>>log2WordSize]&(1<<(i&(wordSize-1))) != 0 +} + +// Set bit i to 1 +func (b *BitSet) Set(i uint) *BitSet { + b.extendSetMaybe(i) + b.set[i>>log2WordSize] |= 1 << (i & (wordSize - 1)) + return b +} + +// Clear bit i to 0 +func (b *BitSet) Clear(i uint) *BitSet { + if i >= b.length { + return b + } + b.set[i>>log2WordSize] &^= 1 << (i & (wordSize - 1)) + return b +} + +// SetTo sets bit i to value +func (b *BitSet) SetTo(i uint, value bool) *BitSet { + if value { + return b.Set(i) + } + return b.Clear(i) +} + +// Flip bit at i +func (b *BitSet) Flip(i uint) *BitSet { + if i >= b.length { + return b.Set(i) + } + b.set[i>>log2WordSize] ^= 1 << (i & (wordSize - 1)) + return b +} + +// String creates a string representation of the Bitmap +func (b *BitSet) String() string { + // follows code from https://github.com/RoaringBitmap/roaring + var buffer bytes.Buffer + start := []byte("{") + buffer.Write(start) + counter := 0 + i, e := b.NextSet(0) + for e { + counter = counter + 1 + // to avoid exhausting the memory + if counter > 0x40000 { + buffer.WriteString("...") + break + } + buffer.WriteString(strconv.FormatInt(int64(i), 10)) + i, e = b.NextSet(i + 1) + if e { + buffer.WriteString(",") + } + } + buffer.WriteString("}") + return buffer.String() +} + +// NextSet returns the next bit set from the specified index, +// including possibly the current index +// along with an error code (true = valid, false = no set bit found) +// for i,e := v.NextSet(0); e; i,e = v.NextSet(i + 1) {...} +func (b *BitSet) NextSet(i uint) (uint, bool) { + x := int(i >> log2WordSize) + if x >= len(b.set) { + return 0, false + } + w := b.set[x] + w = w >> (i & (wordSize - 1)) + if w != 0 { + return i + trailingZeroes64(w), true + } + x = x + 1 + for x < len(b.set) { + if b.set[x] != 0 { + return uint(x)*wordSize + trailingZeroes64(b.set[x]), true + } + x = x + 1 + + } + return 0, false +} + +// ClearAll clears the entire BitSet +func (b *BitSet) ClearAll() *BitSet { + if b != nil && b.set != nil { + for i := range b.set { + b.set[i] = 0 + } + } + return b +} + +// wordCount returns the number of words used in a bit set +func (b *BitSet) wordCount() int { + return wordsNeeded(b.length) +} + +// Clone this BitSet +func (b *BitSet) Clone() *BitSet { + c := New(b.length) + if b.set != nil { // Clone should not modify current object + copy(c.set, b.set) + } + return c +} + +// Copy into a destination BitSet +// Returning the size of the destination BitSet +// like array copy +func (b *BitSet) Copy(c *BitSet) (count uint) { + if c == nil { + return + } + if b.set != nil { // Copy should not modify current object + copy(c.set, b.set) + } + count = c.length + if b.length < c.length { + count = b.length + } + return +} + +// Count (number of set bits) +func (b *BitSet) Count() uint { + if b != nil && b.set != nil { + return uint(popcntSlice(b.set)) + } + return 0 +} + +var deBruijn = [...]byte{ + 0, 1, 56, 2, 57, 49, 28, 3, 61, 58, 42, 50, 38, 29, 17, 4, + 62, 47, 59, 36, 45, 43, 51, 22, 53, 39, 33, 30, 24, 18, 12, 5, + 63, 55, 48, 27, 60, 41, 37, 16, 46, 35, 44, 21, 52, 32, 23, 11, + 54, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, +} + +func trailingZeroes64(v uint64) uint { + return uint(deBruijn[((v&-v)*0x03f79d71b4ca8b09)>>58]) +} + +// Equal tests the equvalence of two BitSets. +// False if they are of different sizes, otherwise true +// only if all the same bits are set +func (b *BitSet) Equal(c *BitSet) bool { + if c == nil { + return false + } + if b.length != c.length { + return false + } + if b.length == 0 { // if they have both length == 0, then could have nil set + return true + } + // testing for equality shoud not transform the bitset (no call to safeSet) + + for p, v := range b.set { + if c.set[p] != v { + return false + } + } + return true +} + +func panicIfNull(b *BitSet) { + if b == nil { + panic(Error("BitSet must not be null")) + } +} + +// Difference of base set and other set +// This is the BitSet equivalent of &^ (and not) +func (b *BitSet) Difference(compare *BitSet) (result *BitSet) { + panicIfNull(b) + panicIfNull(compare) + result = b.Clone() // clone b (in case b is bigger than compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + for i := 0; i < l; i++ { + result.set[i] = b.set[i] &^ compare.set[i] + } + return +} + +// DifferenceCardinality computes the cardinality of the differnce +func (b *BitSet) DifferenceCardinality(compare *BitSet) uint { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + cnt := uint64(0) + cnt += popcntMaskSlice(b.set[:l], compare.set[:l]) + cnt += popcntSlice(b.set[l:]) + return uint(cnt) +} + +// InPlaceDifference computes the difference of base set and other set +// This is the BitSet equivalent of &^ (and not) +func (b *BitSet) InPlaceDifference(compare *BitSet) { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + for i := 0; i < l; i++ { + b.set[i] &^= compare.set[i] + } +} + +// Convenience function: return two bitsets ordered by +// increasing length. Note: neither can be nil +func sortByLength(a *BitSet, b *BitSet) (ap *BitSet, bp *BitSet) { + if a.length <= b.length { + ap, bp = a, b + } else { + ap, bp = b, a + } + return +} + +// Intersection of base set and other set +// This is the BitSet equivalent of & (and) +func (b *BitSet) Intersection(compare *BitSet) (result *BitSet) { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + result = New(b.length) + for i, word := range b.set { + result.set[i] = word & compare.set[i] + } + return +} + +// IntersectionCardinality computes the cardinality of the union +func (b *BitSet) IntersectionCardinality(compare *BitSet) uint { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + cnt := popcntAndSlice(b.set, compare.set) + return uint(cnt) +} + +// InPlaceIntersection destructively computes the intersection of +// base set and the compare set. +// This is the BitSet equivalent of & (and) +func (b *BitSet) InPlaceIntersection(compare *BitSet) { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + for i := 0; i < l; i++ { + b.set[i] &= compare.set[i] + } + for i := l; i < len(b.set); i++ { + b.set[i] = 0 + } + if compare.length > 0 { + b.extendSetMaybe(compare.length - 1) + } + return +} + +// Union of base set and other set +// This is the BitSet equivalent of | (or) +func (b *BitSet) Union(compare *BitSet) (result *BitSet) { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + result = compare.Clone() + for i, word := range b.set { + result.set[i] = word | compare.set[i] + } + return +} + +// UnionCardinality computes the cardinality of the uniton of the base set +// and the compare set. +func (b *BitSet) UnionCardinality(compare *BitSet) uint { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + cnt := popcntOrSlice(b.set, compare.set) + if len(compare.set) > len(b.set) { + cnt += popcntSlice(compare.set[len(b.set):]) + } + return uint(cnt) +} + +// InPlaceUnion creates the destructive union of base set and compare set. +// This is the BitSet equivalent of | (or). +func (b *BitSet) InPlaceUnion(compare *BitSet) { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + if compare.length > 0 { + b.extendSetMaybe(compare.length - 1) + } + for i := 0; i < l; i++ { + b.set[i] |= compare.set[i] + } + if len(compare.set) > l { + for i := l; i < len(compare.set); i++ { + b.set[i] = compare.set[i] + } + } +} + +// SymmetricDifference of base set and other set +// This is the BitSet equivalent of ^ (xor) +func (b *BitSet) SymmetricDifference(compare *BitSet) (result *BitSet) { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + // compare is bigger, so clone it + result = compare.Clone() + for i, word := range b.set { + result.set[i] = word ^ compare.set[i] + } + return +} + +// SymmetricDifferenceCardinality computes the cardinality of the symmetric difference +func (b *BitSet) SymmetricDifferenceCardinality(compare *BitSet) uint { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + cnt := popcntXorSlice(b.set, compare.set) + if len(compare.set) > len(b.set) { + cnt += popcntSlice(compare.set[len(b.set):]) + } + return uint(cnt) +} + +// InPlaceSymmetricDifference creates the destructive SymmetricDifference of base set and other set +// This is the BitSet equivalent of ^ (xor) +func (b *BitSet) InPlaceSymmetricDifference(compare *BitSet) { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + if compare.length > 0 { + b.extendSetMaybe(compare.length - 1) + } + for i := 0; i < l; i++ { + b.set[i] ^= compare.set[i] + } + if len(compare.set) > l { + for i := l; i < len(compare.set); i++ { + b.set[i] = compare.set[i] + } + } +} + +// Is the length an exact multiple of word sizes? +func (b *BitSet) isEven() bool { + return b.length%wordSize == 0 +} + +// Clean last word by setting unused bits to 0 +func (b *BitSet) cleanLastWord() { + if !b.isEven() { + // Mask for cleaning last word + const allBits uint64 = 0xffffffffffffffff + b.set[wordsNeeded(b.length)-1] &= allBits >> (wordSize - b.length%wordSize) + } +} + +// Complement computes the (local) complement of a biset (up to length bits) +func (b *BitSet) Complement() (result *BitSet) { + panicIfNull(b) + result = New(b.length) + for i, word := range b.set { + result.set[i] = ^word + } + result.cleanLastWord() + return +} + +// All returns true if all bits are set, false otherwise. Returns true for +// empty sets. +func (b *BitSet) All() bool { + panicIfNull(b) + return b.Count() == b.length +} + +// None returns true if no bit is set, false otherwise. Retursn true for +// empty sets. +func (b *BitSet) None() bool { + panicIfNull(b) + if b != nil && b.set != nil { + for _, word := range b.set { + if word > 0 { + return false + } + } + return true + } + return true +} + +// Any returns true if any bit is set, false otherwise +func (b *BitSet) Any() bool { + panicIfNull(b) + return !b.None() +} + +// IsSuperSet returns true if this is a superset of the other set +func (b *BitSet) IsSuperSet(other *BitSet) bool { + for i, e := other.NextSet(0); e; i, e = other.NextSet(i + 1) { + if !b.Test(i) { + return false + } + } + return true +} + +// IsStrictSuperSet returns true if this is a strict superset of the other set +func (b *BitSet) IsStrictSuperSet(other *BitSet) bool { + return b.Count() > other.Count() && b.IsSuperSet(other) +} + +// DumpAsBits dumps a bit set as a string of bits +func (b *BitSet) DumpAsBits() string { + if b.set == nil { + return "." + } + buffer := bytes.NewBufferString("") + i := len(b.set) - 1 + for ; i >= 0; i-- { + fmt.Fprintf(buffer, "%064b.", b.set[i]) + } + return string(buffer.Bytes()) +} + +// BinaryStorageSize returns the binary storage requirements +func (b *BitSet) BinaryStorageSize() int { + return binary.Size(uint64(0)) + binary.Size(b.set) +} + +// WriteTo writes a BitSet to a stream +func (b *BitSet) WriteTo(stream io.Writer) (int64, error) { + length := uint64(b.length) + + // Write length + err := binary.Write(stream, binary.BigEndian, length) + if err != nil { + return 0, err + } + + // Write set + err = binary.Write(stream, binary.BigEndian, b.set) + return int64(b.BinaryStorageSize()), err +} + +// ReadFrom reads a BitSet from a stream written using WriteTo +func (b *BitSet) ReadFrom(stream io.Reader) (int64, error) { + var length uint64 + + // Read length first + err := binary.Read(stream, binary.BigEndian, &length) + if err != nil { + return 0, err + } + newset := New(uint(length)) + + if uint64(newset.length) != length { + return 0, errors.New("Unmarshalling error: type mismatch") + } + + // Read remaining bytes as set + err = binary.Read(stream, binary.BigEndian, newset.set) + if err != nil { + return 0, err + } + + *b = *newset + return int64(b.BinaryStorageSize()), nil +} + +// MarshalBinary encodes a BitSet into a binary form and returns the result. +func (b *BitSet) MarshalBinary() ([]byte, error) { + var buf bytes.Buffer + writer := bufio.NewWriter(&buf) + + _, err := b.WriteTo(writer) + if err != nil { + return []byte{}, err + } + + err = writer.Flush() + + return buf.Bytes(), err +} + +// UnmarshalBinary decodes the binary form generated by MarshalBinary. +func (b *BitSet) UnmarshalBinary(data []byte) error { + buf := bytes.NewReader(data) + reader := bufio.NewReader(buf) + + _, err := b.ReadFrom(reader) + + return err +} + +// MarshalJSON marshals a BitSet as a JSON structure +func (b *BitSet) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBuffer(make([]byte, 0, b.BinaryStorageSize())) + _, err := b.WriteTo(buffer) + if err != nil { + return nil, err + } + + // URLEncode all bytes + return json.Marshal(base64.URLEncoding.EncodeToString(buffer.Bytes())) +} + +// UnmarshalJSON unmarshals a BitSet from JSON created using MarshalJSON +func (b *BitSet) UnmarshalJSON(data []byte) error { + // Unmarshal as string + var s string + err := json.Unmarshal(data, &s) + if err != nil { + return err + } + + // URLDecode string + buf, err := base64.URLEncoding.DecodeString(s) + if err != nil { + return err + } + + _, err = b.ReadFrom(bytes.NewReader(buf)) + return err +} diff --git a/vendor/github.com/willf/bitset/popcnt.go b/vendor/github.com/willf/bitset/popcnt.go new file mode 100644 index 00000000..76577a83 --- /dev/null +++ b/vendor/github.com/willf/bitset/popcnt.go @@ -0,0 +1,53 @@ +package bitset + +// bit population count, take from +// https://code.google.com/p/go/issues/detail?id=4988#c11 +// credit: https://code.google.com/u/arnehormann/ +func popcount(x uint64) (n uint64) { + x -= (x >> 1) & 0x5555555555555555 + x = (x>>2)&0x3333333333333333 + x&0x3333333333333333 + x += x >> 4 + x &= 0x0f0f0f0f0f0f0f0f + x *= 0x0101010101010101 + return x >> 56 +} + +func popcntSliceGo(s []uint64) uint64 { + cnt := uint64(0) + for _, x := range s { + cnt += popcount(x) + } + return cnt +} + +func popcntMaskSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] &^ m[i]) + } + return cnt +} + +func popcntAndSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] & m[i]) + } + return cnt +} + +func popcntOrSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] | m[i]) + } + return cnt +} + +func popcntXorSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] ^ m[i]) + } + return cnt +} diff --git a/vendor/github.com/willf/bitset/popcnt_amd64.go b/vendor/github.com/willf/bitset/popcnt_amd64.go new file mode 100644 index 00000000..665a8644 --- /dev/null +++ b/vendor/github.com/willf/bitset/popcnt_amd64.go @@ -0,0 +1,67 @@ +// +build amd64,!appengine + +package bitset + +// *** the following functions are defined in popcnt_amd64.s + +//go:noescape + +func hasAsm() bool + +// useAsm is a flag used to select the GO or ASM implementation of the popcnt function +var useAsm = hasAsm() + +//go:noescape + +func popcntSliceAsm(s []uint64) uint64 + +//go:noescape + +func popcntMaskSliceAsm(s, m []uint64) uint64 + +//go:noescape + +func popcntAndSliceAsm(s, m []uint64) uint64 + +//go:noescape + +func popcntOrSliceAsm(s, m []uint64) uint64 + +//go:noescape + +func popcntXorSliceAsm(s, m []uint64) uint64 + +func popcntSlice(s []uint64) uint64 { + if useAsm { + return popcntSliceAsm(s) + } + return popcntSliceGo(s) +} + +func popcntMaskSlice(s, m []uint64) uint64 { + if useAsm { + return popcntMaskSliceAsm(s, m) + } + return popcntMaskSliceGo(s, m) +} + +func popcntAndSlice(s, m []uint64) uint64 { + if useAsm { + return popcntAndSliceAsm(s, m) + } + return popcntAndSliceGo(s, m) +} + +func popcntOrSlice(s, m []uint64) uint64 { + if useAsm { + return popcntOrSliceAsm(s, m) + } + return popcntOrSliceGo(s, m) +} + +func popcntXorSlice(s, m []uint64) uint64 { + if useAsm { + return popcntXorSliceAsm(s, m) + } + return popcntXorSliceGo(s, m) +} diff --git a/vendor/github.com/willf/bitset/popcnt_amd64.s b/vendor/github.com/willf/bitset/popcnt_amd64.s new file mode 100644 index 00000000..18f58784 --- /dev/null +++ b/vendor/github.com/willf/bitset/popcnt_amd64.s @@ -0,0 +1,103 @@ +// +build amd64,!appengine + +TEXT ·hasAsm(SB),4,$0-1 +MOVQ $1, AX +CPUID +SHRQ $23, CX +ANDQ $1, CX +MOVB CX, ret+0(FP) +RET + +#define POPCNTQ_DX_DX BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0xd2 + +TEXT ·popcntSliceAsm(SB),4,$0-32 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntSliceEnd +popcntSliceLoop: +BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0x16 // POPCNTQ (SI), DX +ADDQ DX, AX +ADDQ $8, SI +LOOP popcntSliceLoop +popcntSliceEnd: +MOVQ AX, ret+24(FP) +RET + +TEXT ·popcntMaskSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntMaskSliceEnd +MOVQ m+24(FP), DI +popcntMaskSliceLoop: +MOVQ (DI), DX +NOTQ DX +ANDQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntMaskSliceLoop +popcntMaskSliceEnd: +MOVQ AX, ret+48(FP) +RET + +TEXT ·popcntAndSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntAndSliceEnd +MOVQ m+24(FP), DI +popcntAndSliceLoop: +MOVQ (DI), DX +ANDQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntAndSliceLoop +popcntAndSliceEnd: +MOVQ AX, ret+48(FP) +RET + +TEXT ·popcntOrSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntOrSliceEnd +MOVQ m+24(FP), DI +popcntOrSliceLoop: +MOVQ (DI), DX +ORQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntOrSliceLoop +popcntOrSliceEnd: +MOVQ AX, ret+48(FP) +RET + +TEXT ·popcntXorSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntXorSliceEnd +MOVQ m+24(FP), DI +popcntXorSliceLoop: +MOVQ (DI), DX +XORQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntXorSliceLoop +popcntXorSliceEnd: +MOVQ AX, ret+48(FP) +RET diff --git a/vendor/github.com/willf/bitset/popcnt_generic.go b/vendor/github.com/willf/bitset/popcnt_generic.go new file mode 100644 index 00000000..6b21cb7a --- /dev/null +++ b/vendor/github.com/willf/bitset/popcnt_generic.go @@ -0,0 +1,23 @@ +// +build !amd64 appengine + +package bitset + +func popcntSlice(s []uint64) uint64 { + return popcntSliceGo(s) +} + +func popcntMaskSlice(s, m []uint64) uint64 { + return popcntMaskSliceGo(s, m) +} + +func popcntAndSlice(s, m []uint64) uint64 { + return popcntAndSliceGo(s, m) +} + +func popcntOrSlice(s, m []uint64) uint64 { + return popcntOrSliceGo(s, m) +} + +func popcntXorSlice(s, m []uint64) uint64 { + return popcntXorSliceGo(s, m) +} diff --git a/vendor/golang.org/x/sys/unix/syscall_linux.go b/vendor/golang.org/x/sys/unix/syscall_linux.go index 6d10c9cf..d3dc1ddd 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux.go @@ -361,6 +361,16 @@ func (sa *SockaddrUnix) sockaddr() (unsafe.Pointer, _Socklen, error) { return unsafe.Pointer(&sa.raw), sl, nil } +func (sa *SockaddrVsock) sockaddr() (unsafe.Pointer, _Socklen, error) { + if sa.Port > 0xFFFF { + return nil, 0, EINVAL + } + sa.raw.Family = AF_VSOCK + sa.raw.Port = sa.Port + sa.raw.Cid = sa.Cid + return unsafe.Pointer(&sa.raw), SizeofSockaddrAny, nil +} + type SockaddrLinklayer struct { Protocol uint16 Ifindex int @@ -485,6 +495,12 @@ func anyToSockaddr(rsa *RawSockaddrAny) (Sockaddr, error) { sa.Addr[i] = pp.Addr[i] } return sa, nil + case AF_VSOCK: + pp := (*SockaddrVsock)(unsafe.Pointer(rsa)) + sa := new(SockaddrVsock) + sa.Cid = pp.Cid + sa.Port = pp.Port + return sa, nil } return nil, EAFNOSUPPORT } diff --git a/vendor/golang.org/x/sys/unix/syscall_unix.go b/vendor/golang.org/x/sys/unix/syscall_unix.go index b46b2502..15ccd72c 100644 --- a/vendor/golang.org/x/sys/unix/syscall_unix.go +++ b/vendor/golang.org/x/sys/unix/syscall_unix.go @@ -164,6 +164,12 @@ type SockaddrUnix struct { raw RawSockaddrUnix } +type SockaddrVsock struct { + Cid uint32 + Port uint32 + raw RawSockaddrVsock +} + func Bind(fd int, sa Sockaddr) (err error) { ptr, n, err := sa.sockaddr() if err != nil { diff --git a/vendor/golang.org/x/sys/unix/types_darwin.go b/vendor/golang.org/x/sys/unix/types_darwin.go index 11532618..b5fbb611 100644 --- a/vendor/golang.org/x/sys/unix/types_darwin.go +++ b/vendor/golang.org/x/sys/unix/types_darwin.go @@ -131,6 +131,8 @@ type RawSockaddrInet6 C.struct_sockaddr_in6 type RawSockaddrUnix C.struct_sockaddr_un +type RawSockaddrVsock C.struct_sockaddr_vm + type RawSockaddrDatalink C.struct_sockaddr_dl type RawSockaddr C.struct_sockaddr diff --git a/vendor/golang.org/x/sys/unix/types_linux.go b/vendor/golang.org/x/sys/unix/types_linux.go index 7dea79a8..97d352be 100644 --- a/vendor/golang.org/x/sys/unix/types_linux.go +++ b/vendor/golang.org/x/sys/unix/types_linux.go @@ -51,6 +51,7 @@ package unix #include #include #include +#include #include #include #include @@ -209,6 +210,8 @@ type RawSockaddrInet6 C.struct_sockaddr_in6 type RawSockaddrUnix C.struct_my_sockaddr_un +type RawSockaddrVsock C.struct_sockaddr_vm + type RawSockaddrLinklayer C.struct_sockaddr_ll type RawSockaddrNetlink C.struct_sockaddr_nl @@ -252,6 +255,7 @@ const ( SizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 SizeofSockaddrAny = C.sizeof_struct_sockaddr_any SizeofSockaddrUnix = C.sizeof_struct_sockaddr_un + SizeofSockaddrVsock = C.sizeof_struct_sockaddr_vm SizeofSockaddrLinklayer = C.sizeof_struct_sockaddr_ll SizeofSockaddrNetlink = C.sizeof_struct_sockaddr_nl SizeofSockaddrHCI = C.sizeof_struct_sockaddr_hci diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_386.go b/vendor/golang.org/x/sys/unix/zerrors_linux_386.go index 8f920124..dbe0bf4a 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_386.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_386.go @@ -51,6 +51,7 @@ const ( AF_TIPC = 0x1e AF_UNIX = 0x1 AF_UNSPEC = 0x0 + AF_VSOCK = 0x28 AF_WANPIPE = 0x19 AF_X25 = 0x9 ARPHRD_ADAPT = 0x108 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go index 49b6c354..92948764 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go @@ -51,6 +51,7 @@ const ( AF_TIPC = 0x1e AF_UNIX = 0x1 AF_UNSPEC = 0x0 + AF_VSOCK = 0x28 AF_WANPIPE = 0x19 AF_X25 = 0x9 ARPHRD_ADAPT = 0x108 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go b/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go index f036758f..e9456617 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go @@ -50,6 +50,7 @@ const ( AF_TIPC = 0x1e AF_UNIX = 0x1 AF_UNSPEC = 0x0 + AF_VSOCK = 0x28 AF_WANPIPE = 0x19 AF_X25 = 0x9 ARPHRD_ADAPT = 0x108 diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_386.go b/vendor/golang.org/x/sys/unix/ztypes_linux_386.go index f3ddf534..454dbd67 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_386.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_386.go @@ -180,6 +180,14 @@ type RawSockaddrUnix struct { Path [108]int8 } +type RawSockaddrVsock struct { + Family uint16 + Reserved uint16 + Port uint32 + Cid uint32 + Zero [4]uint8 +} + type RawSockaddrLinklayer struct { Family uint16 Protocol uint16 diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_amd64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_amd64.go index a923bef3..0859bc07 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_amd64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_amd64.go @@ -182,6 +182,14 @@ type RawSockaddrUnix struct { Path [108]int8 } +type RawSockaddrVsock struct { + Family uint16 + Reserved uint16 + Port uint32 + Cid uint32 + Zero [4]uint8 +} + type RawSockaddrLinklayer struct { Family uint16 Protocol uint16 diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_arm.go b/vendor/golang.org/x/sys/unix/ztypes_linux_arm.go index 817ac9c2..f0f4a3e3 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_arm.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_arm.go @@ -175,6 +175,14 @@ type RawSockaddrUnix struct { Path [108]int8 } +type RawSockaddrVsock struct { + Family uint16 + Reserved uint16 + Port uint32 + Cid uint32 + Zero [4]uint8 +} + type RawSockaddrLinklayer struct { Family uint16 Protocol uint16 diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_arm64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_arm64.go index e786addf..06e1ec76 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_arm64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_arm64.go @@ -183,6 +183,14 @@ type RawSockaddrUnix struct { Path [108]int8 } +type RawSockaddrVsock struct { + Family uint16 + Reserved uint16 + Port uint32 + Cid uint32 + Zero [4]uint8 +} + type RawSockaddrLinklayer struct { Family uint16 Protocol uint16 diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_mips64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_mips64.go index b29894de..3b7a420e 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_mips64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_mips64.go @@ -183,6 +183,14 @@ type RawSockaddrUnix struct { Path [108]int8 } +type RawSockaddrVsock struct { + Family uint16 + Reserved uint16 + Port uint32 + Cid uint32 + Zero [4]uint8 +} + type RawSockaddrLinklayer struct { Family uint16 Protocol uint16 diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_mips64le.go b/vendor/golang.org/x/sys/unix/ztypes_linux_mips64le.go index d9af71b6..ec37cd0d 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_mips64le.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_mips64le.go @@ -183,6 +183,14 @@ type RawSockaddrUnix struct { Path [108]int8 } +type RawSockaddrVsock struct { + Family uint16 + Reserved uint16 + Port uint32 + Cid uint32 + Zero [4]uint8 +} + type RawSockaddrLinklayer struct { Family uint16 Protocol uint16 diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64.go index 4218170a..92504b27 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64.go @@ -184,6 +184,14 @@ type RawSockaddrUnix struct { Path [108]int8 } +type RawSockaddrVsock struct { + Family uint16 + Reserved uint16 + Port uint32 + Cid uint32 + Zero [4]uint8 +} + type RawSockaddrLinklayer struct { Family uint16 Protocol uint16 diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64le.go b/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64le.go index 7db4c78c..eebfac5e 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64le.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64le.go @@ -184,6 +184,14 @@ type RawSockaddrUnix struct { Path [108]int8 } +type RawSockaddrVsock struct { + Family uint16 + Reserved uint16 + Port uint32 + Cid uint32 + Zero [4]uint8 +} + type RawSockaddrLinklayer struct { Family uint16 Protocol uint16 diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_s390x.go b/vendor/golang.org/x/sys/unix/ztypes_linux_s390x.go index 76ee57cb..203ba75c 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_s390x.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_s390x.go @@ -183,6 +183,14 @@ type RawSockaddrUnix struct { Path [108]int8 } +type RawSockaddrVsock struct { + Family uint16 + Reserved uint16 + Port uint32 + Cid uint32 + Zero [4]uint8 +} + type RawSockaddrLinklayer struct { Family uint16 Protocol uint16