atg. Fix incorrect Not() algo, add tests for Not(). use conversion in…

…stead of toIntUnsigned.

rle.go

@@ -1149,6 +1149,9 @@ func (rc *runContainer32) invertLastInterval(origin uint32, lastIdx int) []inter
 	}
 }
 
+// invert returns a new container (not inplace), that is
+// the inversion of rc. For each bit b in rc, the
+// returned value has !b
 func (rc *runContainer32) invert() *runContainer32 {
 	ni := len(rc.iv)
 	var m []interval32
@@ -1394,6 +1397,7 @@ func (rc *runContainer32) isubtract(del interval32) {
 	}
 }
 
+// compute rc minus b, and return the result as a new value (not inplace).
 // port of run_container_andnot from CRoaring...
 // https://github.com/RoaringBitmap/CRoaring/blob/master/src/containers/run.c#L435-L496
 func (rc *runContainer32) AndNotRunContainer32(b *runContainer32) *runContainer32 {

rle16.go

@@ -1149,6 +1149,9 @@ func (rc *runContainer16) invertLastInterval(origin uint16, lastIdx int) []inter
 	}
 }
 
+// invert returns a new container (not inplace), that is
+// the inversion of rc. For each bit b in rc, the
+// returned value has !b
 func (rc *runContainer16) invert() *runContainer16 {
 	ni := len(rc.iv)
 	var m []interval16
@@ -1394,6 +1397,7 @@ func (rc *runContainer16) isubtract(del interval16) {
 	}
 }
 
+// compute rc minus b, and return the result as a new value (not inplace).
 // port of run_container_andnot from CRoaring...
 // https://github.com/RoaringBitmap/CRoaring/blob/master/src/containers/run.c#L435-L496
 func (rc *runContainer16) AndNotRunContainer16(b *runContainer16) *runContainer16 {

rlei.go

@@ -64,6 +64,8 @@ func (rc *runContainer16) iand(a container) container {
 
 func (rc *runContainer16) inplaceIntersect(rc2 *runContainer16) container {
 	// TODO: optimize by doing less allocation, possibly?
+
+	// sect will be new
 	sect := rc.intersect(rc2)
 	*rc = *sect
 	return rc
@@ -155,20 +157,34 @@ func (rc *runContainer16) not(firstOfRange, lastOfRange int) container {
 	return rc.Not(firstOfRange, lastOfRange)
 }
 
-// Not flips the values in the range [firstOfRange,lastOfRange)
-func (rc *runContainer16) Not(firstOfRange, lastOfRange int) *runContainer16 {
+// Not flips the values in the range [firstOfRange,endx).
+// This is not inplace. Only the returned value has the flipped bits.
+func (rc *runContainer16) Not(firstOfRange, endx int) *runContainer16 {
 
-	if firstOfRange >= lastOfRange {
+	//p("top of Not with interval [%v, %v): rc is %s", firstOfRange, endx, rc.String())
+	if firstOfRange >= endx {
+		//p("returning early with clone, first >= endx")
 		return rc.Clone()
 	}
-	x := interval16{start: uint16(firstOfRange), last: uint16(lastOfRange - 1)}
-	xs := []interval16{x}
 
-	isect := rc.intersect(newRunContainer16TakeOwnership(xs))
-	rc2 := rc.Clone()
-	rc2.isubtract(x)
-	invertedIsect := isect.invert()
-	rc2 = rc2.union(invertedIsect)
+	a := rc
+	// algo:
+	// (!A intersect B) union (A minus B)
+
+	nota := a.invert()
+
+	bs := []interval16{interval16{start: uint16(firstOfRange), last: uint16(endx - 1)}}
+	b := newRunContainer16TakeOwnership(bs)
+	//p("b is %s", b)
+
+	notAintersectB := nota.intersect(b)
+	//p("notAintersectB is %s", notAintersectB)
+
+	aMinusB := a.AndNotRunContainer16(b)
+	//p("aMinusB is %s", aMinusB)
+
+	rc2 := notAintersectB.union(aMinusB)
+	//p("rc = ((!A intersect B) union (A minus B)) is %s", rc2)
 	return rc2
 }
 

rlei_test.go

@@ -118,6 +118,11 @@ func TestRle16RandomUnionAgainstOtherContainers011(t *testing.T) {
 
 		trials := []trial{
 			trial{n: 100, percentFill: .95, ntrial: 1},
+			/*			trial{n: 100, percentFill: .01, ntrial: 10},
+						trial{n: 100, percentFill: .99, ntrial: 10},
+						trial{n: 100, percentFill: .50, ntrial: 10},
+						trial{n: 10, percentFill: 1.0, ntrial: 10},
+			*/
 		}
 
 		tester := func(tr trial) {
@@ -928,3 +933,96 @@ func TestRle16Rank020(t *testing.T) {
 		}
 	}
 }
+
+func TestRle16NotAlsoKnownAsFlipRange021(t *testing.T) {
+
+	Convey("runContainer `Not` operation should flip the bits of a range on the new returned container", t, func() {
+		seed := int64(42)
+		p("seed is %v", seed)
+		rand.Seed(seed)
+
+		trials := []trial{
+			trial{n: 100, percentFill: .8, ntrial: 2},
+			/*			trial{n: 10, percentFill: .01, ntrial: 10},
+						trial{n: 10, percentFill: .50, ntrial: 10},
+						trial{n: 1000, percentFill: .50, ntrial: 10},
+						trial{n: 1000, percentFill: .99, ntrial: 10},
+			*/
+		}
+
+		tester := func(tr trial) {
+			for j := 0; j < tr.ntrial; j++ {
+				p("TestRle16NotAlsoKnownAsFlipRange021 on check# j=%v", j)
+
+				// what is the interval we are going to flip?
+
+				ma := make(map[int]bool)
+				flipped := make(map[int]bool)
+
+				n := tr.n
+				a := []uint16{}
+
+				draw := int(float64(n) * tr.percentFill)
+				p("draw is %v", draw)
+				for i := 0; i < draw; i++ {
+					r0 := rand.Intn(n)
+					a = append(a, uint16(r0))
+					ma[r0] = true
+					flipped[r0] = true
+					p("draw r0=%v is being added to a and ma", r0)
+				}
+
+				// pick an interval to flip
+				begin := rand.Intn(n)
+				last := rand.Intn(n)
+				if last < begin {
+					begin, last = last, begin
+				}
+				p("our interval to flip is [%v, %v]", begin, last)
+
+				// do the flip on the hash `flipped`
+				for i := begin; i <= last; i++ {
+					if flipped[i] {
+						delete(flipped, i)
+					} else {
+						flipped[i] = true
+					}
+				}
+
+				//showArray16(a, "a")
+				// can be too big to print:
+				//showHash("hash (correct) version of flipped is:", flipped)
+
+				// RunContainer's Not
+				rc := newRunContainer16FromVals(false, a...)
+				flp := rc.Not(begin, last+1)
+
+				//p("rc from a is %v", rc)
+				//p("rc.cardinality = %v", rc.cardinality())
+
+				//p("flp of a (has card=%v) is %v. card of our flipped hash is %v", flp.cardinality(), flp, len(flipped))
+
+				So(flp.cardinality(), ShouldEqual, len(flipped))
+
+				for k := 0; k < n; k++ {
+					if flipped[k] {
+						//p("flipped has %v, checking flp", k)
+						So(flp.contains(uint16(k)), ShouldBeTrue)
+					} else {
+						//p("flipped lacks %v, checking flp", k)
+						So(flp.contains(uint16(k)), ShouldBeFalse)
+					}
+				}
+
+				//p("checking for cardinality agreement: flp is %v, len(flipped) is %v", flp.getCardinality(), len(flipped))
+				So(flp.getCardinality(), ShouldEqual, len(flipped))
+			}
+			//p("done with randomized Not() check for trial %#v", tr)
+		}
+
+		for i := range trials {
+			tester(trials[i])
+		}
+
+	})
+}

roaring.go

@@ -97,7 +97,7 @@ func (rb *Bitmap) ToArray() []uint32 {
 	pos2 := 0
 
 	for pos < rb.highlowcontainer.size() {
-		hs := toIntUnsigned(rb.highlowcontainer.getKeyAtIndex(pos)) << 16
+		hs := uint32(rb.highlowcontainer.getKeyAtIndex(pos)) << 16
 		c := rb.highlowcontainer.getContainerAtIndex(pos)
 		pos++
 		c.fillLeastSignificant16bits(array, pos2, hs)
@@ -166,13 +166,13 @@ func (ii *intIterator) HasNext() bool {
 func (ii *intIterator) init() {
 	if ii.highlowcontainer.size() > ii.pos {
 		ii.iter = ii.highlowcontainer.getContainerAtIndex(ii.pos).getShortIterator()
-		ii.hs = toIntUnsigned(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16
+		ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16
 	}
 }
 
 // Next returns the next integer
 func (ii *intIterator) Next() uint32 {
-	x := toIntUnsigned(ii.iter.next()) | ii.hs
+	x := uint32(ii.iter.next()) | ii.hs
 	if !ii.iter.hasNext() {
 		ii.pos = ii.pos + 1
 		ii.init()
@@ -908,6 +908,13 @@ func BitmapOf(dat ...uint32) *Bitmap {
 // while uint64(0x100000000) cannot be represented as a 32-bit value.
 func (rb *Bitmap) Flip(rangeStart, rangeEnd uint64) {
 
+	if rangeEnd > MaxUint32+1 {
+		panic("rangeEnd > MaxUint32+1")
+	}
+	if rangeStart > MaxUint32+1 {
+		panic("rangeStart > MaxUint32+1")
+	}
+
 	if rangeStart >= rangeEnd {
 		return
 	}
@@ -917,28 +924,32 @@ func (rb *Bitmap) Flip(rangeStart, rangeEnd uint64) {
 	hbLast := highbits(uint32(rangeEnd - 1))
 	lbLast := lowbits(uint32(rangeEnd - 1))
 
-	max := toIntUnsigned(maxLowBit())
+	var max uint32 = maxLowBit
 	for hb := hbStart; hb <= hbLast; hb++ {
-		containerStart := uint32(0)
+		var containerStart uint32
 		if hb == hbStart {
-			containerStart = toIntUnsigned(lbStart)
+			containerStart = uint32(lbStart)
 		}
 		containerLast := max
 		if hb == hbLast {
-			containerLast = toIntUnsigned(lbLast)
+			containerLast = uint32(lbLast)
 		}
 
 		i := rb.highlowcontainer.getIndex(hb)
 
 		if i >= 0 {
+			//fmt.Printf("\n\n i = %v track \n", i)
+
 			c := rb.highlowcontainer.getWritableContainerAtIndex(i).inot(int(containerStart), int(containerLast)+1)
+			//fmt.Printf("\n\n c = %v \n", c)
 			if c.getCardinality() > 0 {
 				rb.highlowcontainer.setContainerAtIndex(i, c)
 			} else {
 				rb.highlowcontainer.removeAtIndex(i)
 			}
 		} else { // *think* the range of ones must never be
 			// empty.
+			//fmt.Printf("\n\n empty track\n")
 			rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, rangeOfOnes(int(containerStart), int(containerLast)))
 		}
 	}
@@ -957,12 +968,12 @@ func (rb *Bitmap) AddRange(rangeStart, rangeEnd uint64) {
 		return
 	}
 
-	hbStart := toIntUnsigned(highbits(uint32(rangeStart)))
-	lbStart := toIntUnsigned(lowbits(uint32(rangeStart)))
-	hbLast := toIntUnsigned(highbits(uint32(rangeEnd - 1)))
-	lbLast := toIntUnsigned(lowbits(uint32(rangeEnd - 1)))
+	hbStart := uint32(highbits(uint32(rangeStart)))
+	lbStart := uint32(lowbits(uint32(rangeStart)))
+	hbLast := uint32(highbits(uint32(rangeEnd - 1)))
+	lbLast := uint32(lowbits(uint32(rangeEnd - 1)))
 
-	max := toIntUnsigned(maxLowBit())
+	var max uint32 = maxLowBit
 	for hb := uint16(hbStart); hb <= uint16(hbLast); hb++ {
 		containerStart := uint32(0)
 		if hb == uint16(hbStart) {
@@ -993,12 +1004,12 @@ func (rb *Bitmap) RemoveRange(rangeStart, rangeEnd uint64) {
 		return
 	}
 
-	hbStart := toIntUnsigned(highbits(uint32(rangeStart)))
-	lbStart := toIntUnsigned(lowbits(uint32(rangeStart)))
-	hbLast := toIntUnsigned(highbits(uint32(rangeEnd - 1)))
-	lbLast := toIntUnsigned(lowbits(uint32(rangeEnd - 1)))
+	hbStart := uint32(highbits(uint32(rangeStart)))
+	lbStart := uint32(lowbits(uint32(rangeStart)))
+	hbLast := uint32(highbits(uint32(rangeEnd - 1)))
+	lbLast := uint32(lowbits(uint32(rangeEnd - 1)))
 
-	max := toIntUnsigned(maxLowBit())
+	var max uint32 = maxLowBit
 
 	if hbStart == hbLast {
 		i := rb.highlowcontainer.getIndex(uint16(hbStart))
@@ -1054,6 +1065,13 @@ func Flip(bm *Bitmap, rangeStart, rangeEnd uint64) *Bitmap {
 		return bm.Clone()
 	}
 
+	if rangeStart > MaxUint32 {
+		panic("rangeStart > MaxUint32")
+	}
+	if rangeEnd-1 > MaxUint32 {
+		panic("rangeEnd-1 > MaxUint32")
+	}
+
 	answer := NewBitmap()
 	hbStart := highbits(uint32(rangeStart))
 	lbStart := lowbits(uint32(rangeStart))
@@ -1063,15 +1081,15 @@ func Flip(bm *Bitmap, rangeStart, rangeEnd uint64) *Bitmap {
 	// copy the containers before the active area
 	answer.highlowcontainer.appendCopiesUntil(bm.highlowcontainer, hbStart)
 
-	max := toIntUnsigned(maxLowBit())
+	var max uint32 = maxLowBit
 	for hb := hbStart; hb <= hbLast; hb++ {
-		containerStart := uint32(0)
+		var containerStart uint32
 		if hb == hbStart {
-			containerStart = toIntUnsigned(lbStart)
+			containerStart = uint32(lbStart)
 		}
 		containerLast := max
 		if hb == hbLast {
-			containerLast = toIntUnsigned(lbLast)
+			containerLast = uint32(lbLast)
 		}
 
 		i := bm.highlowcontainer.getIndex(hb)

roaring_test.go

@@ -1734,20 +1734,12 @@ func TestAndNot(t *testing.T) {
 	})
 }
 
-/*
-func TestMinimizedDoubleAdd3(t *testing.T) {
-	Convey("minimized doubleadd3 ", t, func() {
+func TestFlipVerySmall(t *testing.T) {
+	Convey("very small basic Flip test", t, func() {
 		rb := NewBitmap()
-		var k uint64 = 3
-		rb.AddRange(65533, 65536*k+10)
-		rb.AddRange(65530, 65536*k+10)
-		rb2 := NewBitmap()
-		rb2.AddRange(65530, 65536*k+10)
-		So(rb.GetCardinality(), ShouldEqual, 65536*(k-1)+10+(65536-65530))
-		So(rb2.GetCardinality(), ShouldEqual, 65536*(k-1)+10+(65536-65530))
-		So(rb.Equals(rb2), ShouldEqual, true)
-		rb2.RemoveRange(65530, 65536*k+1)
-		So(rb2.GetCardinality(), ShouldEqual, 9)
+		rb.Flip(0, 10) // got [0,9], card is 10
+		rb.Flip(0, 1)  // give back the number 0, card goes to 9
+		rbcard := rb.GetCardinality()
+		So(rbcard, ShouldEqual, 9)
 	})
 }
-*/