73% test coverage

go/arrow/bitutil/bitmaps.go

@@ -17,6 +17,7 @@
 package bitutil
 
 import (
+	"bytes"
 	"math/bits"
 	"unsafe"
 
@@ -449,25 +450,34 @@ func alignedBitmapOp(op bitOp, left, right []byte, lOffset, rOffset int64, out [
 	left = left[lOffset/8:]
 	right = right[rOffset/8:]
 	out = out[outOffset/8:]
+	endMask := (lOffset + length%8)
 	switch nbytes {
 	case 0:
 		return
 	case 1: // everything within a single byte
 		// (length+lOffset%8) <= 8
-		mask := PrecedingBitmask[lOffset%8] | TrailingBitmask[(lOffset+length)%8]
+		mask := PrecedingBitmask[lOffset%8]
+		if endMask != 0 {
+			mask |= TrailingBitmask[endMask]
+		}
 		out[0] = (out[0] & mask) | (op.opByte(left[0], right[0]) &^ mask)
 	case 2: // don't send zero length to opAligned
 		firstByteMask := PrecedingBitmask[lOffset%8]
 		out[0] = (out[0] & firstByteMask) | (op.opByte(left[0], right[0]) &^ firstByteMask)
-		lastByteMask := TrailingBitmask[(lOffset+length)%8]
+		lastByteMask := byte(0)
+		if endMask != 0 {
+			lastByteMask = TrailingBitmask[endMask]
+		}
 		out[1] = (out[1] & lastByteMask) | (op.opByte(left[1], right[1]) &^ lastByteMask)
 	default:
 		firstByteMask := PrecedingBitmask[lOffset%8]
 		out[0] = (out[0] & firstByteMask) | (op.opByte(left[0], right[0]) &^ firstByteMask)
 
 		op.opAligned(left[1:nbytes-1], right[1:nbytes-1], out[1:nbytes-1])
-
-		lastByteMask := TrailingBitmask[(lOffset+length)%8]
+		lastByteMask := byte(0)
+		if endMask != 0 {
+			lastByteMask = TrailingBitmask[endMask]
+		}
 		out[nbytes-1] = (out[nbytes-1] & lastByteMask) | (op.opByte(left[nbytes-1], right[nbytes-1]) &^ lastByteMask)
 	}
 }
@@ -520,3 +530,45 @@ func BitmapAndAlloc(mem memory.Allocator, left, right []byte, lOffset, rOffset i
 func BitmapOrAlloc(mem memory.Allocator, left, right []byte, lOffset, rOffset int64, length, outOffset int64) *memory.Buffer {
 	return BitmapOpAlloc(mem, bitOrOp, left, right, lOffset, rOffset, length, outOffset)
 }
+
+func BitmapEquals(left, right []byte, lOffset, rOffset int64, length int64) bool {
+	if lOffset%8 == 0 && rOffset%8 == 0 {
+		// byte aligned, fast path, can use bytes.Equal (memcmp)
+		byteLen := length / 8
+		lStart := lOffset / 8
+		rStart := rOffset / 8
+		if !bytes.Equal(left[lStart:lStart+byteLen], right[rStart:rStart+byteLen]) {
+			return false
+		}
+
+		// check trailing bits
+		for i := (length / 8) * 8; i < length; i++ {
+			if BitIsSet(left, int(lOffset+i)) != BitIsSet(right, int(rOffset+i)) {
+				return false
+			}
+		}
+		return true
+	}
+
+	lrdr := NewBitmapWordReader(left, int(lOffset), int(length))
+	rrdr := NewBitmapWordReader(right, int(rOffset), int(length))
+
+	nwords := lrdr.Words()
+	for nwords > 0 {
+		nwords--
+		if lrdr.NextWord() != rrdr.NextWord() {
+			return false
+		}
+	}
+
+	nbytes := lrdr.TrailingBytes()
+	for nbytes > 0 {
+		nbytes--
+		lbt, _ := lrdr.NextTrailingByte()
+		rbt, _ := rrdr.NextTrailingByte()
+		if lbt != rbt {
+			return false
+		}
+	}
+	return true
+}

go/arrow/compute/datum.go

@@ -28,27 +28,6 @@ import (
 //go:generate go run golang.org/x/tools/cmd/stringer -type=ValueShape -linecomment
 //go:generate go run golang.org/x/tools/cmd/stringer -type=DatumKind -linecomment
 
-// ValueShape is a brief description of the shape of a value (array, scalar or otherwise)
-type ValueShape int8
-
-const (
-	// either Array or Scalar
-	ShapeAny    ValueShape = iota // any
-	ShapeArray                    // array
-	ShapeScalar                   // scalar
-)
-
-// ValueDescr is a descriptor type giving both the shape and the datatype of a value
-// but without the data.
-type ValueDescr struct {
-	Shape ValueShape
-	Type  arrow.DataType
-}
-
-func (v *ValueDescr) String() string {
-	return fmt.Sprintf("%s [%s]", v.Shape, v.Type)
-}
-
 // DatumKind is an enum used for denoting which kind of type a datum is encapsulating
 type DatumKind int
 
@@ -90,8 +69,6 @@ type Datum interface {
 // a slice with 1 element for Array, and the slice of chunks for a chunked array.
 type ArrayLikeDatum interface {
 	Datum
-	Shape() ValueShape
-	Descr() ValueDescr
 	NullN() int64
 	Type() arrow.DataType
 	Chunks() []arrow.Array
@@ -122,12 +99,10 @@ type ScalarDatum struct {
 }
 
 func (ScalarDatum) Kind() DatumKind         { return KindScalar }
-func (ScalarDatum) Shape() ValueShape       { return ShapeScalar }
 func (ScalarDatum) Len() int64              { return 1 }
 func (ScalarDatum) Chunks() []arrow.Array   { return nil }
 func (d *ScalarDatum) Type() arrow.DataType { return d.Value.DataType() }
 func (d *ScalarDatum) String() string       { return d.Value.String() }
-func (d *ScalarDatum) Descr() ValueDescr    { return ValueDescr{ShapeScalar, d.Value.DataType()} }
 func (d *ScalarDatum) ToScalar() (scalar.Scalar, error) {
 	return d.Value, nil
 }
@@ -163,11 +138,9 @@ type ArrayDatum struct {
 }
 
 func (ArrayDatum) Kind() DatumKind           { return KindArray }
-func (ArrayDatum) Shape() ValueShape         { return ShapeArray }
 func (d *ArrayDatum) Type() arrow.DataType   { return d.Value.DataType() }
 func (d *ArrayDatum) Len() int64             { return int64(d.Value.Len()) }
 func (d *ArrayDatum) NullN() int64           { return int64(d.Value.NullN()) }
-func (d *ArrayDatum) Descr() ValueDescr      { return ValueDescr{ShapeArray, d.Value.DataType()} }
 func (d *ArrayDatum) String() string         { return fmt.Sprintf("Array:{%s}", d.Value.DataType()) }
 func (d *ArrayDatum) MakeArray() arrow.Array { return array.MakeFromData(d.Value) }
 func (d *ArrayDatum) Chunks() []arrow.Array  { return []arrow.Array{d.MakeArray()} }
@@ -199,11 +172,9 @@ type ChunkedDatum struct {
 }
 
 func (ChunkedDatum) Kind() DatumKind          { return KindChunked }
-func (ChunkedDatum) Shape() ValueShape        { return ShapeArray }
 func (d *ChunkedDatum) Type() arrow.DataType  { return d.Value.DataType() }
 func (d *ChunkedDatum) Len() int64            { return int64(d.Value.Len()) }
 func (d *ChunkedDatum) NullN() int64          { return int64(d.Value.NullN()) }
-func (d *ChunkedDatum) Descr() ValueDescr     { return ValueDescr{ShapeArray, d.Value.DataType()} }
 func (d *ChunkedDatum) String() string        { return fmt.Sprintf("Array:{%s}", d.Value.DataType()) }
 func (d *ChunkedDatum) Chunks() []arrow.Array { return d.Value.Chunks() }