decode,interp: Add arbitrary large integer support (BigInt)

Was already handled in fq in various places as gojq uses them

Update msgpack to support negative integers that can't represented as int64
Rename read try* number functions to make them more explicit

format/cbor/cbor.go

@@ -9,10 +9,12 @@ package cbor
 import (
 	"bytes"
 	"embed"
+	"math/big"
 	"strings"
 
 	"github.com/wader/fq/format"
 	"github.com/wader/fq/format/registry"
+	"github.com/wader/fq/internal/num"
 	"github.com/wader/fq/pkg/bitio"
 	"github.com/wader/fq/pkg/decode"
 	"github.com/wader/fq/pkg/scalar"
@@ -109,11 +111,13 @@ func decodeCBORValue(d *decode.D) interface{} {
 	majorTypeMap := majorTypeEntries{
 		majorTypePositiveInt: {s: scalar.S{Sym: "positive_int"}, d: func(d *decode.D, shortCount uint64, count uint64) interface{} {
 			d.FieldValueU("value", count)
-			return count
+			return nil
 		}},
 		majorTypeNegativeInt: {s: scalar.S{Sym: "negative_int"}, d: func(d *decode.D, shortCount uint64, count uint64) interface{} {
-			d.FieldValueS("value", int64(^count))
-			return count
+			n := new(big.Int)
+			n.SetUint64(count).Neg(n).Sub(n, num.BigIntOne)
+			d.FieldValueBigInt("value", n)
+			return nil
 		}},
 		majorTypeBytes: {s: scalar.S{Sym: "bytes"}, d: func(d *decode.D, shortCount uint64, count uint64) interface{} {
 			if shortCount == shortCountIndefinite {
@@ -204,7 +208,7 @@ func decodeCBORValue(d *decode.D) interface{} {
 		majorTypeSematic: {s: scalar.S{Sym: "semantic"}, d: func(d *decode.D, shortCount uint64, count uint64) interface{} {
 			d.FieldValueU("tag", count, tagMap)
 			d.FieldStruct("value", func(d *decode.D) { decodeCBORValue(d) })
-			return count
+			return nil
 		}},
 		majorTypeSpecialFloat: {s: scalar.S{Sym: "special_float"}, d: func(d *decode.D, shortCount uint64, count uint64) interface{} {
 			switch shortCount {

format/cbor/testdata/appendix_a.fqtest

@@ -1,6 +1,8 @@
 # appendix_a.json from https://github.com/cbor/test-vectors
-# TODO: "O///////////" fails due lack of bigint support (is smaller thax mmin int64), also the test JSON has issues truncating decoded value
 # TODO: "w0kBAAAAAAAAAAA=" "wkkBAAAAAAAAAAA=" semantic bigint
+# NOTE: "O///////////" test uses bigint and is correct but test success currently relay on -18446744073709551616
+# in input json being turned into a float as it can't be represented in json and cbor decoded bigint will also be
+# converted to a float when comparing.
 $ fq -i -d json . appendix_a.json
 json> length
 82
@@ -19,15 +21,6 @@ json> map(select(.decoded) | (.cbor | base64 | cbor | torepr) as $a | select( .d
       "roundtrip": true
     }
   },
-  {
-    "actual": 0,
-    "test": {
-      "cbor": "O///////////",
-      "decoded": -18446744073709552000,
-      "hex": "3bffffffffffffffff",
-      "roundtrip": true
-    }
-  },
   {
     "actual": {
       "major_type": "bytes",
@@ -106,7 +99,7 @@ json> .[] | select(.decoded) | .cbor | base64 | cbor | v
 0x0|3b                                             |;               |  major_type: "negative_int" (1) 0x0-0x0.2 (0.3)
 0x0|3b                                             |;               |  short_count: "64bit" (27) 0x0.3-0x0.7 (0.5)
 0x0|   ff ff ff ff ff ff ff ff|                    | ........|      |  variable_count: 18446744073709551615 0x1-0x8.7 (8)
-   |                                               |                |  value: 0 0x9-NA (0)
+   |                                               |                |  value: -18446744073709551616 0x9-NA (0)
    |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: (cbor) 0x0-0xa.7 (11)
 0x0|c3                                             |.               |  major_type: "semantic" (6) 0x0-0x0.2 (0.3)
 0x0|c3                                             |.               |  short_count: 3 0x0.3-0x0.7 (0.5)

internal/num/big.go

@@ -0,0 +1,13 @@
+package num
+
+import "math/big"
+
+var BigIntOne = big.NewInt(1)
+
+func BigIntSetBytesSigned(n *big.Int, buf []byte) *big.Int {
+	n.SetBytes(buf)
+	if len(buf) > 0 && buf[0]&0x80 > 0 {
+		n.Sub(n, new(big.Int).Lsh(BigIntOne, uint(len(buf))*8))
+	}
+	return n
+}

internal/num/big_test.go

@@ -0,0 +1,37 @@
+package num_test
+
+import (
+	"fmt"
+	"math/big"
+	"testing"
+
+	"github.com/wader/fq/internal/num"
+)
+
+func TestBigIntSetBytesSigned(t *testing.T) {
+	testCases := []struct {
+		buf []byte
+		s   string
+	}{
+		{[]byte{1}, "1"},
+		{[]byte{0b1111_1111}, "-1"},
+		{[]byte{0b1000_0000}, "-128"},
+		{[]byte{0b0111_1111}, "127"},
+		{[]byte{0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, "9223372036854775807"},
+		{[]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, "-1"},
+		{[]byte{0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, "-9223372036854775808"},
+		{[]byte{0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, "2361183241434822606847"},
+		{[]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, "-1"},
+		{[]byte{0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, "-2361183241434822606848"},
+	}
+	for _, tC := range testCases {
+		t.Run(fmt.Sprintf("%v %s", tC.buf, tC.s), func(t *testing.T) {
+			var n big.Int
+			expected := tC.s
+			actual := num.BigIntSetBytesSigned(&n, tC.buf).String()
+			if expected != actual {
+				t.Errorf("expected %s, got %s", expected, actual)
+			}
+		})
+	}
+}

internal/num/num.go

@@ -3,6 +3,7 @@ package num
 import (
 	"fmt"
 	"math"
+	"math/big"
 	"strconv"
 	"strings"
 
@@ -46,8 +47,12 @@ func PadFormatInt(i int64, base int, basePrefix bool, width int) string {
 
 func PadFormatUint(i uint64, base int, basePrefix bool, width int) string {
 	return padFormatNumber(strconv.FormatUint(i, base), base, basePrefix, width)
+}
 
+func PadFormatBigInt(i *big.Int, base int, basePrefix bool, width int) string {
+	return padFormatNumber(i.Text(base), base, basePrefix, width)
 }
+
 func MaxUInt64(a, b uint64) uint64 {
 	if a < b {
 		return b

pkg/decode/decode.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math/big"
 
 	"github.com/wader/fq/internal/recoverfn"
 	"github.com/wader/fq/pkg/bitio"
@@ -636,6 +637,10 @@ func (d *D) FieldValueS(name string, a int64, sms ...scalar.Mapper) {
 	d.FieldScalarFn(name, func(_ scalar.S) (scalar.S, error) { return scalar.S{Actual: a}, nil }, sms...)
 }
 
+func (d *D) FieldValueBigInt(name string, a *big.Int, sms ...scalar.Mapper) {
+	d.FieldScalarFn(name, func(_ scalar.S) (scalar.S, error) { return scalar.S{Actual: a}, nil }, sms...)
+}
+
 func (d *D) FieldValueBool(name string, a bool, sms ...scalar.Mapper) {
 	d.FieldScalarFn(name, func(_ scalar.S) (scalar.S, error) { return scalar.S{Actual: a}, nil }, sms...)
 }