Skip to content

Commit

Permalink
interp: Rework buffer, still confusing
Browse files Browse the repository at this point in the history
  • Loading branch information
@wader
wader committed Oct 5, 2021
1 parent b0694f5 commit 1c3c65b
Show file tree
Hide file tree
Showing 19 changed files with 195 additions and 182 deletions.
13 changes: 8 additions & 5 deletions pkg/bitio/bitio.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -372,7 +372,10 @@ func NewMultiBitReader(rs []BitReadAtSeeker) (*MultiBitReader, error) {
}

func (m *MultiBitReader) ReadBitsAt(p []byte, nBits int, bitOff int64) (n int, err error) {
end := m.readerEnds[len(m.readers)-1]
var end int64
if len(m.readers) > 0 {
end = m.readerEnds[len(m.readers)-1]
}
if end <= bitOff {
return 0, io.EOF
}
Expand Down Expand Up @@ -406,7 +409,10 @@ func (m *MultiBitReader) ReadBits(p []byte, nBits int) (n int, err error) {

func (m *MultiBitReader) SeekBits(bitOff int64, whence int) (int64, error) {
var p int64
end := m.readerEnds[len(m.readerEnds)-1]
var end int64
if len(m.readers) > 0 {
end = m.readerEnds[len(m.readers)-1]
}

switch whence {
case io.SeekStart:
Expand All @@ -431,9 +437,6 @@ func (m *MultiBitReader) Read(p []byte) (n int, err error) {
n, err = m.ReadBitsAt(p, len(p)*8, m.pos)
m.pos += int64(n)

// log.Printf("n: %#+v\n", n)
// log.Printf("err: %#+v\n", err)

if err != nil {
return int(BitsByteCount(int64(n))), err
}
Expand Down
165 changes: 70 additions & 95 deletions pkg/interp/buffer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,31 +8,40 @@ import (

"github.com/wader/fq/internal/gojqextra"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/ranges"
)

var _ Value = (*BufferView)(nil)
var _ ToBuffer = (*BufferView)(nil)
var _ Value = BufferView{}
var _ ToBufferView = BufferView{}

type BufferView struct {
bb *bitio.Buffer
r ranges.Range
unit int
}

// TODO: JQArray

func newBifBufObject(bb *bitio.Buffer, unit int) BufferView {
func bufferViewFromBuffer(bb *bitio.Buffer, unit int) BufferView {
return BufferView{
bb: bb,
r: ranges.Range{Start: 0, Len: bb.Len()},
unit: unit,
}
}

func (*BufferView) DisplayName() string { return "buffer" }
func (*BufferView) ExtKeys() []string {
func (bv BufferView) toBytesBuffer(r ranges.Range) (*bytes.Buffer, error) {
bb, err := bv.bb.BitBufRange(r.Start, r.Len)
if err != nil {
return nil, err
}
buf := &bytes.Buffer{}
if _, err := io.Copy(buf, bb.Copy()); err != nil {
return nil, err
}
return buf, nil
}

func (BufferView) DisplayName() string { return "buffer" }
func (BufferView) ExtKeys() []string {
return []string{
"size",
"start",
Expand All @@ -42,116 +51,100 @@ func (*BufferView) ExtKeys() []string {
}
}

func (bo BufferView) JQValueLength() interface{} {
return int(bo.r.Len / int64(bo.unit))
func (bv BufferView) ToBufferView() (BufferView, error) {
return bv, nil
}
func (bo BufferView) JQValueSliceLen() interface{} {
return bo.JQValueLength()

func (bv BufferView) JQValueLength() interface{} {
return int(bv.r.Len / int64(bv.unit))
}
func (bo BufferView) JQValueIndex(index int) interface{} {
// TODO: use bitio
/*
pos, err := bo.bbr.bb.Pos()
if err != nil {
return err
}
if _, err := bo.bbr.bb.SeekAbs(int64(index) * int64(bo.unit)); err != nil {
return err
}
v, err := bo.bbr.bb.U(bo.unit)
if err != nil {
return err
}
if _, err := bo.bbr.bb.SeekAbs(pos); err != nil {
return err
}
return int(v)
*/
return nil
func (bv BufferView) JQValueSliceLen() interface{} {
return bv.JQValueLength()
}
func (bv BufferView) JQValueIndex(index int) interface{} {
if index < 0 {
return ""
}
buf, err := bv.toBytesBuffer(ranges.Range{Start: bv.r.Start + int64(index*bv.unit), Len: int64(bv.unit)})
if err != nil {
return err
}
s := buf.String()
return s[0:1]
}
func (bo BufferView) JQValueSlice(start int, end int) interface{} {
rStart := int64(start * bo.unit)
rLen := int64((end - start) * bo.unit)
func (bv BufferView) JQValueSlice(start int, end int) interface{} {
rStart := int64(start * bv.unit)
rLen := int64((end - start) * bv.unit)

return BufferView{
bb: bo.bb,
r: ranges.Range{Start: bo.r.Start + rStart, Len: rLen},
unit: bo.unit,
bb: bv.bb,
r: ranges.Range{Start: bv.r.Start + rStart, Len: rLen},
unit: bv.unit,
}
}
func (bo BufferView) JQValueKey(name string) interface{} {
func (bv BufferView) JQValueKey(name string) interface{} {
switch name {
case "size":
return new(big.Int).SetInt64(bo.r.Len / int64(bo.unit))
return new(big.Int).SetInt64(bv.r.Len / int64(bv.unit))
case "start":
return new(big.Int).SetInt64(bo.r.Start / int64(bo.unit))
return new(big.Int).SetInt64(bv.r.Start / int64(bv.unit))
case "stop":
stop := bo.r.Stop()
stopUnits := stop / int64(bo.unit)
if stop%int64(bo.unit) != 0 {
stop := bv.r.Stop()
stopUnits := stop / int64(bv.unit)
if stop%int64(bv.unit) != 0 {
stopUnits++
}
return new(big.Int).SetInt64(stopUnits)
case "bits":
if bo.unit == 1 {
return bo
if bv.unit == 1 {
return bv
}
return BufferView{bb: bo.bb, r: bo.r, unit: 1}
return BufferView{bb: bv.bb, r: bv.r, unit: 1}
case "bytes":
if bo.unit == 8 {
return bo
if bv.unit == 8 {
return bv
}
return BufferView{bb: bo.bb, r: bo.r, unit: 8}
return BufferView{bb: bv.bb, r: bv.r, unit: 8}
}
return nil
}
func (bo BufferView) JQValueEach() interface{} {
func (bv BufferView) JQValueEach() interface{} {
return nil
}
func (bo BufferView) JQValueType() string {
func (bv BufferView) JQValueType() string {
return "buffer"
}
func (bo BufferView) JQValueKeys() interface{} {
func (bv BufferView) JQValueKeys() interface{} {
return gojqextra.FuncTypeError{Name: "keys", Typ: "buffer"}
}
func (bo BufferView) JQValueHas(key interface{}) interface{} {
func (bv BufferView) JQValueHas(key interface{}) interface{} {
return gojqextra.HasKeyTypeError{L: "buffer", R: fmt.Sprintf("%v", key)}
}
func (bo BufferView) JQValueToNumber() interface{} {
bb, err := bo.ToBuffer()
func (bv BufferView) JQValueToNumber() interface{} {
buf, err := bv.toBytesBuffer(bv.r)
if err != nil {
return err
}
buf := &bytes.Buffer{}
if _, err := io.Copy(buf, bb.Copy()); err != nil {
return err
}
extraBits := uint((8 - bo.r.Len%8) % 8)
extraBits := uint((8 - bv.r.Len%8) % 8)
return new(big.Int).Rsh(new(big.Int).SetBytes(buf.Bytes()), extraBits)
}
func (bo BufferView) JQValueToString() interface{} {
return bo.JQValueToGoJQ()
func (bv BufferView) JQValueToString() interface{} {
return bv.JQValueToGoJQ()
}

func (bo BufferView) JQValueToGoJQ() interface{} {
bb, err := bo.ToBuffer()
func (bv BufferView) JQValueToGoJQ() interface{} {
buf, err := bv.toBytesBuffer(bv.r)
if err != nil {
return err
}
buf := &bytes.Buffer{}
if _, err := io.Copy(buf, bb.Copy()); err != nil {
return err
}
return buf.String()
}

func (bo BufferView) JQValueUpdate(key interface{}, u interface{}, delpath bool) interface{} {
func (bv BufferView) JQValueUpdate(key interface{}, u interface{}, delpath bool) interface{} {
return notUpdateableError{Key: fmt.Sprintf("%v", key), Typ: "buffer"}
}

func (bo BufferView) Display(w io.Writer, opts Options) error {
func (bv BufferView) Display(w io.Writer, opts Options) error {
if opts.RawOutput {
bb, err := bo.ToBuffer()
bb, err := bv.toBuffer()
if err != nil {
return err
}
Expand All @@ -161,27 +154,9 @@ func (bo BufferView) Display(w io.Writer, opts Options) error {
return nil
}

unitNames := map[int]string{
1: "bits",
8: "bytes",
}
unitName := unitNames[bo.unit]
if unitName == "" {
unitName = "units"
}

// TODO: hack
return dump(
&decode.Value{
Range: bo.r,
RootBitBuf: bo.bb.Copy(),
Description: fmt.Sprintf("%d %s", bo.r.Len/int64(bo.unit), unitName),
},
w,
opts,
)
return hexdump(w, bv, opts)
}

func (bo BufferView) ToBuffer() (*bitio.Buffer, error) {
return bo.bb.BitBufRange(bo.r.Start, bo.r.Len)
func (bv BufferView) toBuffer() (*bitio.Buffer, error) {
return bv.bb.BitBufRange(bv.r.Start, bv.r.Len)
}
13 changes: 13 additions & 0 deletions pkg/interp/dump.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -349,3 +349,16 @@ func dump(v *decode.Value, w io.Writer, opts Options) error {
return dumpEx(v, buf, cw, depth, rootV, rootDepth, maxAddrIndentWidth-rootDepth, opts)
}))
}

func hexdump(w io.Writer, bv BufferView, opts Options) error {
// TODO: hack
opts.Verbose = true
return dump(
&decode.Value{
Range: bv.r,
RootBitBuf: bv.bb.Copy(),
},
w,
opts,
)
}
35 changes: 26 additions & 9 deletions pkg/interp/funcs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ func (i *Interp) makeFunctions() []Function {

{[]string{"format"}, 0, 0, i.format, nil},
{[]string{"_display"}, 1, 1, nil, i._display},
{[]string{"_hexdump"}, 1, 1, nil, i._hexdump},

{[]string{"tobytes"}, 0, 0, i.toBytes, nil},
{[]string{"tobits"}, 0, 0, i.toBits, nil},
Expand Down Expand Up @@ -122,7 +123,7 @@ func makeStringBitBufTransformFn(
}
outBB := bitio.NewBufferFromBytes(buf.Bytes(), -1)

return newBifBufObject(outBB, 8)
return bufferViewFromBuffer(outBB, 8)
default:
bb, err := toBuffer(c)
if err != nil {
Expand Down Expand Up @@ -168,7 +169,7 @@ func makeBitBufTransformFn(fn func(r io.Reader) (io.Reader, error)) func(c inter

outBB := bitio.NewBufferFromBytes(outBuf.Bytes(), -1)

return newBifBufObject(outBB, 8)
return bufferViewFromBuffer(outBB, 8)
}
}

Expand All @@ -190,7 +191,7 @@ func makeHashFn(fn func() (hash.Hash, error)) func(c interface{}, a []interface{

outBB := bitio.NewBufferFromBytes(h.Sum(nil), -1)

return newBifBufObject(outBB, 8)
return bufferViewFromBuffer(outBB, 8)
}
}

Expand Down Expand Up @@ -488,15 +489,15 @@ type bitBufFile struct {
progressFn progressreadseeker.ProgressFn
}

var _ ToBuffer = (*bitBufFile)(nil)
var _ ToBufferView = (*bitBufFile)(nil)

func (bbf *bitBufFile) Display(w io.Writer, opts Options) error {
_, err := fmt.Fprintln(w, bbf.JQValue.JQValueToString())
return err
}

func (bbf *bitBufFile) ToBuffer() (*bitio.Buffer, error) {
return bbf.bb.Copy(), nil
func (bbf *bitBufFile) ToBufferView() (BufferView, error) {
return bufferViewFromBuffer(bbf.bb, 8), nil
}

// def open: #:: string| => buffer
Expand Down Expand Up @@ -700,15 +701,15 @@ func (i *Interp) toBytes(c interface{}, a []interface{}) interface{} {
if err != nil {
return err
}
return newBifBufObject(bb, 8)
return bufferViewFromBuffer(bb, 8)
}

func (i *Interp) toBits(c interface{}, a []interface{}) interface{} {
bb, err := toBuffer(c)
if err != nil {
return err
}
return newBifBufObject(bb, 1)
return bufferViewFromBuffer(bb, 1)
}

func (i *Interp) toValue(c interface{}, a []interface{}) interface{} {
Expand Down Expand Up @@ -833,8 +834,24 @@ func (i *Interp) find(c interface{}, a []interface{}) gojq.Iter {
return gojq.NewIter()
}

bbo := newBifBufObject(bb, 8)
bbo := bufferViewFromBuffer(bb, 8)
// log.Printf("bbo: %#+v\n", bbo)

return gojq.NewIter(bbo)
}

func (i *Interp) _hexdump(c interface{}, a []interface{}) gojq.Iter {
opts, err := i.Options(a...)
if err != nil {
return gojq.NewIter(err)
}
bv, err := toBufferView(c)
if err != nil {
return gojq.NewIter(err)
}
if err := hexdump(i.evalContext.output, bv, opts); err != nil {
return gojq.NewIter(err)
}

return gojq.NewIter()
}

0 comments on commit 1c3c65b

Please sign in to comment.