Skip to content
This repository has been archived by the owner on Feb 8, 2022. It is now read-only.

Commit

Permalink
Issue #68 - Add Encoder support for proto2 *required* fields (#69)
Browse files Browse the repository at this point in the history 
Pulling this into master so it goes out with the next release.
  • Loading branch information
@jhugard
jhugard committed Feb 23, 2017
1 parent c5f89f2 commit 717ae69
Show file tree
Hide file tree
Showing 3 changed files with 190 additions and 18 deletions.
115 changes: 114 additions & 1 deletion Serialization.Test/TestEncoding.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,11 @@ module Encode =
|> toArray
|> should equal Array.empty

ZCB(2)
|> Encode.fromRequiredVarint fid 0UL
|> toArray
|> should equal [| 0x08uy; 0uy |]

ZCB(2)
|> Encode.fromVarint fid 2UL
|> toArray
Expand All @@ -273,6 +278,11 @@ module Encode =
|> toArray
|> should equal Array.empty

ZCB(2)
|> Encode.fromRequiredSInt32 fid 0
|> toArray
|> should equal [| 0x08uy; 0uy|]

ZCB(2)
|> Encode.fromSInt32 fid -1
|> toArray
Expand All @@ -290,6 +300,11 @@ module Encode =
|> toArray
|> should equal Array.empty

ZCB(2)
|> Encode.fromRequiredSInt64 fid 0L
|> toArray
|> should equal [| 0x08uy; 0uy |]

ZCB(2)
|> Encode.fromSInt64 fid -1L
|> toArray
Expand All @@ -307,83 +322,180 @@ module Encode =
|> toArray
|> should equal Array.empty

ZCB(2)
|> Encode.fromRequiredBool fid false
|> toArray
|> should equal [| 0x08uy; 0uy |]

ZCB(2)
|> Encode.fromBool fid true
|> toArray
|> should equal [| 0x08uy; 1uy |]

[<Fact>]
let ``Encode Fixed32`` () =
ZCB(5)
|> Encode.fromFixed32 fid 0u
|> toArray
|> should equal Array.empty

ZCB(5)
|> Encode.fromRequiredFixed32 fid 0u
|> toArray
|> should equal [| 0x08uy ||| 5uy; 0uy;0uy;0uy;0uy |]

ZCB(5)
|> Encode.fromFixed32 fid 5u
|> toArray
|> should equal [| 0x08uy ||| 5uy; 5uy;0uy;0uy;0uy |]


[<Fact>]
let ``Encode Fixed64`` () =
ZCB(9)
|> Encode.fromFixed64 fid 0UL
|> toArray
|> should equal Array.empty

ZCB(9)
|> Encode.fromRequiredFixed64 fid 0UL
|> toArray
|> should equal [| 0x08uy ||| 1uy; 0uy;0uy;0uy;0uy; 0uy;0uy;0uy;0uy |]

ZCB(9)
|> Encode.fromFixed64 fid 5UL
|> toArray
|> should equal [| 0x08uy ||| 1uy; 5uy;0uy;0uy;0uy; 0uy;0uy;0uy;0uy |]

[<Fact>]
let ``Encode SFixed32`` () =
ZCB(5)
|> Encode.fromSFixed32 fid 0
|> toArray
|> should equal Array.empty

ZCB(5)
|> Encode.fromRequiredSFixed32 fid 0
|> toArray
|> should equal [| 0x08uy ||| 5uy; 0uy;0uy;0uy;0uy |]

ZCB(5)
|> Encode.fromSFixed32 fid 5
|> toArray
|> should equal [| 0x08uy ||| 5uy; 5uy;0uy;0uy;0uy |]

[<Fact>]
let ``Encode SFixed64`` () =
ZCB(9)
|> Encode.fromSFixed64 fid 0L
|> toArray
|> should equal Array.empty

ZCB(9)
|> Encode.fromRequiredSFixed64 fid 0L
|> toArray
|> should equal [| 0x08uy ||| 1uy; 0uy;0uy;0uy;0uy; 0uy;0uy;0uy;0uy |]

ZCB(9)
|> Encode.fromSFixed64 fid 5L
|> toArray
|> should equal [| 0x08uy ||| 1uy; 5uy;0uy;0uy;0uy; 0uy;0uy;0uy;0uy |]

[<Fact>]
let ``Encode Single`` () =
ZCB(5)
|> Encode.fromSingle fid 0.0f
|> toArray
|> should equal Array.empty

ZCB(5)
|> Encode.fromRequiredSingle fid 0.0f
|> toArray
|> should equal [| 0x08uy ||| 5uy; 0uy; 0uy; 0b00000000uy; 0b00000000uy |]

ZCB(5)
|> Encode.fromSingle fid 2.0f
|> toArray
|> should equal [| 0x08uy ||| 5uy; 0uy; 0uy; 0b00000000uy; 0b01000000uy |]

[<Fact>]
let ``Encode Double`` () =
ZCB(9)
|> Encode.fromDouble fid 0.0
|> toArray
|> should equal Array.empty

ZCB(9)
|> Encode.fromRequiredDouble fid 0.0
|> toArray
|> should equal [| 0x08uy ||| 1uy; 0x00uy; 0x00uy; 0x00uy; 0x00uy; 0x00uy; 0x00uy; 0x00uy; 0x00uy |]

ZCB(9)
|> Encode.fromDouble fid 0.10
|> toArray
|> should equal [| 0x08uy ||| 1uy; 0x9Auy; 0x99uy; 0x99uy; 0x99uy; 0x99uy; 0x99uy; 0xB9uy; 0x3Fuy |]

[<Fact>]
let ``Encode String`` () =
ZCB(6)
|> Encode.fromString fid ""
|> toArray
|> should equal Array.empty

ZCB(6)
|> Encode.fromRequiredString fid ""
|> toArray
|> should equal [| 0x08uy ||| 2uy; 0uy |]

ZCB(6)
|> Encode.fromString fid "0ABC"
|> toArray
|> should equal [| 0x08uy ||| 2uy; 4uy; 0x30uy; 0x41uy; 0x42uy; 0x43uy |]

[<Fact>]
let ``Encode Bytes`` () =
ZCB(6)
|> Encode.fromBytes fid (ArraySegment(Array.empty))
|> toArray
|> should equal Array.empty

ZCB(6)
|> Encode.fromRequiredBytes fid (ArraySegment(Array.empty))
|> toArray
|> should equal [| 0x08uy ||| 2uy; 0uy |]

ZCB(6)
|> Encode.fromBytes fid (ArraySegment([| 3uy; 4uy; 5uy; 6uy; |]))
|> toArray
|> should equal [| 0x08uy ||| 2uy; 4uy; 3uy; 4uy; 5uy; 6uy |]


[<Fact>]
let ``Encode with default value`` () =
// Verify non-default value results in an actual value
ZCB(2)
|> Encode.fromDefaultedVarint 0UL fid 2UL
|> toArray
|> should equal [| 0x08uy; 2uy |]

ZCB(6)
|> Encode.fromDefaultedBytes (ArraySegment[| 3uy; 4uy |]) fid (ArraySegment([| 3uy; 4uy; 5uy; 6uy; |]))
|> toArray
|> should equal [| 0x08uy ||| 2uy; 4uy; 3uy; 4uy; 5uy; 6uy |]

ZCB(6)
|> Encode.fromDefaultedBytes (ArraySegment[| 3uy; 4uy; 5uy; 7uy |]) fid (ArraySegment([| 3uy; 4uy; 5uy; 6uy; |]))
|> toArray
|> should equal [| 0x08uy ||| 2uy; 4uy; 3uy; 4uy; 5uy; 6uy |]

// Now, check that default value result in an empty array (value is elided)
let checkGetsElided f =
ZCB(16)
|> f
|> toArray
|> should equal Array.empty

checkGetsElided <| Encode.fromDefaultedVarint 1UL fid 1UL
checkGetsElided <| Encode.fromDefaultedVarint 5UL fid 5UL
checkGetsElided <| Encode.fromDefaultedSInt32 2 fid 2
checkGetsElided <| Encode.fromDefaultedSInt64 3L fid 3L
checkGetsElided <| Encode.fromDefaultedBool true fid true
Expand All @@ -395,6 +507,7 @@ module Encode =
checkGetsElided <| Encode.fromDefaultedDouble 0.70 fid 0.70
checkGetsElided <| Encode.fromDefaultedString "Hello" fid "Hello"
checkGetsElided <| Encode.fromDefaultedBytes (ArraySegment([|8uy;9uy|])) fid (ArraySegment([|8uy;9uy|]))
checkGetsElided <| Encode.fromDefaultedBytes (ArraySegment([||])) fid (ArraySegment([||]))

[<Fact>]
let ``Encode Packed Varint`` () =
Expand Down
92 changes: 75 additions & 17 deletions Serialization/Encoding.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,29 +149,81 @@ module Encode =
/// If value = default, then elide the field (don't serialize)
let inline elideDefault defV v f =
if v = defV
then id
else f
then
id
else
f

/// Generic Encode for all varint types, excepted for signed & bool:
/// int32, int64, uint32, uint64, enum
let inline fromDefaultedVarint defV fldNum v = elideDefault defV v <| Pack.toFieldVarint fldNum (uint64 v)
let inline fromNondefaultedVarint fldNum v = Pack.toFieldVarint fldNum (uint64 v)

let fromDefaultedSInt32 defV fldNum v = elideDefault defV v <| Pack.toFieldVarint fldNum (zigZag32 v |> uint64)
let fromDefaultedSInt64 defV fldNum v = elideDefault defV v <| Pack.toFieldVarint fldNum (zigZag64 v |> uint64)
let fromDefaultedBool defV fldNum v = elideDefault defV v <| fromNondefaultedVarint fldNum (boolToInt64 v)
let fromDefaultedFixed32 defV fldNum v = elideDefault defV v <| Pack.toFieldFixed32 fldNum (uint32 v)
let fromDefaultedFixed64 defV fldNum v = elideDefault defV v <| Pack.toFieldFixed64 fldNum (uint64 v)
let fromDefaultedSFixed32 defV fldNum (v:int32) = elideDefault defV v <| Pack.toFieldFixed32 fldNum (uint32 v)
let fromDefaultedSFixed64 defV fldNum v = elideDefault defV v <| Pack.toFieldFixed64 fldNum (uint64 v)
// For proto2 "required" values
let inline fromRequiredVarint fldNum v = Pack.toFieldVarint fldNum (uint64 v)
let fromRequiredSInt32 fldNum v = Pack.toFieldVarint fldNum (zigZag32 v |> uint64)
let fromRequiredSInt64 fldNum v = Pack.toFieldVarint fldNum (zigZag64 v |> uint64)
let fromRequiredBool fldNum v = fromRequiredVarint fldNum (boolToInt64 v)
let fromRequiredFixed32 fldNum v = Pack.toFieldFixed32 fldNum (uint32 v)
let fromRequiredFixed64 fldNum v = Pack.toFieldFixed64 fldNum (uint64 v)
let fromRequiredSFixed32 fldNum (v:int32) = Pack.toFieldFixed32 fldNum (uint32 v)
let fromRequiredSFixed64 fldNum v = Pack.toFieldFixed64 fldNum (uint64 v)
let fromRequiredSingle fldNum v = Pack.toFieldSingle fldNum v
let fromRequiredDouble fldNum v = Pack.toFieldDouble fldNum v
let fromRequiredString fldNum v = Pack.toFieldString fldNum v
let fromRequiredBytes fldNum v = Pack.toFieldBytes fldNum v

[<Obsolete("fromNondefaultedVarint is deprecated; please use fromRequiredVarint instead")>]
let inline fromNondefaultedVarint fldNum v = fromRequiredVarint fldNum v

// For proto2 "defaulted" values
let inline fromDefaultedVarint defV fldNum v = elideDefault defV v <| Pack.toFieldVarint fldNum (uint64 v)

let fromDefaultedSingle defV fldNum v = elideDefault defV v <| Pack.toFieldSingle fldNum v
let fromDefaultedDouble defV fldNum v = elideDefault defV v <| Pack.toFieldDouble fldNum v
let fromDefaultedSInt32 defV fldNum v = elideDefault defV v <| fromRequiredSInt32 fldNum v
let fromDefaultedSInt64 defV fldNum v = elideDefault defV v <| fromRequiredSInt64 fldNum v
let fromDefaultedBool defV fldNum v = elideDefault defV v <| fromRequiredBool fldNum v

let fromDefaultedFixed32 defV fldNum v = elideDefault defV v <| fromRequiredFixed32 fldNum v
let fromDefaultedFixed64 defV fldNum v = elideDefault defV v <| fromRequiredFixed64 fldNum v
let fromDefaultedSFixed32 defV fldNum v = elideDefault defV v <| fromRequiredSFixed32 fldNum v
let fromDefaultedSFixed64 defV fldNum v = elideDefault defV v <| fromRequiredSFixed64 fldNum v

let fromDefaultedSingle defV fldNum v = elideDefault defV v <| fromRequiredSingle fldNum v
let fromDefaultedDouble defV fldNum v = elideDefault defV v <| fromRequiredDouble fldNum v

let fromDefaultedString defV fldNum v = elideDefault defV v <| fromRequiredString fldNum v
let fromDefaultedBytes defV fldNum (v:ArraySegment<byte>) =
let mutable bDefault = true

// Default if the ArraySegments are equal (same array, same offset, same count)
if v = defV then
begin end
// Default if both are empty (even if the empty arrays are different)
else if v.Count = 0 && defV.Count = 0 then
begin end
// NOT default if counts are different
else if v.Count <> defV.Count then
bDefault <- false
// Otherwise same size: do a byte-for-byte compare
else
let mutable i = v.Offset
let mutable j = defV.Offset
let mutable k = defV.Count
let a = v.Array
let b = defV.Array
while k > 0 do
if a.[i] <> b.[j] then
bDefault <- false
k <- 0
else
i <- i + 1
j <- j + 1
k <- k - 1

if bDefault
then id
else fromRequiredBytes fldNum v

let fromDefaultedString defV fldNum v = elideDefault defV v <| Pack.toFieldString fldNum v
let fromDefaultedBytes defV fldNum v = elideDefault defV v <| Pack.toFieldBytes fldNum v

// For proto3, which uses "0" as the default
let inline fromVarint fldNum (v:'a) = fromDefaultedVarint (Unchecked.defaultof<'a>) fldNum v

let fromSInt32 fldNum v = fromDefaultedSInt32 0 fldNum v
Expand All @@ -188,7 +240,10 @@ module Encode =
let fromDouble fldNum v = fromDefaultedDouble 0.0 fldNum v

let fromString fldNum v = fromDefaultedString "" fldNum v
let fromBytes fldNum v = fromDefaultedBytes (ArraySegment ([||]:byte array)) fldNum v
let fromBytes fldNum (v:ArraySegment<byte>) =
if v.Count <> 0
then fromRequiredBytes fldNum v
else id


(* Encode Repeated Packed Numeric Values *)
Expand Down Expand Up @@ -303,6 +358,9 @@ module Encode =
Pack.toTag fieldNum WireType.LengthDelimited
>> fn m

/// Same as fromMessage
let fromRequiredMessage = fromMessage

/// Helper to encode an optional message, or nothing if None
let fromOptionalMessage (fn:'m -> ZeroCopyBuffer -> ZeroCopyBuffer) fieldNum (m:'m option) =
m |> Option.foldBack (fun o -> fromMessage fn fieldNum o)
Expand Down
1 change: 1 addition & 0 deletions release_notes.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
### 0.4.0 _ under construction
* [#3](https://github.com/ctaggart/froto/issues/3) F# type provider
* [#15](https://github.com/ctaggart/froto/issues/15) froto.exe code generator for proto3
* [#68](https://github.com/ctaggart/froto/issues/68) Add Encoders for proto2 *required* fields


### 0.3.1 _ 2016-06
Expand Down

1 comment on commit 717ae69

@ctaggart
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Love seeing all those unit tests @jhugard. 👍

Please sign in to comment.