Allow to marshal slices

[sedyh]

Problem

An event cannot be made a slice or thrown anywhere in the structure.

On slice instead of structure:
panic: reflect: call of reflect.Value.NumField on slice Value

On slice field inside structure even if it has BinaryMarshal:
read error: redis: can't marshal []struct (implement encoding.BinaryMarshaler)

[dranikpg]

Hi! I might add support for slices soon (as it's requested now), but the simplest workaround for you would be using the Convertible interfaces to do custom parsing

Thanks for reporting 🙂

[sedyh]

I'm not sure if this works. Here's an example:

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"log"

	"github.com/dranikpg/gtrs"
	"github.com/redis/go-redis/v9"
)

type Data struct {
	Items []Item `json:"items"`
}

type Item struct {
	Name string `json:"name"`
}

func (d *Data) ToMap() (m map[string]any, e error) {
	data, err := json.Marshal(d)
	if err != nil {
		return nil, fmt.Errorf("marsal struct to json: %w", err)
	}

	if err = json.Unmarshal(data, &m); err != nil {
		return nil, fmt.Errorf("marsal json to map: %w", err)
	}

	return m, nil
}

func (d *Data) FromMap(m map[string]any) error {
	data, err := json.Marshal(m)
	if err != nil {
		return fmt.Errorf("marsal map to json: %w", err)
	}

	if err = json.Unmarshal(data, &d); err != nil {
		return fmt.Errorf("marsal json to struct: %w", err)
	}

	return nil
}

func main() {
	client := redis.NewClient(&redis.Options{
		Addr:     "0.0.0.0:7777",
		Password: "sider",
		DB:       0,
	})

	stream := gtrs.NewStream[Data](
		client, "main", &gtrs.Options{},
	)

	_, err := stream.Add(
		context.Background(),
		Data{
			Items: []Item{
				{Name: "a"},
				{Name: "b"},
				{Name: "c"},
			},
		},
	)
	if err != nil {
		log.Fatal(err)
	}
}

It produces:

read error: redis: can't marshal []main.Item (implement encoding.BinaryMarshaler)

Why does it require implementing another marshaler if Convertible will produce the whole thing?

This kinda works, but it doesn't need Convertible at all. As far as I can see, only BinaryMarshaler works and Convertible is ignored:

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"log"

	"github.com/dranikpg/gtrs"
	"github.com/redis/go-redis/v9"
)

type Items []Item

type Data struct {
	Items Items `json:"items"`
}

type Item struct {
	Name string `json:"name"`
}

func (d *Data) ToMap() (m map[string]any, e error) {
	data, err := json.Marshal(d)
	if err != nil {
		return nil, fmt.Errorf("marsal struct to json: %w", err)
	}

	if err = json.Unmarshal(data, &m); err != nil {
		return nil, fmt.Errorf("marsal json to map: %w", err)
	}

	return m, nil
}

func (d *Data) FromMap(m map[string]any) error {
	data, err := json.Marshal(m)
	if err != nil {
		return fmt.Errorf("marsal map to json: %w", err)
	}

	if err = json.Unmarshal(data, &d); err != nil {
		return fmt.Errorf("marsal json to struct: %w", err)
	}

	return nil
}

func (i Items) MarshalBinary() ([]byte, error) {
	return json.Marshal(i)
}

func main() {
	client := redis.NewClient(&redis.Options{
		Addr:     "0.0.0.0:7777",
		Password: "sider",
		DB:       0,
	})

	stream := gtrs.NewStream[Data](
		client, "main", &gtrs.Options{},
	)

	_, err := stream.Add(
		context.Background(),
		Data{
			Items: []Item{
				{Name: "a"},
				{Name: "b"},
				{Name: "c"},
			},
		},
	)
	if err != nil {
		log.Fatal(err)
	}
}

[sedyh]

Also, looks like you can't unmarshal slice back after consuming:

fail to parse stream: map[items:[{"name":"a"},{"name":"b"},{"name":"c"}]] parsing error: cause: failed to parse field Items, got [{"name":"a"},{"name":"b"},{"name":"c"}], because unsupported field type

package main

import (
	"context"
	"encoding/json"
	"log"

	"github.com/dranikpg/gtrs"
	"github.com/redis/go-redis/v9"
)

type Items []Item

type Data struct {
	Items Items `json:"items"`
}

type Item struct {
	Name string `json:"name"`
}

func (i Items) UnmarshalBinary(data []byte) error {
	return json.Unmarshal(data, &i)
}

func main() {
	client := redis.NewClient(&redis.Options{
		Addr:     "0.0.0.0:7777",
		Password: "sider",
		DB:       0,
	})

	stream := gtrs.NewConsumer[Data](
		context.Background(), client,
		gtrs.StreamIDs{"main": "0-0"},
	)
	defer stream.Close()

	log.Println("consuming")

	for msg := range stream.Chan() {
		switch v := msg.Err.(type) {
		case gtrs.AckError:
			log.Println("fail to ack stream:", msg.Stream, msg.ID)
			continue
		case gtrs.ParseError:
			log.Println("fail to parse stream:", v.Data, msg.Err)
			continue
		case gtrs.ReadError:
			log.Println("fail to read stream:", msg.Err)
			return
		case nil:
			for _, item := range msg.Data.Items {
				log.Println("item:", item.Name)
			}
			log.Println()
		}
	}
}

Producer:

package main

import (
	"context"
	"encoding/json"
	"log"

	"github.com/dranikpg/gtrs"
	"github.com/redis/go-redis/v9"
)

type Items []Item

type Data struct {
	Items Items `json:"items"`
}

type Item struct {
	Name string `json:"name"`
}

func (i Items) MarshalBinary() ([]byte, error) {
	return json.Marshal(i)
}

func main() {
	client := redis.NewClient(&redis.Options{
		Addr:     "0.0.0.0:7777",
		Password: "sider",
		DB:       0,
	})

	stream := gtrs.NewStream[Data](
		client, "main", &gtrs.Options{},
	)

	_, err := stream.Add(
		context.Background(),
		Data{
			Items: []Item{
				{Name: "a"},
				{Name: "b"},
				{Name: "c"},
			},
		},
	)
	if err != nil {
		log.Fatal(err)
	}
}

[sedyh]

This is not a solution for this case:

func (i Items) MarshalBinary() ([]byte, error) {
	return json.Marshal(i)
}

It will result to escaped nested arrays, therefore they cannot be parsed through marshal map->string + unmarshal string->struct:

{"items":"[{\"name\":\"a\"},{\"name\":\"b\"},{\"name\":\"c\"}]"}

parsing error: cause: marsal string to json: json: cannot unmarshal string into Go struct field Data.items of type main.Items

It could work if the consumer output called the UnmarshalBinary implementation without Convertible, but it's ignored.

parsing error: cause: failed to parse field Items, got [{"name":"a"},{"name":"b"},{"name":"c"}], because unsupported field type

[sedyh]

Please confirm my observations: redis (and streams too) does not support nested objects and they have to be escaped into strings and folded into each other, and then expanded back. As far as I understand, this process was done for structures in the library, but not for slices, right?

[dranikpg]

Why does it require implementing another marshaler if Convertible will produce the whole thing?
This kinda works, but it doesn't need Convertible at all. As far as I can see, only BinaryMarshaler works and Convertible is ignored:

So the simple answer (or more like my thoughts when I implemented it):

When I implemented it, I saw Convertible just as a way to manipulate the key/value pairs at the highest level. All values are passed on to the redis client, as it's technically not the libraries job to handle complex serialization. Not a feature that was promised 😄

It could work if the consumer output called the UnmarshalBinary implementation without Convertible, but it's ignored.

Yes, I agree that's a problem, because for serialization it happens automatically - here not. The only way currently is to do it with Convertible -> unnecessary boilerplate

I see that #17 adds this functionality, so we can actually incorporate it

[elee1766]

@dranikpg mind pushing a new release and we can close this i think ?

[dranikpg]

Closed as with https://github.com/dranikpg/gtrs/releases/tag/v0.6.0