Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Поддержка zstd словаря #184

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions api/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -98,9 +98,7 @@ VK API способно возвращать ответ в виде [MessagePack
СЛОМАННУЮ КОДИРОВКУ.

Для сжатия, вместо классического gzip, можно использовать
[zstd](https://github.com/facebook/zstd). Сейчас vksdk поддерживает zstd без
словаря. Если кто знает как получать словарь,
[отпишитесь сюда](https://github.com/SevereCloud/vksdk/issues/180).
[zstd](https://github.com/facebook/zstd).

```go
vk := api.NewVK(os.Getenv("USER_TOKEN"))
Expand Down Expand Up @@ -128,6 +126,18 @@ if err != nil {
log.Println("msgpack:", len(r)) // msgpack: 650775
```

Чтобы загрузить и использовать zstd словарь, воспользуйтесь
методом `vk.UpdateZstdDict()`

```go
vk.EnableZstd()

err := vk.UpdateZstdDict()
if err != nil {
log.Fatal(err)
}
```

### Запрос любого метода

Пример запроса [users.get](https://vk.com/dev/users.get)
Expand Down
15 changes: 15 additions & 0 deletions api/account.go
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,21 @@ func (vk *VK) AccountGetPushSettings(params Params) (response AccountGetPushSett
return
}

// AccountGetZSTDDictResponse struct.
type AccountGetZSTDDictResponse struct {
Link string `json:"link"`
Version string `json:"version"`
Hash string `json:"hash"`
}

// AccountGetZSTDDict method.
//
// https://vk.com/dev/account.getZSTDDict
func (vk *VK) AccountGetZSTDDict(params Params) (response AccountGetZSTDDictResponse, err error) {
err = vk.RequestUnmarshal("account.getZSTDDict", &response, params)
return
}

// AccountRegisterDevice subscribes an iOS/Android/Windows/Mac based device to receive push notifications.
//
// https://vk.com/dev/account.registerDevice
Expand Down
10 changes: 10 additions & 0 deletions api/account_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,16 @@ func TestVK_AccountGetPushSettings(t *testing.T) {
noError(t, err)
}

func TestVK_AccountGetZSTDDict(t *testing.T) {
t.Parallel()

needUserToken(t)

vkUser.EnableZstd()
err := vkUser.UpdateZstdDict()
noError(t, err)
}

// func TestVK_AccountRegisterDevice(t *testing.T) {
// TODO: Add test cases
// }
Expand Down
58 changes: 57 additions & 1 deletion api/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,9 @@ type VK struct {
msgpack bool
zstd bool

zstdVersion string
zstdDict zstd.DOption

mux sync.Mutex
lastTime time.Time
rps int
Expand Down Expand Up @@ -260,6 +263,10 @@ func (vk *VK) DefaultHandler(method string, sliceParams ...Params) (Response, er

req.Header.Set("Accept-Encoding", acceptEncoding)

if vk.zstdVersion != "" {
req.Header.Set("x-zstd-dict-version", vk.zstdVersion)
}

var reader io.Reader

resp, err := vk.Client.Do(req)
Expand All @@ -269,7 +276,7 @@ func (vk *VK) DefaultHandler(method string, sliceParams ...Params) (Response, er

switch resp.Header.Get("Content-Encoding") {
case "zstd":
reader, _ = zstd.NewReader(resp.Body)
reader, _ = zstd.NewReader(resp.Body, vk.zstdDict)
case "gzip":
reader, _ = gzip.NewReader(resp.Body)
default:
Expand Down Expand Up @@ -370,6 +377,55 @@ func (vk *VK) EnableZstd() {
vk.zstd = true
}

// SetZstdDict set vk zstd dict.
func (vk *VK) SetZstdDict(data []byte) error {
// zstd dictionary start with Magic_Number: 4 bytes ID, value 0xEC30A437,
// little-endian format.
//
// https://datatracker.ietf.org/doc/html/rfc8878#section-5
indexMagicNumber := bytes.Index(data, []byte{0x37, 0xa4, 0x30, 0xec})
if indexMagicNumber == -1 {
return fmt.Errorf("api: zstd dictionary not found. %w", zstd.ErrMagicMismatch)
}

vk.zstdVersion = string(data[:indexMagicNumber])
vk.zstdDict = zstd.WithDecoderDicts(data[indexMagicNumber:])

return nil
}

// LoadZstdDict return zstd dict.
func (vk *VK) LoadZstdDict() (data []byte, err error) {
r, err := vk.AccountGetZSTDDict(nil)
if err != nil {
return
}

resp, err := vk.Client.Get(r.Link)
if err != nil {
return
}

data, err = io.ReadAll(resp.Body)
if err != nil {
return
}

err = resp.Body.Close()

return
}

// UpdateZstdDict update vk zstd dict.
func (vk *VK) UpdateZstdDict() error {
data, err := vk.LoadZstdDict()
if err != nil {
return err
}

return vk.SetZstdDict(data)
}

func fmtReflectValue(value reflect.Value, depth int) string {
switch f := value; value.Kind() {
case reflect.Invalid:
Expand Down
2 changes: 1 addition & 1 deletion doc.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@ package vksdk

// Module constants.
const (
Version = "2.13.0"
Version = "2.14.0"
API = "5.131"
)