-
Notifications
You must be signed in to change notification settings - Fork 22
/
embedding.go
124 lines (106 loc) · 2.82 KB
/
embedding.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
/*
Copyright 2023 KubeAGI.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package dashscope
import (
"compress/gzip"
"io"
"net/http"
"os"
"path/filepath"
)
type EmbeddingRequest struct {
Model Model `json:"model"`
Input EmbeddingInput `json:"input"`
Parameters EmbeddingParameters `json:"parameters"`
}
type EmbeddingInput struct {
*EmbeddingInputSync
*EmbeddingInputAsync
}
type EmbeddingInputSync struct {
Texts []string `json:"texts,omitempty"`
}
type EmbeddingInputAsync struct {
URL string `json:"url,omitempty"`
}
type EmbeddingParameters struct {
TextType TextType `json:"text_type"`
}
type TextType string
const (
TextTypeQuery TextType = "query"
TextTypeDocument TextType = "document"
)
type EmbeddingResponse struct {
CommonResponse
Output EmbeddingOutput `json:"output"`
Usage EmbeddingUsage `json:"usage"`
}
type EmbeddingOutput struct {
*EmbeddingOutputSync
*EmbeddingOutputASync
}
type EmbeddingOutputSync struct {
Embeddings []Embeddings `json:"embeddings"`
}
type EmbeddingOutputASync struct {
TaskID string `json:"task_id"`
TaskStatus TaskStatus `json:"task_status"`
URL string `json:"url"`
SubmitTime string `json:"submit_time,omitempty"`
ScheduledTime string `json:"scheduled_time,omitempty"`
EndTime string `json:"end_time,omitempty"`
// when failed
Code string `json:"code,omitempty"`
Message string `json:"message,omitempty"`
}
type Embeddings struct {
TextIndex int `json:"text_index"`
Embedding []float32 `json:"embedding"`
}
type EmbeddingUsage struct {
TotalTokens int `json:"total_tokens"`
}
type TaskStatus string
const (
TaskStatusPending TaskStatus = "PENDING"
TaskStatusRunning TaskStatus = "RUNNING"
TaskStatusSucceeded TaskStatus = "SUCCEEDED"
TaskStatusFailed TaskStatus = "FAILED"
TaskStatusUnknown TaskStatus = "UNKNOWN"
)
func DownloadAndExtract(url string, dest string) error {
if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
return err
}
destFile, err := os.Create(dest)
if err != nil {
return err
}
defer destFile.Close()
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
gzReader, err := gzip.NewReader(resp.Body)
if err != nil {
return err
}
defer gzReader.Close()
_, err = io.Copy(destFile, gzReader)
if err != nil {
return err
}
return nil
}