forked from GoogleCloudPlatform/golang-samples
-
Notifications
You must be signed in to change notification settings - Fork 0
/
multimodal-video.go
115 lines (93 loc) · 3.14 KB
/
multimodal-video.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// multimodal video shows generation of content with video and text input
package main
// [START aiplatform_gemini_single_turn_video]
import (
"context"
"fmt"
"io"
"log"
"os"
"path"
"strings"
"time"
"cloud.google.com/go/storage"
"cloud.google.com/go/vertexai/genai"
)
func main() {
projectID := os.Getenv("GOOGLE_CLOUD_PROJECT")
location := "us-central1"
modelName := "gemini-pro-vision"
if projectID == "" {
log.Fatal("require environment variable GOOGLE_CLOUD_PROJECT")
}
ctx := context.Background()
// create video part
video, err := partFromGCSURI(ctx, "gs://cloud-samples-data/video/animals.mp4")
if err != nil {
log.Fatalf("unable to process media: %v", err)
}
// assemble a multimodal (multipart) prompt
prompt := []genai.Part{
genai.Text("What is in the video? "),
video,
}
// generate the response
err = generateMultimodalContent(ctx, os.Stdout, prompt, projectID, location, modelName)
if err != nil {
log.Fatalf("unable to generate: %v", err)
}
}
// generateMultimodalContent provide a generated response using multimodal input
func generateMultimodalContent(ctx context.Context, w io.Writer, parts []genai.Part, projectID, location, modelName string) error {
client, err := genai.NewClient(ctx, projectID, location)
if err != nil {
return fmt.Errorf("unable to create client: %v", err)
}
defer client.Close()
model := client.GenerativeModel(modelName)
model.Temperature = 0.6
res, err := model.GenerateContent(ctx, parts...)
if err != nil {
return fmt.Errorf("unable to generate contents: %v", err)
}
_, err = fmt.Fprintf(w, "generated response: %s\n", res.Candidates[0].Content.Parts[0])
return err
}
// partFromGCSURI create a multimodal prompt part from a Google Cloud Storage object URI
func partFromGCSURI(ctx context.Context, gcsPath string) (genai.Part, error) {
gcsPath = strings.Replace(gcsPath, "gs://", "", -1)
bucket := strings.SplitN(gcsPath, "/", 3)[0]
object := strings.Join(strings.SplitN(gcsPath, "/", 3)[1:], "/")
client, err := storage.NewClient(ctx)
if err != nil {
return nil, fmt.Errorf("storage.NewClient: %w", err)
}
defer client.Close()
ctx, cancel := context.WithTimeout(ctx, time.Second*50)
defer cancel()
rc, err := client.Bucket(bucket).Object(object).NewReader(ctx)
if err != nil {
return nil, fmt.Errorf("Object(%q).NewReader: %w", object, err)
}
defer rc.Close()
data, err := io.ReadAll(rc)
if err != nil {
return nil, fmt.Errorf("ioutil.ReadAll: %w", err)
}
ext := path.Ext(object)
return genai.Blob{MIMEType: "video/" + ext, Data: data}, nil
}
// [END aiplatform_gemini_single_turn_video]