Skip to content

update vertexai preview to main #402

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Nov 1, 2024
Merged

update vertexai preview to main #402

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e71e5a0
update vertexai preview to main
morganchen12 Oct 11, 2024
70d721b
actually update snippets
morganchen12 Oct 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 18 additions & 6 deletions vertexai/VertexAISnippets.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,9 @@
8D40F4122BD1CDC40020872A /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 8D40F4112BD1CDC40020872A /* Assets.xcassets */; };
8D40F4162BD1CDC40020872A /* Preview Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 8D40F4152BD1CDC40020872A /* Preview Assets.xcassets */; };
8D40F4262BD1CE1A0020872A /* FirebaseAppCheck in Frameworks */ = {isa = PBXBuildFile; productRef = 8D40F4252BD1CE1A0020872A /* FirebaseAppCheck */; };
8D40F43B2BD1CE3E0020872A /* FirebaseVertexAI-Preview in Frameworks */ = {isa = PBXBuildFile; productRef = 8D40F43A2BD1CE3E0020872A /* FirebaseVertexAI-Preview */; };
8D40F43D2BD1CE910020872A /* VertexAISnippets.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8D40F43C2BD1CE910020872A /* VertexAISnippets.swift */; };
8D7B83012CD4127C0024A604 /* FirebaseAuth in Frameworks */ = {isa = PBXBuildFile; productRef = 8D7B83002CD4127C0024A604 /* FirebaseAuth */; };
8D7B83032CD4127C0024A604 /* FirebaseVertexAI in Frameworks */ = {isa = PBXBuildFile; productRef = 8D7B83022CD4127C0024A604 /* FirebaseVertexAI */; };
/* End PBXBuildFile section */

/* Begin PBXFileReference section */
Expand All @@ -31,7 +32,8 @@
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
8D40F43B2BD1CE3E0020872A /* FirebaseVertexAI-Preview in Frameworks */,
8D7B83032CD4127C0024A604 /* FirebaseVertexAI in Frameworks */,
8D7B83012CD4127C0024A604 /* FirebaseAuth in Frameworks */,
8D40F4262BD1CE1A0020872A /* FirebaseAppCheck in Frameworks */,
);
runOnlyForDeploymentPostprocessing = 0;
Expand Down Expand Up @@ -102,7 +104,8 @@
name = VertexAISnippets;
packageProductDependencies = (
8D40F4252BD1CE1A0020872A /* FirebaseAppCheck */,
8D40F43A2BD1CE3E0020872A /* FirebaseVertexAI-Preview */,
8D7B83002CD4127C0024A604 /* FirebaseAuth */,
8D7B83022CD4127C0024A604 /* FirebaseVertexAI */,
);
productName = VertexAISnippets;
productReference = 8D40F40A2BD1CDC30020872A /* VertexAISnippets.app */;
Expand All @@ -116,7 +119,7 @@
attributes = {
BuildIndependentTargetsInParallel = 1;
LastSwiftUpdateCheck = 1520;
LastUpgradeCheck = 1520;
LastUpgradeCheck = 1530;
TargetAttributes = {
8D40F4092BD1CDC30020872A = {
CreatedOnToolsVersion = 15.2;
Expand Down Expand Up @@ -204,6 +207,7 @@
CLANG_WARN_UNREACHABLE_CODE = YES;
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
COPY_PHASE_STRIP = NO;
DEAD_CODE_STRIPPING = YES;
DEBUG_INFORMATION_FORMAT = dwarf;
ENABLE_STRICT_OBJC_MSGSEND = YES;
ENABLE_TESTABILITY = YES;
Expand Down Expand Up @@ -265,6 +269,7 @@
CLANG_WARN_UNREACHABLE_CODE = YES;
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
COPY_PHASE_STRIP = NO;
DEAD_CODE_STRIPPING = YES;
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
ENABLE_NS_ASSERTIONS = NO;
ENABLE_STRICT_OBJC_MSGSEND = YES;
Expand Down Expand Up @@ -292,6 +297,7 @@
CODE_SIGN_ENTITLEMENTS = VertexAISnippets/VertexAISnippets.entitlements;
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
DEAD_CODE_STRIPPING = YES;
DEVELOPMENT_ASSET_PATHS = "\"VertexAISnippets/Preview Content\"";
ENABLE_PREVIEWS = YES;
GENERATE_INFOPLIST_FILE = YES;
Expand Down Expand Up @@ -328,6 +334,7 @@
CODE_SIGN_ENTITLEMENTS = VertexAISnippets/VertexAISnippets.entitlements;
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
DEAD_CODE_STRIPPING = YES;
DEVELOPMENT_ASSET_PATHS = "\"VertexAISnippets/Preview Content\"";
ENABLE_PREVIEWS = YES;
GENERATE_INFOPLIST_FILE = YES;
Expand Down Expand Up @@ -396,10 +403,15 @@
package = 8D40F41C2BD1CE1A0020872A /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */;
productName = FirebaseAppCheck;
};
8D40F43A2BD1CE3E0020872A /* FirebaseVertexAI-Preview */ = {
8D7B83002CD4127C0024A604 /* FirebaseAuth */ = {
isa = XCSwiftPackageProductDependency;
package = 8D40F41C2BD1CE1A0020872A /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */;
productName = "FirebaseVertexAI-Preview";
productName = FirebaseAuth;
};
8D7B83022CD4127C0024A604 /* FirebaseVertexAI */ = {
isa = XCSwiftPackageProductDependency;
package = 8D40F41C2BD1CE1A0020872A /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */;
productName = FirebaseVertexAI;
};
/* End XCSwiftPackageProductDependency section */
};
Expand Down
51 changes: 22 additions & 29 deletions vertexai/VertexAISnippets/VertexAISnippets.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import AppKit

// [START import_vertexai]
import FirebaseVertexAI
import FirebaseCore
// [END import_vertexai]

class Snippets {
Expand Down Expand Up @@ -91,7 +92,7 @@ class Snippets {
let prompt = "Write a story about a magic backpack."

// To stream generated text output, call generateContentStream with the text input
let contentStream = model.generateContentStream(prompt)
let contentStream = try model.generateContentStream(prompt)
for try await chunk in contentStream {
if let text = chunk.text {
print(text)
Expand Down Expand Up @@ -125,7 +126,7 @@ class Snippets {
let prompt = "What's in this picture?"

// To stream generated text output, call generateContentStream and pass in the prompt
let contentStream = model.generateContentStream(image, prompt)
let contentStream = try model.generateContentStream(image, prompt)
for try await chunk in contentStream {
if let text = chunk.text {
print(text)
Expand Down Expand Up @@ -167,7 +168,7 @@ class Snippets {
let prompt = "What's different between these pictures?"

// To stream generated text output, call generateContentStream and pass in the prompt
let contentStream = model.generateContentStream(image1, image2, prompt)
let contentStream = try model.generateContentStream(image1, image2, prompt)
for try await chunk in contentStream {
if let text = chunk.text {
print(text)
Expand Down Expand Up @@ -203,7 +204,7 @@ class Snippets {
withExtension: "mp4") else { fatalError() }
let video = try Data(contentsOf: fileURL)
let prompt = "What's in this video?"
let videoContent = ModelContent.Part.data(mimetype: "video/mp4", video)
let videoContent = InlineDataPart(data: video, mimeType: "video/mp4")

// To generate text output, call generateContent and pass in the prompt
let response = try await model.generateContent(videoContent, prompt)
Expand All @@ -219,10 +220,10 @@ class Snippets {
withExtension: "mp4") else { fatalError() }
let video = try Data(contentsOf: fileURL)
let prompt = "What's in this video?"
let videoContent = ModelContent.Part.data(mimetype: "video/mp4", video)
let videoContent = InlineDataPart(data: video, mimeType: "video/mp4")

// To stream generated text output, call generateContentStream and pass in the prompt
let contentStream = model.generateContentStream(videoContent, prompt)
let contentStream = try model.generateContentStream(videoContent, prompt)
for try await chunk in contentStream {
if let text = chunk.text {
print(text)
Expand All @@ -243,7 +244,7 @@ class Snippets {
let chat = model.startChat(history: history)

// To stream generated text output, call sendMessageStream and pass in the message
let contentStream = chat.sendMessageStream("How many paws are in my house?")
let contentStream = try chat.sendMessageStream("How many paws are in my house?")
for try await chunk in contentStream {
if let text = chunk.text {
print(text)
Expand Down Expand Up @@ -275,7 +276,7 @@ class Snippets {
// [START count_tokens_text]
let response = try await model.countTokens("Why is the sky blue?")
print("Total Tokens: \(response.totalTokens)")
print("Total Billable Characters: \(response.totalBillableCharacters)")
print("Total Billable Characters: \(response.totalBillableCharacters ?? 0)")
// [END count_tokens_text]
}

Expand All @@ -288,7 +289,7 @@ class Snippets {
// [START count_tokens_text_image]
let response = try await model.countTokens(image, "What's in this picture?")
print("Total Tokens: \(response.totalTokens)")
print("Total Billable Characters: \(response.totalBillableCharacters)")
print("Total Billable Characters: \(response.totalBillableCharacters ?? 0)")
// [END count_tokens_text_image]
}

Expand All @@ -303,19 +304,19 @@ class Snippets {
// [START count_tokens_multi_image]
let response = try await model.countTokens(image1, image2, "What's in this picture?")
print("Total Tokens: \(response.totalTokens)")
print("Total Billable Characters: \(response.totalBillableCharacters)")
print("Total Billable Characters: \(response.totalBillableCharacters ?? 0)")
// [END count_tokens_multi_image]
}

func countTokensChat() async throws {
// [START count_tokens_chat]
let chat = model.startChat()
let history = chat.history
let message = try ModelContent(role: "user", "Why is the sky blue?")
let message = ModelContent(role: "user", parts: "Why is the sky blue?")
let contents = history + [message]
let response = try await model.countTokens(contents)
print("Total Tokens: \(response.totalTokens)")
print("Total Billable Characters: \(response.totalBillableCharacters)")
print("Total Billable Characters: \(response.totalBillableCharacters ?? 0)")
// [END count_tokens_chat]
}

Expand Down Expand Up @@ -361,16 +362,13 @@ class Snippets {
name: "getExchangeRate",
description: "Get the exchange rate for currencies between countries",
parameters: [
"currencyFrom": Schema(
type: .string,
"currencyFrom": Schema.string(
description: "The currency to convert from."
),
"currencyTo": Schema(
type: .string,
"currencyTo": Schema.string(
description: "The currency to convert to."
),
],
requiredParameters: ["currencyFrom", "currencyTo"]
]
)
// [END create_function_metadata]

Expand All @@ -383,7 +381,7 @@ class Snippets {
let model = vertex.generativeModel(
modelName: "gemini-1.5-flash",
// Specify the function declaration.
tools: [Tool(functionDeclarations: [getExchangeRate])]
tools: [Tool.functionDeclarations([getExchangeRate])]
)
// [END initialize_model_function]

Expand Down Expand Up @@ -418,10 +416,7 @@ class Snippets {
// displayed to the user.
let response = try await chat.sendMessage([ModelContent(
role: "function",
parts: [.functionResponse(FunctionResponse(
name: functionCall.name,
response: apiResponse
))]
parts: [FunctionResponsePart(name: functionCall.name, response: apiResponse)]
)])

// Log the text response.
Expand All @@ -436,20 +431,18 @@ class Snippets {
let getExchangeRate = FunctionDeclaration(
name: "getExchangeRate",
description: "Get the exchange rate for currencies between countries",
parameters: nil,
requiredParameters: nil
parameters: [:]
)

// [START function_modes]
let model = VertexAI.vertexAI().generativeModel(
// Setting a function calling mode is only available in Gemini 1.5 Pro
modelName: "gemini-1.5-pro",
// Pass the function declaration
tools: [Tool(functionDeclarations: [getExchangeRate])],
tools: [Tool.functionDeclarations([getExchangeRate])],
toolConfig: ToolConfig(
functionCallingConfig: FunctionCallingConfig(
// Only call functions (model won't generate text)
mode: FunctionCallingConfig.Mode.any,
// Only call functions (model won't generate text)
functionCallingConfig: FunctionCallingConfig.any(
// This should only be set when the Mode is .any.
allowedFunctionNames: ["getExchangeRate"]
)
Expand Down