-
Notifications
You must be signed in to change notification settings - Fork 3
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
added: command plugin to download model files. #79
Open
Arpit160399
wants to merge
2
commits into
soto-project:main
Choose a base branch
from
Arpit160399:ModelDownloder-Plugin
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
Show all changes
2 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,69 @@ | ||
// swift-tools-version: 5.9 | ||
import PackageDescription | ||
|
||
let package = Package( | ||
name: "soto-codegenerator", | ||
platforms: [.macOS(.v10_15)], | ||
products: [ | ||
.executable(name: "SotoCodeGenerator", targets: ["SotoCodeGenerator"]), | ||
.plugin(name: "SotoCodeGeneratorPlugin", targets: ["SotoCodeGeneratorPlugin"]), | ||
.plugin(name: "SotoCodeModelDownloaderPlugin",targets: ["SotoCodeModelDownloaderPlugin"]) | ||
], | ||
dependencies: [ | ||
.package(url: "https://github.com/soto-project/soto-smithy.git", from: "0.3.1"), | ||
.package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.0.0"), | ||
.package(url: "https://github.com/hummingbird-project/hummingbird-mustache.git", from: "1.0.3"), | ||
.package(url: "https://github.com/apple/swift-log.git", from: "1.4.0"), | ||
.package(url: "https://github.com/swift-server/async-http-client.git", from: "1.9.0") | ||
], | ||
targets: [ | ||
.executableTarget( | ||
name: "SotoCodeGenerator", | ||
dependencies: [ | ||
.byName(name: "SotoCodeGeneratorLib"), | ||
.product(name: "ArgumentParser", package: "swift-argument-parser"), | ||
.product(name: "Logging", package: "swift-log") | ||
] | ||
), | ||
.executableTarget( | ||
name: "SotoModelDownloader", | ||
dependencies: [ | ||
.product(name: "ArgumentParser", package: "swift-argument-parser"), | ||
.product(name: "AsyncHTTPClient", package: "async-http-client") | ||
] | ||
), | ||
.target( | ||
name: "SotoCodeGeneratorLib", | ||
dependencies: [ | ||
.product(name: "SotoSmithy", package: "soto-smithy"), | ||
.product(name: "SotoSmithyAWS", package: "soto-smithy"), | ||
.product(name: "HummingbirdMustache", package: "hummingbird-mustache"), | ||
.product(name: "Logging", package: "swift-log") | ||
] | ||
), | ||
.plugin( | ||
name: "SotoCodeGeneratorPlugin", | ||
capability: .buildTool(), | ||
dependencies: ["SotoCodeGenerator"] | ||
), | ||
.plugin( | ||
name: "SotoCodeModelDownloaderPlugin", | ||
capability: .command( | ||
intent: .custom( | ||
verb: "get-soto-models", | ||
description: "Download the required Model file schema required for soto code genrator to work"), | ||
permissions: [ | ||
.writeToPackageDirectory(reason: "Write the Model files into target project"), | ||
.allowNetworkConnections( | ||
scope: .all(ports: []), | ||
reason: "The plugin needs to download resource's from remote server" | ||
) | ||
]), | ||
dependencies: ["SotoModelDownloader"] | ||
), | ||
.testTarget( | ||
name: "SotoCodeGeneratorTests", | ||
dependencies: ["SotoCodeGeneratorLib"] | ||
) | ||
] | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,103 @@ | ||
//===----------------------------------------------------------------------===// | ||
// | ||
// This source file is part of the Soto for AWS open source project | ||
// | ||
// Copyright (c) 2017-2024 the Soto project authors | ||
// Licensed under Apache License v2.0 | ||
// | ||
// See LICENSE.txt for license information | ||
// See CONTRIBUTORS.txt for the list of Soto project authors | ||
// | ||
// SPDX-License-Identifier: Apache-2.0 | ||
// | ||
//===----------------------------------------------------------------------===// | ||
import PackagePlugin | ||
import Foundation | ||
|
||
@main | ||
struct SotoCodeModelDownloader: CommandPlugin { | ||
|
||
struct ModelDownloaderArguments { | ||
let inputFolder: String | ||
let outputFolder: String | ||
let configFile: String? | ||
|
||
func getArguments() -> [String] { | ||
// Construct the command line arguments for the model downloader | ||
var arguments = ["--input-folder", inputFolder, | ||
"--output-folder", outputFolder] | ||
if let configFilePath = configFile { | ||
arguments.append(contentsOf: ["--config", configFilePath]) | ||
} | ||
return arguments | ||
} | ||
} | ||
|
||
func performCommand(context: PackagePlugin.PluginContext, arguments: [String]) async throws { | ||
// Get the path to the SotoModelDownloader executable | ||
let sotoModelDownloaderTool = try context.tool(named: "SotoModelDownloader") | ||
let sotoModelDownloaderURL = URL(filePath: sotoModelDownloaderTool.path.string) | ||
|
||
// Find the config file (soto.config.json) in the package targets | ||
let configFile = context.package.targets | ||
.compactMap({ $0 as? SourceModuleTarget }) | ||
.compactMap({ $0.sourceFiles.first(where: { $0.path.lastComponent.contains("soto.config.json") })?.path }) | ||
.first | ||
|
||
// Ensure the config file is found | ||
guard let configFile else { | ||
Diagnostics.error("Cannot find the soto.config.json file in the target") | ||
return | ||
} | ||
|
||
// Determine the main directory and output folder for the downloaded resources | ||
let mainDirectory = configFile.removingLastComponent() | ||
let outputFolderPath = mainDirectory.appending("aws").string | ||
|
||
// Prepare arguments for downloading model files | ||
let modelDownloaderArgs = ModelDownloaderArguments( | ||
inputFolder: Repo.modelDirectory, | ||
outputFolder: outputFolderPath + "/models", | ||
configFile: configFile.string | ||
) | ||
|
||
// Prepare arguments for downloading endpoint files | ||
let endpointDownloaderArgs = ModelDownloaderArguments( | ||
inputFolder: Repo.endpointsDirectory, | ||
outputFolder: outputFolderPath, | ||
configFile: nil | ||
) | ||
|
||
// Iterate over the download tasks (models and endpoints) | ||
for resourceArgs in [endpointDownloaderArgs, modelDownloaderArgs] { | ||
// Set up the process to run the SotoModelDownloader | ||
let process = Process() | ||
process.executableURL = sotoModelDownloaderURL | ||
process.arguments = resourceArgs.getArguments() | ||
|
||
// Run the process and wait for it to complete | ||
try process.run() | ||
process.waitUntilExit() | ||
|
||
// Check the process termination status | ||
if process.terminationReason == .exit && process.terminationStatus == 0 { | ||
print("Downloaded resources to: \(outputFolderPath)") | ||
} else { | ||
let terminationDescription = "\(process.terminationReason):\(process.terminationStatus)" | ||
throw "get-soto-models invocation failed: \(terminationDescription)" | ||
} | ||
} | ||
} | ||
} | ||
|
||
extension String: Error {} | ||
|
||
// MARK: - Model and Endpoint URLs - | ||
|
||
enum Repo { | ||
/// Model files github directory. | ||
static let modelDirectory = "https://github.com/soto-project/soto/tree/main/models" | ||
|
||
/// Endpoints github directory. | ||
static let endpointsDirectory = "https://github.com/soto-project/soto/tree/main/models/endpoints" | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,158 @@ | ||
//===----------------------------------------------------------------------===// | ||
// | ||
// This source file is part of the Soto for AWS open source project | ||
// | ||
// Copyright (c) 2017-2023 the Soto project authors | ||
// Licensed under Apache License v2.0 | ||
// | ||
// See LICENSE.txt for license information | ||
// See CONTRIBUTORS.txt for the list of Soto project authors | ||
// | ||
// SPDX-License-Identifier: Apache-2.0 | ||
// | ||
//===----------------------------------------------------------------------===// | ||
|
||
import Foundation | ||
import AsyncHTTPClient | ||
import NIOHTTP1 | ||
/// Downloads file from GitHub repositories. | ||
struct GitHubResource { | ||
|
||
/// Defines errors specific to GitHub resource operations. | ||
enum GitHubResourceError: Error, CustomDebugStringConvertible { | ||
case invalidURL | ||
case missingURLComponents | ||
case invalidGitHubResponse | ||
|
||
var debugDescription: String { | ||
switch self { | ||
case .invalidURL: | ||
return "The provided URL is invalid." | ||
case .missingURLComponents: | ||
return "The URL is missing necessary components." | ||
case .invalidGitHubResponse: | ||
return "Received an invalid response from GitHub." | ||
} | ||
} | ||
} | ||
|
||
/// The input folder containing the GitHub repository URL. | ||
var inputFolder: String | ||
|
||
/// The output folder where the downloaded files will be saved. | ||
var outputFolder: String | ||
|
||
/// An array of expected services to be downloaded. | ||
var modelFilter: [String] = [] | ||
|
||
/// The base URL for the GitHub API. | ||
static var gitHubApi: URL { URL(string: "https://api.github.com/repos")! } | ||
|
||
/// trigger Downloading files from the GitHub repository. | ||
func download() async throws { | ||
|
||
// Ensure that the input folder contains a valid GitHub repository URL. | ||
guard inputFolder.contains("github.com") else { throw GitHubResourceError.invalidURL } | ||
guard let gitHubURL = URL(string: inputFolder) else { throw GitHubResourceError.invalidURL } | ||
|
||
// Extract user, repository, directory, and reference components from the GitHub URL. | ||
let (user, repository, directory, reference) = try extractGitHubComponents(from: gitHubURL.absoluteString) | ||
|
||
// Fetch the list of files from the GitHub repository. | ||
let files = try await fetchGitHubTree(user: user, repository: repository, directory: directory) | ||
|
||
// Create the output folder if it does not exist. | ||
var isDirectory = ObjCBool(false) | ||
let exists = FileManager.default.fileExists(atPath: outputFolder, isDirectory: &isDirectory) | ||
if !(exists && isDirectory.boolValue) { | ||
try FileManager.default.createDirectory(atPath: outputFolder, withIntermediateDirectories: true) | ||
} | ||
|
||
// Download files concurrently using a task group. | ||
try await withThrowingTaskGroup(of: Void.self) { group in | ||
for filePath in files { | ||
group.addTask { | ||
let escapedPath = filePath.replacingOccurrences(of: "#", with: "%23") | ||
let downloadPath = [user, repository, reference, escapedPath].joined(separator: "/") | ||
if let fileName = escapedPath.components(separatedBy: "/").last { | ||
print("Downloading: \(fileName)") | ||
} | ||
try await downloadFile(at: downloadPath, to: escapedPath) | ||
} | ||
} | ||
try await group.waitForAll() | ||
} | ||
} | ||
|
||
/// Fetches the list of files from the GitHub repository using the GitHub Trees API. | ||
private func fetchGitHubTree(user: String, repository: String, directory: String) async throws -> [String] { | ||
|
||
let directoryName = directory.components(separatedBy: "/").last ?? "" | ||
var requestURLString = GitHubResource.gitHubApi.absoluteString | ||
requestURLString.append("/\(user)/\(repository)/git/trees/HEAD?recursive=1") | ||
|
||
var httpRequest = try HTTPClient.Request(url: requestURLString) | ||
httpRequest.headers.add(name: "User-Agent", value: "AsyncHttpClient") | ||
let response = try await HTTPClient.shared.execute(request: httpRequest).get() | ||
guard response.status == .ok, let data = response.body else { throw GitHubResourceError.invalidGitHubResponse } | ||
|
||
let gitHubDirectoryTree = try JSONDecoder().decode(GitHubDirectoryTree.self, from: data) | ||
|
||
var filePaths = [String]() | ||
for item in gitHubDirectoryTree.tree { | ||
let serviceName = (item.path.components(separatedBy: "/").last?.components(separatedBy: ".").first) ?? "" | ||
if !modelFilter.isEmpty, !modelFilter.contains(serviceName) { | ||
continue | ||
} | ||
// Check for subdirectory | ||
let currentDirectory = item.path.components(separatedBy: "/").dropLast().last ?? "" | ||
if item.type == "blob" && currentDirectory.elementsEqual(directoryName) { | ||
filePaths.append(item.path) | ||
} | ||
} | ||
|
||
return filePaths | ||
} | ||
|
||
/// Extracts user, repository, directory, and reference components from the GitHub repository URL. | ||
private func extractGitHubComponents(from urlString: String) throws -> (user: String, repository: String, directory: String, reference: String) { | ||
guard let url = URL(string: urlString), | ||
url.host == "github.com", | ||
let pathComponents = URLComponents(url: url, resolvingAgainstBaseURL: false)?.path.components(separatedBy: "/").dropFirst(1).map({ $0 }) | ||
else { | ||
throw GitHubResourceError.invalidURL | ||
} | ||
|
||
guard pathComponents.count > 4 else { throw GitHubResourceError.missingURLComponents } | ||
|
||
let user = pathComponents[0] | ||
let repository = pathComponents[1] | ||
let reference = pathComponents[3] | ||
let directory = pathComponents[4...].joined(separator: "/") | ||
|
||
return (user, repository, directory, reference) | ||
} | ||
|
||
/// Downloads a raw file from the GitHub repository. | ||
private func downloadFile(at path: String, to directory: String) async throws { | ||
let client = HTTPClient(eventLoopGroupProvider: .singleton) | ||
|
||
let downloadUrlString = "https://raw.githubusercontent.com/" + path | ||
let downloadRequest = try HTTPClient.Request(url: downloadUrlString) | ||
let fileName = directory.components(separatedBy: "/").last ?? directory | ||
let filePath = outputFolder + "/\(fileName)" | ||
|
||
var downloadStatus: HTTPResponseStatus = .noContent | ||
let delegate = try FileDownloadDelegate(path: filePath) { _, response in | ||
downloadStatus = response.status | ||
} | ||
|
||
_ = try await client.execute(request: downloadRequest, delegate: delegate).get() | ||
|
||
guard downloadStatus == .ok else { throw GitHubResourceError.invalidGitHubResponse } | ||
|
||
try await client.shutdown() | ||
} | ||
} | ||
|
||
|
31 changes: 31 additions & 0 deletions
31
Sources/SotoModelDownloader/Model/ConfigFile + JsonDecode.swift
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
//===----------------------------------------------------------------------===// | ||
// | ||
// This source file is part of the Soto for AWS open source project | ||
// | ||
// Copyright (c) 2017-2023 the Soto project authors | ||
// Licensed under Apache License v2.0 | ||
// | ||
// See LICENSE.txt for license information | ||
// See CONTRIBUTORS.txt for the list of Soto project authors | ||
// | ||
// SPDX-License-Identifier: Apache-2.0 | ||
// | ||
//===----------------------------------------------------------------------===// | ||
|
||
import Foundation | ||
struct ConfigFile: Decodable { | ||
struct ServiceConfig: Decodable { | ||
let operations: [String]? | ||
} | ||
|
||
let services: [String: ServiceConfig]? | ||
} | ||
|
||
extension ConfigFile { | ||
|
||
static func decodeFrom(file configFile: String) throws -> Self { | ||
let data = try Data(contentsOf: URL(fileURLWithPath: configFile)) | ||
let sotoConfig = try JSONDecoder().decode(ConfigFile.self, from: data) | ||
return sotoConfig | ||
} | ||
} |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
In Soto (as with SwiftNIO) we look to support the last three versions of Swift. So for the moment that'd be 5.8, 5.9 and 5.10. Currently with your setup the download plugin is only available for 5.9, not 5.10. I would instead create a
Package.@swift-5.8.swift
which is a copy of the currentPackage.swift
and make this version the defaultPackage.swift
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
So, with the current package setup, if the current Swift version doesn't match any version-specific manifest, the package manager will pick the manifest with the most compatible tools version. This means the package manager will pick
Package.swift
for Swift 5.8 andPackage@swift-5.9.swift
for Swift 5.9 and above, as its tools version will be most compatible with future versions of the package manager. I read this in the documentation here.In terms of readability and maintainability, I agree with your approach of keeping the 5.8 version tag instead of 5.9.