-
Notifications
You must be signed in to change notification settings - Fork 13
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
benchmarking infrastructure #61
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,7 +3,6 @@ | |
/Packages | ||
/*.xcodeproj | ||
xcuserdata/ | ||
/Package.resolved | ||
/.swiftpm | ||
/.vscode | ||
/.idea |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
{ | ||
"object": { | ||
"pins": [ | ||
{ | ||
"package": "swift-argument-parser", | ||
"repositoryURL": "https://github.com/apple/swift-argument-parser", | ||
"state": { | ||
"branch": null, | ||
"revision": "223d62adc52d51669ae2ee19bdb8b7d9fd6fcd9c", | ||
"version": "0.0.6" | ||
} | ||
}, | ||
{ | ||
"package": "Benchmark", | ||
"repositoryURL": "https://github.com/google/swift-benchmark.git", | ||
"state": { | ||
"branch": "master", | ||
"revision": "a952f1d7deed2368805ac29f09da6a676dde8015", | ||
"version": null | ||
} | ||
} | ||
] | ||
}, | ||
"version": 1 | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,70 @@ | ||
// Copyright 2020 The SwiftFusion Authors. All Rights Reserved. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
/// Benchmarks Pose2SLAM solutions. | ||
|
||
import Benchmark | ||
import SwiftFusion | ||
|
||
let pose2SLAM = BenchmarkSuite(name: "Pose2SLAM") { suite in | ||
|
||
let intelDataset = try! G2OFactorGraph(fromG2O: try! cachedDataset("input_INTEL_g2o.txt")) | ||
check(intelDataset.graph.error(intelDataset.initialGuess), near: 73565.64, accuracy: 1e-2) | ||
|
||
// Uses `NonlinearFactorGraph` on the Intel dataset. | ||
// The solvers are configured to run for a constant number of steps. | ||
// The nonlinear solver is 5 iterations of Gauss-Newton. | ||
// The linear solver is 100 iterations of CGLS. | ||
suite.benchmark( | ||
"NonlinearFactorGraph, Intel, 5 Gauss-Newton steps, 100 CGLS steps", | ||
settings: .iterations(1) | ||
) { | ||
var val = intelDataset.initialGuess | ||
for _ in 0..<5 { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Usually, it's better to avoid loops in benchmarks. Instead, you can put just the loop body as a benchmark, and then customize number of iterations with There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This loop is part of the solver logic that we are benchmarking, so I think it makes sense to have it in the benchmark. Eventually we're going to have an API for the solver and the benchmark body will look like There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Also in the final solver this should be a |
||
let gfg = intelDataset.graph.linearize(val) | ||
let optimizer = CGLS(precision: 0, max_iteration: 100) | ||
var dx = VectorValues() | ||
for i in 0..<val.count { | ||
dx.insert(i, Vector(zeros: 3)) | ||
} | ||
optimizer.optimize(gfg: gfg, initial: &dx) | ||
val.move(along: dx) | ||
} | ||
check(intelDataset.graph.error(val), near: 35.59, accuracy: 1e-2) | ||
} | ||
} | ||
|
||
/// Builds an initial guess and a factor graph from a g2o file. | ||
struct G2OFactorGraph: G2OReader { | ||
/// The initial guess. | ||
var initialGuess: Values = Values() | ||
|
||
/// The factor graph representing the measurements. | ||
var graph: NonlinearFactorGraph = NonlinearFactorGraph() | ||
|
||
public mutating func addInitialGuess(index: Int, pose: Pose2) { | ||
initialGuess.insert(index, pose) | ||
} | ||
|
||
public mutating func addMeasurement(frameIndex: Int, measuredIndex: Int, pose: Pose2) { | ||
graph += BetweenFactor(frameIndex, measuredIndex, pose) | ||
} | ||
} | ||
|
||
func check(_ actual: Double, near expected: Double, accuracy: Double) { | ||
if abs(actual - expected) > accuracy { | ||
print("ERROR: \(actual) != \(expected) (accuracy \(accuracy))") | ||
fatalError() | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
// Copyright 2020 The SwiftFusion Authors. All Rights Reserved. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
import Benchmark | ||
|
||
Benchmark.main([ | ||
pose2SLAM | ||
]) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,70 @@ | ||
// Copyright 2020 The SwiftFusion Authors. All Rights Reserved. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
import Foundation | ||
#if !os(macOS) | ||
import FoundationNetworking | ||
#endif | ||
|
||
/// Dictionary from dataset id to dataset url. | ||
fileprivate let datasets = [ | ||
"input_INTEL_g2o.txt": | ||
"https://github.com/pkchen1129/Mobile-Robotics/raw/8551df10ba8af36801403daeba710c1f9c9e54cd/ps4/code/dataset/input_INTEL_g2o.txt" | ||
] | ||
|
||
/// Returns a dataset cached on the local system. | ||
/// | ||
/// If the dataset with `id` is available on the local system, returns it immediately. | ||
/// Otherwise, downloads the dataset to the cache and then returns it. | ||
/// | ||
/// - Parameter `cacheDirectory`: The directory where the cached datasets are stored. | ||
public func cachedDataset( | ||
_ id: String, | ||
cacheDirectory: URL = URL(fileURLWithPath: NSHomeDirectory()) | ||
.appendingPathComponent(".SwiftFusionDatasetCache", isDirectory: true) | ||
) throws -> URL { | ||
try createDirectoryIfMissing(at: cacheDirectory.path) | ||
let cacheEntry = cacheDirectory.appendingPathComponent(id) | ||
if FileManager.default.fileExists(atPath: cacheEntry.path) { | ||
return cacheEntry | ||
} | ||
guard let url = datasets[id] else { | ||
throw DatasetCacheError(message: "No such dataset: \(id)") | ||
} | ||
print("Downloading \(url) to \(cacheEntry)") | ||
guard let source = URL(string: url) else { | ||
throw DatasetCacheError(message: "Could not parse URL: \(url)") | ||
} | ||
let data = try Data.init(contentsOf: source) | ||
try data.write(to: cacheEntry) | ||
print("Downloaded \(cacheEntry)!") | ||
return cacheEntry | ||
} | ||
|
||
/// An error from getting a cached dataset. | ||
public struct DatasetCacheError: Swift.Error { | ||
public let message: String | ||
} | ||
|
||
/// Creates a directory at a path, if missing. If the directory exists, this does nothing. | ||
/// | ||
/// - Parameters: | ||
/// - path: The path of the desired directory. | ||
fileprivate func createDirectoryIfMissing(at path: String) throws { | ||
guard !FileManager.default.fileExists(atPath: path) else { return } | ||
try FileManager.default.createDirectory( | ||
atPath: path, | ||
withIntermediateDirectories: true, | ||
attributes: nil) | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Results are going to be quite noisy if you only run a single iteration. Is it not practical to run it for longer?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is benchmarking a whole solver, and it's pretty slow right now (~12 seconds).
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I see, thanks for the clarification.