Skip to content

Fix container logs -n line truncation and four performance defects #2023

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
gnavadev wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Fix container logs -n line truncation and four performance defects #2023

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a497aad
Fix O(n2) log tailing, cache compiled globs, and narrow disk-usage lo…
gnavadev Jul 27, 2026
11c23e6
Preserve blank lines in log output, buffer partial lines when following.
gnavadev Jul 27, 2026
6a3f964
Merge branch 'main' into perf-audit-fixes
gnavadev Jul 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 9 additions & 43 deletions Sources/ContainerBuild/BuildFSSync.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,45 +163,21 @@ actor BuildFSSync: BuildPipelineHandler {
sender.yield(response)
}

private struct DirEntry: Hashable {
let url: URL
private struct DirEntry {
let isDirectory: Bool
let relativePath: String

func hash(into hasher: inout Hasher) {
hasher.combine(relativePath)
}

static func == (lhs: DirEntry, rhs: DirEntry) -> Bool {
lhs.relativePath == rhs.relativePath
}
}

/// Packs requested context paths into a tar archive and streams it to the shim.
///
/// This is the primary data path for build-context transfer. BuildKit sends
/// a `Walk` request whose `followpaths` field names the context paths needed
/// for the current build step (e.g. the source of a `COPY` instruction).
/// The host resolves those globs, builds an entry set, and passes it to
/// `Archiver.compress` to produce the tar.
///
/// For any symlink in the entry set whose target lies within the context
/// root, the target is added to the entry set so BuildKit can dereference
/// the symlink during `COPY`/`ADD` processing without a separate request.
/// Symlinks whose targets lie outside the context root are included as
/// symlink entries but their targets are not; BuildKit will resolve them
/// against the shim's local filesystem on Linux, not the macOS host.
///
/// Dockerignore filtering is the shim's responsibility and is applied after
/// the tar is unpacked; this method has no knowledge of `.dockerignore`.
private typealias DirEntries = [String: [String: DirEntry]]

func walk(
_ sender: AsyncStream<ClientStream>.Continuation,
_ packet: BuildTransfer,
_ buildID: String
) async throws {
let wantsTar = packet.mode() == "tar"

var entries: [String: Set<DirEntry>] = [:]
var entries: DirEntries = [:]
let followPaths: [String] = packet.followPaths() ?? []

let followPathsWalked = try walk(root: self.contextDir, includePatterns: followPaths)
Expand All @@ -215,16 +191,14 @@ actor BuildFSSync: BuildPipelineHandler {

let relPath = try url.relativeChildPath(to: contextDir)
let parentPath = try url.deletingLastPathComponent().relativeChildPath(to: contextDir)
let entry = DirEntry(url: url, isDirectory: url.hasDirectoryPath, relativePath: relPath)
entries[parentPath, default: []].insert(entry)
entries[parentPath, default: [:]][relPath] = DirEntry(isDirectory: url.hasDirectoryPath, relativePath: relPath)

if url.isSymlink {
let target: URL = url.resolvingSymlinksInPath()
if self.contextDir.parentOf(target) {
let relPath = try target.relativeChildPath(to: self.contextDir)
let entry = DirEntry(url: target, isDirectory: target.hasDirectoryPath, relativePath: relPath)
let parentPath: String = try target.deletingLastPathComponent().relativeChildPath(to: self.contextDir)
entries[parentPath, default: []].insert(entry)
entries[parentPath, default: [:]][relPath] = DirEntry(isDirectory: target.hasDirectoryPath, relativePath: relPath)
}
}
}
Expand Down Expand Up @@ -280,15 +254,7 @@ actor BuildFSSync: BuildPipelineHandler {
return nil
}

guard let items = entries[parent] else {
return nil
}

let include = items.contains { item in
item.relativePath == rel
}

guard include else {
guard entries[parent]?[rel] != nil else {
return nil
}

Expand Down Expand Up @@ -367,15 +333,15 @@ actor BuildFSSync: BuildPipelineHandler {

private func processDirectory(
_ currentDir: String,
inputEntries: [String: Set<DirEntry>],
inputEntries: DirEntries,
processedPaths: inout [String]
) throws {
guard let entries = inputEntries[currentDir] else {
return
}

// Sort purely by lexicographical order of relativePath
let sortedEntries = entries.sorted { $0.relativePath < $1.relativePath }
let sortedEntries = entries.values.sorted { $0.relativePath < $1.relativePath }

for entry in sortedEntries {
processedPaths.append(entry.relativePath)
Expand Down
16 changes: 15 additions & 1 deletion Sources/ContainerBuild/Globber.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ public class Globber {
let input: URL
var results: Set<URL> = .init()

private var compiledGlobs: [String: NSRegularExpression] = [:]

public init(_ input: URL) {
self.input = input
}
Expand Down Expand Up @@ -131,6 +133,16 @@ public class Globber {
}

func glob(_ input: String, _ pattern: String) throws -> Bool {
let regex = try compiledGlob(for: pattern)
let range = NSRange(input.startIndex..., in: input)
return regex.firstMatch(in: input, options: [], range: range) != nil
}

private func compiledGlob(for pattern: String) throws -> NSRegularExpression {
if let cached = compiledGlobs[pattern] {
return cached
}

let regexPattern =
"^"
+ NSRegularExpression.escapedPattern(for: pattern)
Expand All @@ -142,7 +154,9 @@ public class Globber {

// validate the regex pattern created
let _ = try Regex(regexPattern)
return input.range(of: regexPattern, options: .regularExpression) != nil
let regex = try NSRegularExpression(pattern: regexPattern)
compiledGlobs[pattern] = regex
return regex
}
}

Expand Down
87 changes: 1 addition & 86 deletions Sources/ContainerCommands/Container/ContainerLogs.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,6 @@

import ArgumentParser
import ContainerAPIClient
import ContainerizationError
import Darwin
import Dispatch
import Foundation

extension Application {
Expand Down Expand Up @@ -50,93 +47,11 @@ extension Application {
let fhs = try await client.logs(id: containerId)
let fileHandle = boot ? fhs[1] : fhs[0]

try await Self.tail(
try await LogTailer.tail(
fh: fileHandle,
n: numLines,
follow: follow
)
}

private static func tail(
fh: FileHandle,
n: Int?,
follow: Bool
) async throws {
if let n {
var buffer = Data()
let size = try fh.seekToEnd()
var offset = size
var lines: [String] = []

while offset > 0, lines.count < n {
let readSize = min(1024, offset)
offset -= readSize
try fh.seek(toOffset: offset)

let data = fh.readData(ofLength: Int(readSize))
buffer.insert(contentsOf: data, at: 0)

if let chunk = String(data: buffer, encoding: .utf8) {
lines = chunk.components(separatedBy: .newlines)
lines = lines.filter { !$0.isEmpty }
}
}

lines = Array(lines.suffix(n))
for line in lines {
print(line)
}
} else {
// Fast path if all they want is the full file.
guard let data = try fh.readToEnd() else {
// Seems you get nil if it's a zero byte read, or you
// try and read from dev/null.
return
}
guard let str = String(data: data, encoding: .utf8) else {
throw ContainerizationError(
.internalError,
message: "failed to convert container logs to utf8"
)
}
print(str.trimmingCharacters(in: .newlines))
}

fflush(stdout)
if follow {
setbuf(stdout, nil)
try await Self.followFile(fh: fh)
}
}

private static func followFile(fh: FileHandle) async throws {
_ = try fh.seekToEnd()
let stream = AsyncStream<String> { cont in
fh.readabilityHandler = { handle in
let data = handle.availableData
if data.isEmpty {
// Triggers on container restart - can exit here as well
do {
_ = try fh.seekToEnd() // To continue streaming existing truncated log files
} catch {
fh.readabilityHandler = nil
cont.finish()
return
}
}
if let str = String(data: data, encoding: .utf8), !str.isEmpty {
var lines = str.components(separatedBy: .newlines)
lines = lines.filter { !$0.isEmpty }
for line in lines {
cont.yield(line)
}
}
}
}

for await line in stream {
print(line)
}
}
}
}
68 changes: 42 additions & 26 deletions Sources/ContainerCommands/Container/ContainerStats.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,43 +160,59 @@ extension Application {
}

private static func collectStats(client: ContainerClient, for containers: [ContainerSnapshot]) async throws -> [StatsSnapshot] {
var snapshots: [StatsSnapshot] = []
let running = containers.filter { $0.status == .running }

// First sample
for container in containers {
guard container.status == .running else { continue }
do {
let stats1 = try await client.stats(id: container.id)
snapshots.append(StatsSnapshot(container: container, stats1: stats1, stats2: stats1))
} catch {
// Skip containers that error out
continue
// First sample. Containers that error out are skipped.
let firstSample = await sample(client: client, ids: running.map { $0.id })
var snapshots = running.compactMap { container -> StatsSnapshot? in
guard let stats = firstSample[container.id] else {
return nil
}
return StatsSnapshot(container: container, stats1: stats, stats2: stats)
}

// Wait 2 seconds for CPU delta calculation
if !snapshots.isEmpty {
try await Task.sleep(for: .seconds(2))
guard !snapshots.isEmpty else {
return snapshots
}

// Second sample
for i in 0..<snapshots.count {
do {
let stats2 = try await client.stats(id: snapshots[i].container.id)
snapshots[i] = StatsSnapshot(
container: snapshots[i].container,
stats1: snapshots[i].stats1,
stats2: stats2
)
} catch {
// Keep the original stats if second sample fails
continue
}
// Wait 2 seconds for CPU delta calculation
try await Task.sleep(for: .seconds(2))

// Second sample
let secondSample = await sample(client: client, ids: snapshots.map { $0.container.id })
snapshots = snapshots.map { snapshot -> StatsSnapshot in
guard let stats2 = secondSample[snapshot.container.id] else {
// Keep the original stats if second sample fails
return snapshot
}
return StatsSnapshot(container: snapshot.container, stats1: snapshot.stats1, stats2: stats2)
}

return snapshots
}

/// Fetch stats for every container concurrently, omitting the ones whose
/// stats could not be read. One round trip per container in series adds
/// up quickly once more than a handful of containers are running.
private static func sample(client: ContainerClient, ids: [String]) async -> [String: ContainerResource.ContainerStats] {
await withTaskGroup(
of: (String, ContainerResource.ContainerStats?).self,
returning: [String: ContainerResource.ContainerStats].self
) { group in
for id in ids {
group.addTask {
(id, try? await client.stats(id: id))
}
}

var results = [String: ContainerResource.ContainerStats]()
for await (id, stats) in group {
results[id] = stats
}
return results
}
}

/// Calculate CPU percentage from two stat snapshots
/// - Parameters:
/// - cpuUsageUsec1: CPU usage in microseconds from first sample
Expand Down
Loading