From 280b4ca00ee0208ba790e4521e96ce2749d745d2 Mon Sep 17 00:00:00 2001 From: dimlio <122263440+dimlio@users.noreply.github.com> Date: Tue, 14 Feb 2023 17:02:58 +0200 Subject: [PATCH 1/5] fix out-of-bound array access Can happen when recording value above highest trackable to non-autoresizing histogram. --- Sources/Histogram/Histogram.swift | 8 ++++++-- Tests/HistogramTests/HistogramTests.swift | 6 ++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Sources/Histogram/Histogram.swift b/Sources/Histogram/Histogram.swift index 1967cc0..c86b008 100644 --- a/Sources/Histogram/Histogram.swift +++ b/Sources/Histogram/Histogram.swift @@ -226,8 +226,12 @@ public struct Histogram { return false } - if index >= counts.count && autoResize { - resize(newHighestTrackableValue: value) + if index >= counts.count { + if autoResize { + resize(newHighestTrackableValue: value) + } else { + return false + } } incrementCountForIndex(index, by: count) diff --git a/Tests/HistogramTests/HistogramTests.swift b/Tests/HistogramTests/HistogramTests.swift index a8604aa..9b2f56d 100644 --- a/Tests/HistogramTests/HistogramTests.swift +++ b/Tests/HistogramTests/HistogramTests.swift @@ -183,6 +183,12 @@ final class HistogramTests: XCTestCase { XCTAssertEqual(1, h.countForValue(Self.value)) XCTAssertEqual(1, h.totalCount) + // try to record value above highest + XCTAssertFalse(h.record(Self.highestTrackableValue * 2)) + + XCTAssertEqual(1, h.countForValue(Self.value)) + XCTAssertEqual(1, h.totalCount) + self.verifyMaxValue(histogram: h) } From de6d5521b21cd359d3db26556adcf2ebd6301403 Mon Sep 17 00:00:00 2001 From: dimlio <122263440+dimlio@users.noreply.github.com> Date: Tue, 14 Feb 2023 18:16:33 +0200 Subject: [PATCH 2/5] add percentileLevelIteratedTo property to IterationValue Differs from 'percentile' field for percentile iteration. --- Sources/Histogram/Histogram.swift | 19 +++++++++++--- .../HistogramDataAccessTests.swift | 25 +++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/Sources/Histogram/Histogram.swift b/Sources/Histogram/Histogram.swift index c86b008..f0b48b4 100644 --- a/Sources/Histogram/Histogram.swift +++ b/Sources/Histogram/Histogram.swift @@ -519,7 +519,7 @@ public struct Histogram { /** * Represents a value point iterated through in a Histogram, with associated stats. */ - public struct IterationValue { + public struct IterationValue: Equatable { /** * The actual value level that was iterated to by the iterator. */ @@ -541,6 +541,15 @@ public struct Histogram { */ public let percentile: Double + /** + * The percentile level that the iterator returning this ``IterationValue`` had iterated to. + * Generally, `percentileLevelIteratedTo` will be equal to or smaller than `percentile`, + * but the same value point can contain multiple iteration levels for some iterators. E.g. a + * percentile iterator can stop multiple times in the exact same value point (if the count at + * that value covers a range of multiple percentiles in the requested percentile iteration points). + */ + public let percentileLevelIteratedTo: Double + /** * The count of recorded values in the histogram that were added to the ``totalCountToThisValue`` as a result * on this iteration step. Since multiple iteration steps may occur with overlapping equivalent value ranges, @@ -604,7 +613,7 @@ public struct Histogram { } } - mutating func makeIterationValueAndUpdatePrev(value: UInt64? = nil) -> IterationValue { + mutating func makeIterationValueAndUpdatePrev(value: UInt64? = nil, percentileIteratedTo: Double? = nil) -> IterationValue { let valueIteratedTo = value ?? self.valueIteratedTo defer { @@ -612,8 +621,10 @@ public struct Histogram { totalCountToPrevIndex = totalCountToCurrentIndex } + let percentile = (100.0 * Double(totalCountToCurrentIndex)) / Double(arrayTotalCount) + return IterationValue(value: valueIteratedTo, prevValue: prevValueIteratedTo, count: countAtThisValue, - percentile: (100.0 * Double(totalCountToCurrentIndex)) / Double(arrayTotalCount), + percentile: percentile, percentileLevelIteratedTo: percentileIteratedTo ?? percentile, countAddedInThisIterationStep: totalCountToCurrentIndex - totalCountToPrevIndex, totalCountToThisValue: totalCountToCurrentIndex, totalValueToThisValue: totalValueToCurrentIndex) } @@ -669,7 +680,7 @@ public struct Histogram { defer { incrementIterationLevel() } - return impl.makeIterationValueAndUpdatePrev() + return impl.makeIterationValueAndUpdatePrev(percentileIteratedTo: percentileLevelToIterateTo) } impl.incrementSubBucket() } diff --git a/Tests/HistogramTests/HistogramDataAccessTests.swift b/Tests/HistogramTests/HistogramDataAccessTests.swift index 8a8d288..2dbcf0d 100644 --- a/Tests/HistogramTests/HistogramDataAccessTests.swift +++ b/Tests/HistogramTests/HistogramDataAccessTests.swift @@ -210,6 +210,31 @@ final class HistogramDataAccessTests: XCTestCase { } } + func testPercentileIterator() { + typealias H = Histogram + var histogram = H(highestTrackableValue: 10_000, numberOfSignificantValueDigits: .three) + + for i in 1...10 { + histogram.record(UInt64(i)) + } + + let expected = [ + H.IterationValue(value: 1, prevValue: 0, count: 1, percentile: 10.0, percentileLevelIteratedTo: 0.0, countAddedInThisIterationStep: 1, totalCountToThisValue: 1, totalValueToThisValue: 1), + H.IterationValue(value: 3, prevValue: 1, count: 1, percentile: 30.0, percentileLevelIteratedTo: 25.0, countAddedInThisIterationStep: 2, totalCountToThisValue: 3, totalValueToThisValue: 6), + H.IterationValue(value: 5, prevValue: 3, count: 1, percentile: 50.0, percentileLevelIteratedTo: 50.0, countAddedInThisIterationStep: 2, totalCountToThisValue: 5, totalValueToThisValue: 15), + H.IterationValue(value: 7, prevValue: 5, count: 1, percentile: 70.0, percentileLevelIteratedTo: 62.5, countAddedInThisIterationStep: 2, totalCountToThisValue: 7, totalValueToThisValue: 28), + H.IterationValue(value: 8, prevValue: 7, count: 1, percentile: 80.0, percentileLevelIteratedTo: 75.0, countAddedInThisIterationStep: 1, totalCountToThisValue: 8, totalValueToThisValue: 36), + H.IterationValue(value: 9, prevValue: 8, count: 1, percentile: 90.0, percentileLevelIteratedTo: 81.25, countAddedInThisIterationStep: 1, totalCountToThisValue: 9, totalValueToThisValue: 45), + H.IterationValue(value: 9, prevValue: 9, count: 1, percentile: 90.0, percentileLevelIteratedTo: 87.5, countAddedInThisIterationStep: 0, totalCountToThisValue: 9, totalValueToThisValue: 45), + H.IterationValue(value: 10, prevValue: 9, count: 1, percentile: 100.0, percentileLevelIteratedTo: 90.625, countAddedInThisIterationStep: 1, totalCountToThisValue: 10, totalValueToThisValue: 55), + H.IterationValue(value: 10, prevValue: 10, count: 1, percentile: 100.0, percentileLevelIteratedTo: 100.0, countAddedInThisIterationStep: 0, totalCountToThisValue: 10, totalValueToThisValue: 55), + ] + + let output = [H.IterationValue](histogram.percentiles(ticksPerHalfDistance: 2)) + + XCTAssertEqual(output, expected) + } + func testLinearBucketValues() { // Note that using linear buckets should work "as expected" as long as the number of linear buckets // is lower than the resolution level determined by largestValueWithSingleUnitResolution From 0ce316bc6e4704bcf35ed033f813c64bcf1cf5fb Mon Sep 17 00:00:00 2001 From: dimlio <122263440+dimlio@users.noreply.github.com> Date: Tue, 14 Feb 2023 18:20:19 +0200 Subject: [PATCH 3/5] drop dependency on TextTable package for printing percentile distribution Use formatting support from Foundation. --- Package.resolved | 9 --- Package.swift | 2 - Sources/Histogram/Histogram.swift | 93 +++++++++++------------ Sources/Histogram/String+Format.swift | 41 ---------- Tests/HistogramTests/HistogramTests.swift | 80 +++++++++++++++++++ 5 files changed, 124 insertions(+), 101 deletions(-) delete mode 100644 Sources/Histogram/String+Format.swift diff --git a/Package.resolved b/Package.resolved index e865e78..6d69d83 100644 --- a/Package.resolved +++ b/Package.resolved @@ -36,15 +36,6 @@ "version" : "1.0.2" } }, - { - "identity" : "texttable", - "kind" : "remoteSourceControl", - "location" : "https://github.com/ordo-one/TextTable", - "state" : { - "revision" : "a27a07300cf4ae322e0079ca0a475c5583dd575f", - "version" : "0.0.2" - } - }, { "identity" : "yams", "kind" : "remoteSourceControl", diff --git a/Package.swift b/Package.swift index 2691416..af81327 100644 --- a/Package.swift +++ b/Package.swift @@ -22,7 +22,6 @@ let package = Package( dependencies: [ // Dependencies declare other packages that this package depends on. .package(url: "https://github.com/apple/swift-numerics", from: "1.0.0"), - .package(url: "https://github.com/ordo-one/TextTable", .upToNextMajor(from: "0.0.1")), .package(url: "https://github.com/apple/swift-docc-plugin", from: "1.1.0"), .package(url: "https://github.com/SwiftPackageIndex/SPIManifest.git", from: "0.12.0"), ], @@ -33,7 +32,6 @@ let package = Package( name: "Histogram", dependencies: [ .product(name: "Numerics", package: "swift-numerics"), - .product(name: "TextTable", package: "TextTable"), ]), .executableTarget( name: "HistogramExample", diff --git a/Sources/Histogram/Histogram.swift b/Sources/Histogram/Histogram.swift index f0b48b4..64612f5 100644 --- a/Sources/Histogram/Histogram.swift +++ b/Sources/Histogram/Histogram.swift @@ -8,8 +8,8 @@ // http://www.apache.org/licenses/LICENSE-2.0 // +import Foundation import Numerics -import TextTable /** * Number of significant digits for values recorded in histogram. @@ -1028,68 +1028,63 @@ public struct Histogram { percentileTicksPerHalfDistance ticks: Int = 5, format: HistogramOutputFormat = .plainText) { - if format == .csv { - return outputPercentileDistributionCsv(to: &stream, outputValueUnitScalingRatio: outputValueUnitScalingRatio, percentileTicksPerHalfDistance: ticks) + func padded(_ s: String, to: Int) -> String { + if s.count < to { + return String(repeating: " ", count: to - s.count) + s + } + return s } - let table = TextTable { - let lastLine = ($0.percentile == 100.0) - - return [ - Column("Value" <- "%.\(self.numberOfSignificantValueDigits.rawValue)f".format(Double($0.value) / outputValueUnitScalingRatio), width: 12, align: .right), - Column("Percentile" <- "%.12f".format($0.percentile / 100.0), width: 14, align: .right), - Column("TotalCount" <- $0.totalCountToThisValue, width: 10, align: .right), - Column("1/(1-Percentile)" <- (lastLine ? "" : "%.2f".format(1.0 / (1.0 - ($0.percentile / 100.0)))), align: .right) - ] + if format == .csv { + stream.write("\"Value\",\"Percentile\",\"TotalCount\",\"1/(1-Percentile)\"\n") + } else { + stream.write("\(padded("Value", to: 12)) \(padded("Percentile", to: 14)) \(padded("TotalCount", to: 10)) \(padded("1/(1-Percentile)", to: 14))\n\n") } - let data = [IterationValue](percentiles(ticksPerHalfDistance: ticks)) - stream.write(table.string(for: data) ?? "unable to render percentile table") - - // Calculate and output mean and std. deviation. - // Note: mean/std. deviation numbers are very often completely irrelevant when - // data is extremely non-normal in distribution (e.g. in cases of strong multi-modal - // response time distribution associated with GC pauses). However, reporting these numbers - // can be very useful for contrasting with the detailed percentile distribution - // reported by outputPercentileDistribution(). It is not at all surprising to find - // percentile distributions where results fall many tens or even hundreds of standard - // deviations away from the mean - such results simply indicate that the data sampled - // exhibits a very non-normal distribution, highlighting situations for which the std. - // deviation metric is a useless indicator. - - let mean = self.mean / outputValueUnitScalingRatio - let stdDeviation = self.stdDeviation / outputValueUnitScalingRatio - - stream.write(("#[Mean = %12.\(numberOfSignificantValueDigits.rawValue)f," + - " StdDeviation = %12.\(numberOfSignificantValueDigits.rawValue)f]\n").format(mean, stdDeviation)) - stream.write(("#[Max = %12.\(numberOfSignificantValueDigits.rawValue)f," + - " Total count = %12d]\n").format(Double(max) / outputValueUnitScalingRatio, totalCount)) - stream.write("#[Buckets = %12d, SubBuckets = %12d]\n".format(bucketCount, subBucketCount)) - } + let percentileFormatString = format == .csv ? + "%.\(numberOfSignificantValueDigits.rawValue)f,%.12f,%d,%.2f\n" : + "%12.\(numberOfSignificantValueDigits.rawValue)f %2.12f %10d %14.2f\n" - private func outputPercentileDistributionCsv( - to stream: inout Stream, - outputValueUnitScalingRatio: Double, - percentileTicksPerHalfDistance ticks: Int = 5) { - stream.write("\"Value\",\"Percentile\",\"TotalCount\",\"1/(1-Percentile)\"\n") - - let percentileFormatString = "%.\(numberOfSignificantValueDigits)f,%.12f,%d,%.2f\n" - let lastLinePercentileFormatString = "%.\(numberOfSignificantValueDigits)f,%.12f,%d,Infinity\n" + let lastLinePercentileFormatString = format == .csv ? + "%.\(numberOfSignificantValueDigits.rawValue)f,%.12f,%d,Infinity\n" : + "%12.\(numberOfSignificantValueDigits.rawValue)f %2.12f %10d\n" for iv in percentiles(ticksPerHalfDistance: ticks) { - if iv.percentile != 100.0 { - stream.write(percentileFormatString.format( + if iv.percentileLevelIteratedTo != 100.0 { + stream.write(String(format: percentileFormatString, Double(iv.value) / outputValueUnitScalingRatio, - iv.percentile / 100.0, + iv.percentileLevelIteratedTo / 100.0, iv.totalCountToThisValue, - 1.0 / (1.0 - (iv.percentile / 100.0)))) + 1.0 / (1.0 - (iv.percentileLevelIteratedTo / 100.0)))) } else { - stream.write(lastLinePercentileFormatString.format( + stream.write(String(format: lastLinePercentileFormatString, Double(iv.value) / outputValueUnitScalingRatio, - iv.percentile / 100.0, + iv.percentileLevelIteratedTo / 100.0, iv.totalCountToThisValue)) } } + + if format != .csv { + // Calculate and output mean and std. deviation. + // Note: mean/std. deviation numbers are very often completely irrelevant when + // data is extremely non-normal in distribution (e.g. in cases of strong multi-modal + // response time distribution associated with GC pauses). However, reporting these numbers + // can be very useful for contrasting with the detailed percentile distribution + // reported by outputPercentileDistribution(). It is not at all surprising to find + // percentile distributions where results fall many tens or even hundreds of standard + // deviations away from the mean - such results simply indicate that the data sampled + // exhibits a very non-normal distribution, highlighting situations for which the std. + // deviation metric is a useless indicator. + // + + let mean = self.mean / outputValueUnitScalingRatio + let stdDeviation = self.stdDeviation / outputValueUnitScalingRatio + stream.write(String(format: "#[Mean = %12.\(numberOfSignificantValueDigits.rawValue)f," + + " StdDeviation = %12.\(numberOfSignificantValueDigits.rawValue)f]\n", mean, stdDeviation)) + stream.write(String(format: "#[Max = %12.\(numberOfSignificantValueDigits.rawValue)f," + + " Total count = %12d]\n", Double(max) / outputValueUnitScalingRatio, totalCount)) + stream.write(String(format: "#[Buckets = %12d, SubBuckets = %12d]\n", bucketCount, subBucketCount)) + } } // MARK: Structure querying support. diff --git a/Sources/Histogram/String+Format.swift b/Sources/Histogram/String+Format.swift deleted file mode 100644 index e5a4249..0000000 --- a/Sources/Histogram/String+Format.swift +++ /dev/null @@ -1,41 +0,0 @@ -// -// Copyright (c) 2023 Ordo One AB. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// - -#if canImport(Darwin) - import Darwin -#elseif canImport(Glibc) - import Glibc -#else - #error("Unsupported Platform") -#endif - -extension String { - /** - * Simple printf-like formatting without Foundation. - * - * FIXME support String arguments - */ - func format(_ arguments: CVarArg...) -> String { - let n = withVaList(arguments) { va_list in - return withCString { cString in - return Int(vsnprintf(nil, 0, cString, va_list)) - } - } - - return withVaList(arguments) { va_list in - return withCString { cString in - // need additional byte for terminating NUL - return String(unsafeUninitializedCapacity: n + 1) { - return Int(vsnprintf($0.baseAddress, n + 1, cString, va_list)) - } - } - } - } -} diff --git a/Tests/HistogramTests/HistogramTests.swift b/Tests/HistogramTests/HistogramTests.swift index 9b2f56d..8e9edf3 100644 --- a/Tests/HistogramTests/HistogramTests.swift +++ b/Tests/HistogramTests/HistogramTests.swift @@ -404,6 +404,86 @@ final class HistogramTests: XCTestCase { verifyMaxValue(histogram: histogram) } + func testOutputPercentileDistributionPlainText() { + var histogram = Histogram(highestTrackableValue: 10_000, numberOfSignificantValueDigits: .three) + + for i in 1...10 { + histogram.record(UInt64(i)) + } + + var output = "" + histogram.outputPercentileDistribution(to: &output, outputValueUnitScalingRatio: 1.0, percentileTicksPerHalfDistance: 5, format: .plainText) + + let expectedOutput = """ + Value Percentile TotalCount 1/(1-Percentile) + + 1.000 0.000000000000 1 1.00 + 1.000 0.100000000000 1 1.11 + 2.000 0.200000000000 2 1.25 + 3.000 0.300000000000 3 1.43 + 4.000 0.400000000000 4 1.67 + 5.000 0.500000000000 5 2.00 + 6.000 0.550000000000 6 2.22 + 6.000 0.600000000000 6 2.50 + 7.000 0.650000000000 7 2.86 + 7.000 0.700000000000 7 3.33 + 8.000 0.750000000000 8 4.00 + 8.000 0.775000000000 8 4.44 + 8.000 0.800000000000 8 5.00 + 9.000 0.825000000000 9 5.71 + 9.000 0.850000000000 9 6.67 + 9.000 0.875000000000 9 8.00 + 9.000 0.887500000000 9 8.89 + 9.000 0.900000000000 9 10.00 + 10.000 0.912500000000 10 11.43 + 10.000 1.000000000000 10 +#[Mean = 5.500, StdDeviation = 2.872] +#[Max = 10.000, Total count = 10] +#[Buckets = 4, SubBuckets = 2048] + +""" + + XCTAssertEqual(output, expectedOutput) + } + + func testOutputPercentileDistributionCsv() { + var histogram = Histogram(highestTrackableValue: 10_000, numberOfSignificantValueDigits: .three) + + for i in 1...10 { + histogram.record(UInt64(i)) + } + + var output = "" + histogram.outputPercentileDistribution(to: &output, outputValueUnitScalingRatio: 1.0, percentileTicksPerHalfDistance: 5, format: .csv) + + let expectedOutput = """ +"Value","Percentile","TotalCount","1/(1-Percentile)" +1.000,0.000000000000,1,1.00 +1.000,0.100000000000,1,1.11 +2.000,0.200000000000,2,1.25 +3.000,0.300000000000,3,1.43 +4.000,0.400000000000,4,1.67 +5.000,0.500000000000,5,2.00 +6.000,0.550000000000,6,2.22 +6.000,0.600000000000,6,2.50 +7.000,0.650000000000,7,2.86 +7.000,0.700000000000,7,3.33 +8.000,0.750000000000,8,4.00 +8.000,0.775000000000,8,4.44 +8.000,0.800000000000,8,5.00 +9.000,0.825000000000,9,5.71 +9.000,0.850000000000,9,6.67 +9.000,0.875000000000,9,8.00 +9.000,0.887500000000,9,8.89 +9.000,0.900000000000,9,10.00 +10.000,0.912500000000,10,11.43 +10.000,1.000000000000,10,Infinity + +""" + + XCTAssertEqual(output, expectedOutput) + } + func verifyMaxValue(histogram h: Histogram) { var computedMaxValue: UInt64 = 0 for i in 0.. Date: Tue, 14 Feb 2023 19:04:57 +0200 Subject: [PATCH 4/5] remove dependency on SwiftPackageIndex/SPIManifest --- Package.resolved | 18 ------------------ Package.swift | 1 - 2 files changed, 19 deletions(-) diff --git a/Package.resolved b/Package.resolved index 6d69d83..4b61605 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,14 +1,5 @@ { "pins" : [ - { - "identity" : "spimanifest", - "kind" : "remoteSourceControl", - "location" : "https://github.com/SwiftPackageIndex/SPIManifest.git", - "state" : { - "revision" : "268fab2006be5c11411994bc76f429d9971a840a", - "version" : "0.15.0" - } - }, { "identity" : "swift-docc-plugin", "kind" : "remoteSourceControl", @@ -35,15 +26,6 @@ "revision" : "0a5bc04095a675662cf24757cc0640aa2204253b", "version" : "1.0.2" } - }, - { - "identity" : "yams", - "kind" : "remoteSourceControl", - "location" : "https://github.com/jpsim/Yams.git", - "state" : { - "revision" : "4889c132978bc6ad3e80f680011ec3dd4fead90c", - "version" : "5.0.4" - } } ], "version" : 2 diff --git a/Package.swift b/Package.swift index af81327..8cac815 100644 --- a/Package.swift +++ b/Package.swift @@ -23,7 +23,6 @@ let package = Package( // Dependencies declare other packages that this package depends on. .package(url: "https://github.com/apple/swift-numerics", from: "1.0.0"), .package(url: "https://github.com/apple/swift-docc-plugin", from: "1.1.0"), - .package(url: "https://github.com/SwiftPackageIndex/SPIManifest.git", from: "0.12.0"), ], targets: [ // Targets are the basic building blocks of a package. A target can define a module or a test suite. From 760aac642911cd5ff05d1fcff6a918275f86a589 Mon Sep 17 00:00:00 2001 From: dimlio <122263440+dimlio@users.noreply.github.com> Date: Wed, 15 Feb 2023 13:32:45 +0200 Subject: [PATCH 5/5] fix/disable SwiftLint warnings --- Sources/Histogram/Histogram.swift | 42 +++---- .../HistogramExample/HistogramExample.swift | 4 +- ...t.swift => HistogramAutosizingTests.swift} | 9 +- .../HistogramDataAccessTests.swift | 111 +++++++++--------- Tests/HistogramTests/HistogramTests.swift | 31 ++--- 5 files changed, 99 insertions(+), 98 deletions(-) rename Tests/HistogramTests/{HistogramAutosizingTest.swift => HistogramAutosizingTests.swift} (92%) diff --git a/Sources/Histogram/Histogram.swift b/Sources/Histogram/Histogram.swift index 64612f5..d66eff7 100644 --- a/Sources/Histogram/Histogram.swift +++ b/Sources/Histogram/Histogram.swift @@ -8,6 +8,8 @@ // http://www.apache.org/licenses/LICENSE-2.0 // +// swiftlint:disable file_length type_body_length line_length identifier_name + import Foundation import Numerics @@ -49,7 +51,6 @@ public enum HistogramOutputFormat { * they are encountered. Note that recording calls that cause auto-resizing may take longer to execute, as resizing * incurs allocation and copying of internal data structures. */ - public struct Histogram { /// The lowest value that can be discerned (distinguished from 0) by the histogram. public let lowestDiscernibleValue: UInt64 @@ -69,17 +70,13 @@ public struct Histogram { // Biggest value that can fit in bucket 0 let subBucketMask: UInt64 - @usableFromInline - var maxValue: UInt64 = 0 + @usableFromInline var maxValue: UInt64 = 0 - @usableFromInline - var minNonZeroValue: UInt64 = .max + @usableFromInline var minNonZeroValue: UInt64 = .max - @usableFromInline - var counts: [Count] + @usableFromInline var counts: [Count] - @usableFromInline - var _totalCount: UInt64 = 0 + @usableFromInline var _totalCount: UInt64 = 0 /// Total count of all recorded values in the histogram public var totalCount: UInt64 { _totalCount } @@ -88,7 +85,7 @@ public struct Histogram { public let numberOfSignificantValueDigits: SignificantDigits /// Control whether or not the histogram can auto-resize and auto-adjust its ``highestTrackableValue``. - public var autoResize: Bool = false + public var autoResize = false let subBucketHalfCountMagnitude: UInt8 @@ -167,7 +164,7 @@ public struct Histogram { // fits in 62 bits is debatable, and it makes it harder to work through the logic. // Sums larger than 64 are totally broken as leadingZeroCountBase would go negative. precondition(unitMagnitude + subBucketHalfCountMagnitude <= 61, - "Invalid arguments: Cannot represent numberOfSignificantValueDigits worth of values beyond lowestDiscernibleValue") + "Invalid arguments: Cannot represent numberOfSignificantValueDigits worth of values beyond lowestDiscernibleValue") // Establish leadingZeroCountBase, used in bucketIndexForValue() fast path: // subtract the bits that would be used by the largest value in bucket 0. @@ -476,7 +473,7 @@ public struct Histogram { * - Returns: The mean value (in value units) of the histogram data. */ public var mean: Double { - if (totalCount == 0) { + if totalCount == 0 { return 0.0 } var totalValue: Double = 0 @@ -492,7 +489,7 @@ public struct Histogram { * - Returns: The standard deviation (in value units) of the histogram data. */ public var stdDeviation: Double { - if (totalCount == 0) { + if totalCount == 0 { return 0.0 } @@ -590,7 +587,7 @@ public struct Histogram { var countAtThisValue: Count = 0 - private var freshSubBucket: Bool = true + private var freshSubBucket = true init(histogram: Histogram) { self.histogram = histogram @@ -623,7 +620,8 @@ public struct Histogram { let percentile = (100.0 * Double(totalCountToCurrentIndex)) / Double(arrayTotalCount) - return IterationValue(value: valueIteratedTo, prevValue: prevValueIteratedTo, count: countAtThisValue, + return IterationValue( + value: valueIteratedTo, prevValue: prevValueIteratedTo, count: countAtThisValue, percentile: percentile, percentileLevelIteratedTo: percentileIteratedTo ?? percentile, countAddedInThisIterationStep: totalCountToCurrentIndex - totalCountToPrevIndex, totalCountToThisValue: totalCountToCurrentIndex, totalValueToThisValue: totalValueToCurrentIndex) @@ -1027,7 +1025,7 @@ public struct Histogram { outputValueUnitScalingRatio: Double, percentileTicksPerHalfDistance ticks: Int = 5, format: HistogramOutputFormat = .plainText) { - + // small helper to pad strings to specific widths, for some reason "%10s"/"%10@" doesn't work in String.init(format:) func padded(_ s: String, to: Int) -> String { if s.count < to { return String(repeating: " ", count: to - s.count) + s @@ -1051,13 +1049,15 @@ public struct Histogram { for iv in percentiles(ticksPerHalfDistance: ticks) { if iv.percentileLevelIteratedTo != 100.0 { - stream.write(String(format: percentileFormatString, + stream.write(String( + format: percentileFormatString, Double(iv.value) / outputValueUnitScalingRatio, iv.percentileLevelIteratedTo / 100.0, iv.totalCountToThisValue, 1.0 / (1.0 - (iv.percentileLevelIteratedTo / 100.0)))) } else { - stream.write(String(format: lastLinePercentileFormatString, + stream.write(String( + format: lastLinePercentileFormatString, Double(iv.value) / outputValueUnitScalingRatio, iv.percentileLevelIteratedTo / 100.0, iv.totalCountToThisValue)) @@ -1267,8 +1267,8 @@ public struct Histogram { private static func bucketsNeededToCoverValue(_ value: UInt64, subBucketCount: Int, unitMagnitude: UInt8) -> Int { var smallestUntrackableValue = UInt64(subBucketCount) << unitMagnitude var bucketsNeeded = 1 - while (smallestUntrackableValue <= value) { - if (smallestUntrackableValue > UInt64.max / 2) { + while smallestUntrackableValue <= value { + if smallestUntrackableValue > UInt64.max / 2 { return bucketsNeeded + 1 } smallestUntrackableValue <<= 1 @@ -1311,6 +1311,7 @@ extension Histogram: Equatable { // resizing. if lhs.counts.count == rhs.counts.count { for i in 0..(numberOfSignificantValueDigits: .three) @@ -74,7 +75,7 @@ final class HistogramAutosizingTests: XCTestCase { //histogram2.add(histogram1) XCTAssert(histogram2.valuesAreEquivalent(histogram2.max, 1_000_000_000), - "Max should be equivalent to 1_000_000_000") + "Max should be equivalent to 1_000_000_000") } func testAutoSizingAcrossContinuousRange() { diff --git a/Tests/HistogramTests/HistogramDataAccessTests.swift b/Tests/HistogramTests/HistogramDataAccessTests.swift index 2dbcf0d..c77d29e 100644 --- a/Tests/HistogramTests/HistogramDataAccessTests.swift +++ b/Tests/HistogramTests/HistogramDataAccessTests.swift @@ -8,24 +8,30 @@ // http://www.apache.org/licenses/LICENSE-2.0 // +// swiftlint:disable file_length identifier_name line_length number_separator trailing_comma + +@testable import Histogram import Numerics import XCTest -@testable import Histogram - +// swiftlint:disable:next type_body_length final class HistogramDataAccessTests: XCTestCase { - static let highestTrackableValue = UInt64(3600) * 1000 * 1000 // e.g. for 1 hr in usec units - static let numberOfSignificantValueDigits = SignificantDigits.three - static let value: UInt64 = 4 + private static let highestTrackableValue = UInt64(3600) * 1000 * 1000 // e.g. for 1 hr in usec units + private static let numberOfSignificantValueDigits = SignificantDigits.three + private static let value: UInt64 = 4 - static var histogram = Histogram(highestTrackableValue: highestTrackableValue, numberOfSignificantValueDigits: numberOfSignificantValueDigits) + private static var histogram = Histogram(highestTrackableValue: highestTrackableValue, numberOfSignificantValueDigits: numberOfSignificantValueDigits) - static var scaledHistogram = Histogram(lowestDiscernibleValue: 1000, highestTrackableValue: highestTrackableValue * 512, + private static var scaledHistogram = Histogram( + lowestDiscernibleValue: 1000, + highestTrackableValue: highestTrackableValue * 512, numberOfSignificantValueDigits: numberOfSignificantValueDigits) - static var rawHistogram = Histogram(highestTrackableValue: highestTrackableValue, numberOfSignificantValueDigits: numberOfSignificantValueDigits) + private static var rawHistogram = Histogram(highestTrackableValue: highestTrackableValue, numberOfSignificantValueDigits: numberOfSignificantValueDigits) - static var scaledRawHistogram = Histogram(lowestDiscernibleValue: 1000, highestTrackableValue: highestTrackableValue * 512, + private static var scaledRawHistogram = Histogram( + lowestDiscernibleValue: 1000, + highestTrackableValue: highestTrackableValue * 512, numberOfSignificantValueDigits: numberOfSignificantValueDigits) override class func setUp() { @@ -69,7 +75,7 @@ final class HistogramDataAccessTests: XCTestCase { func testMean() { let expectedRawMean = ((10_000.0 * 1000) + (1.0 * 100_000_000)) / 10_001 // direct avg. of raw results - let expectedMean = (1000.0 + 50_000_000.0) / 2; // avg. 1 msec for half the time, and 50 sec for other half + let expectedMean = (1000.0 + 50_000_000.0) / 2 // avg. 1 msec for half the time, and 50 sec for other half // We expect to see the mean to be accurate to ~3 decimal points (~0.1%): XCTAssertEqual(expectedRawMean, Self.rawHistogram.mean, accuracy: expectedRawMean * 0.001, "Raw mean is \(expectedRawMean) +/- 0.1%") XCTAssertEqual(expectedMean, Self.histogram.mean, accuracy: expectedMean * 0.001, "Mean is \(expectedMean) +/- 0.1%") @@ -90,9 +96,9 @@ final class HistogramDataAccessTests: XCTestCase { // We expect to see the standard deviations to be accurate to ~3 decimal points (~0.1%): XCTAssertEqual(expectedRawStdDev, Self.rawHistogram.stdDeviation, accuracy: expectedRawStdDev * 0.001, - "Raw standard deviation is \(expectedRawStdDev) +/- 0.1%") + "Raw standard deviation is \(expectedRawStdDev) +/- 0.1%") XCTAssertEqual(expectedStdDev, Self.histogram.stdDeviation, accuracy: expectedStdDev * 0.001, - "Standard deviation is \(expectedStdDev) +/- 0.1%") + "Standard deviation is \(expectedStdDev) +/- 0.1%") } func testMedian() { @@ -133,33 +139,33 @@ final class HistogramDataAccessTests: XCTestCase { func testValueAtPercentile() { XCTAssertEqual(1000.0, Double(Self.rawHistogram.valueAtPercentile(30.0)), - accuracy: 1000.0 * 0.001, "raw 30%'ile is 1 msec +/- 0.1%") + accuracy: 1000.0 * 0.001, "raw 30%'ile is 1 msec +/- 0.1%") XCTAssertEqual(1000.0, Double(Self.rawHistogram.valueAtPercentile(99.0)), - accuracy: 1000.0 * 0.001, "raw 99%'ile is 1 msec +/- 0.1%") + accuracy: 1000.0 * 0.001, "raw 99%'ile is 1 msec +/- 0.1%") XCTAssertEqual(1000.0, Double(Self.rawHistogram.valueAtPercentile(99.99)), - accuracy: 1000.0 * 0.001, "raw 99.99%'ile is 1 msec +/- 0.1%") + accuracy: 1000.0 * 0.001, "raw 99.99%'ile is 1 msec +/- 0.1%") XCTAssertEqual(100_000_000.0, Double(Self.rawHistogram.valueAtPercentile(99.999)), - accuracy: 100_000_000.0 * 0.001, "raw 99.999%'ile is 100 sec +/- 0.1%") + accuracy: 100_000_000.0 * 0.001, "raw 99.999%'ile is 100 sec +/- 0.1%") XCTAssertEqual(100_000_000.0, Double(Self.rawHistogram.valueAtPercentile(100.0)), - accuracy: 100_000_000.0 * 0.001, "raw 100%'ile is 100 sec +/- 0.1%") + accuracy: 100_000_000.0 * 0.001, "raw 100%'ile is 100 sec +/- 0.1%") XCTAssertEqual(1000.0, Double(Self.histogram.valueAtPercentile(30.0)), - accuracy: 1000.0 * 0.001, "30%'ile is 1 msec +/- 0.1%") + accuracy: 1000.0 * 0.001, "30%'ile is 1 msec +/- 0.1%") XCTAssertEqual(1000.0, Double(Self.histogram.valueAtPercentile(50.0)), - accuracy: 1000.0 * 0.001, "50%'ile is 1 msec +/- 0.1%") + accuracy: 1000.0 * 0.001, "50%'ile is 1 msec +/- 0.1%") XCTAssertEqual(50_000_000.0, Double(Self.histogram.valueAtPercentile(75.0)), - accuracy: 50_000_000.0 * 0.001, "75%'ile is 50 sec +/- 0.1%") + accuracy: 50_000_000.0 * 0.001, "75%'ile is 50 sec +/- 0.1%") XCTAssertEqual(80_000_000.0, Double(Self.histogram.valueAtPercentile(90.0)), - accuracy: 80_000_000.0 * 0.001, "90%'ile is 80 sec +/- 0.1%") + accuracy: 80_000_000.0 * 0.001, "90%'ile is 80 sec +/- 0.1%") XCTAssertEqual(98_000_000.0, Double(Self.histogram.valueAtPercentile(99.0)), - accuracy: 98_000_000.0 * 0.001, "99%'ile is 98 sec +/- 0.1%") + accuracy: 98_000_000.0 * 0.001, "99%'ile is 98 sec +/- 0.1%") XCTAssertEqual(100_000_000.0, Double(Self.histogram.valueAtPercentile(99.999)), - accuracy: 100_000_000.0 * 0.001, "99.999%'ile is 100 sec +/- 0.1%") + accuracy: 100_000_000.0 * 0.001, "99.999%'ile is 100 sec +/- 0.1%") XCTAssertEqual(100_000_000.0, Double(Self.histogram.valueAtPercentile(100.0)), - accuracy: 100_000_000.0 * 0.001, "100%'ile is 100 sec +/- 0.1%") + accuracy: 100_000_000.0 * 0.001, "100%'ile is 100 sec +/- 0.1%") } func testValueAtPercentileForLargeHistogram() { @@ -174,36 +180,37 @@ final class HistogramDataAccessTests: XCTestCase { func testPercentileAtOrBelowValue() { XCTAssertEqual(99.99, Self.rawHistogram.percentileAtOrBelowValue(5000), - accuracy: 0.0001, "Raw percentile at or below 5 msec is 99.99% +/- 0.0001") + accuracy: 0.0001, "Raw percentile at or below 5 msec is 99.99% +/- 0.0001") XCTAssertEqual(50.0, Self.histogram.percentileAtOrBelowValue(5000), - accuracy: 0.0001, "Percentile at or below 5 msec is 50% +/- 0.0001%") + accuracy: 0.0001, "Percentile at or below 5 msec is 50% +/- 0.0001%") XCTAssertEqual(100.0, Self.histogram.percentileAtOrBelowValue(100_000_000), - accuracy: 0.0001, "Percentile at or below 100 sec is 100% +/- 0.0001%") + accuracy: 0.0001, "Percentile at or below 100 sec is 100% +/- 0.0001%") } func testCountWithinRange() { XCTAssertEqual(10_000, Self.rawHistogram.count(within: 1000...1000), - "Count of raw values between 1 msec and 1 msec is 1") + "Count of raw values between 1 msec and 1 msec is 1") XCTAssertEqual(1, Self.rawHistogram.count(within: 5000...150_000_000), - "Count of raw values between 5 msec and 150 sec is 1") + "Count of raw values between 5 msec and 150 sec is 1") XCTAssertEqual(10_000, Self.histogram.count(within: 5000...150_000_000), - "Count of values between 5 msec and 150 sec is 10,000") + "Count of values between 5 msec and 150 sec is 10,000") } func testCountForValue() { XCTAssertEqual(0, Self.rawHistogram.count(within: 10_000...10_010), - "Count of raw values at 10 msec is 0") + "Count of raw values at 10 msec is 0") XCTAssertEqual(1, Self.histogram.count(within: 10_000...10_010), - "Count of values at 10 msec is 0") + "Count of values at 10 msec is 0") XCTAssertEqual(10_000, Self.rawHistogram.countForValue(1000), - "Count of raw values at 1 msec is 10,000") + "Count of raw values at 1 msec is 10,000") XCTAssertEqual(10_000, Self.histogram.countForValue(1000), - "Count of values at 1 msec is 10,000") + "Count of values at 1 msec is 10,000") } func testPercentiles() { for iv in Self.histogram.percentiles(ticksPerHalfDistance: 5) { - XCTAssertEqual(iv.value, Self.histogram.highestEquivalentForValue(Self.histogram.valueAtPercentile(iv.percentile)), + XCTAssertEqual( + iv.value, Self.histogram.highestEquivalentForValue(Self.histogram.valueAtPercentile(iv.percentile)), "Iterator value: \(iv.value), count: \(iv.count), percentile: \(iv.percentile)\n" + "histogram valueAtPercentile(\(iv.percentile)): \(Self.histogram.valueAtPercentile(iv.percentile)), " + "highest equivalent value: \(Self.histogram.highestEquivalentForValue(Self.histogram.valueAtPercentile(iv.percentile)))") @@ -211,7 +218,8 @@ final class HistogramDataAccessTests: XCTestCase { } func testPercentileIterator() { - typealias H = Histogram + typealias H = Histogram // swiftlint:disable:this type_name + var histogram = H(highestTrackableValue: 10_000, numberOfSignificantValueDigits: .three) for i in 1...10 { @@ -250,7 +258,7 @@ final class HistogramDataAccessTests: XCTestCase { XCTAssertEqual(10_000, countAddedInThisBucket, "Raw Linear 100 msec bucket # 0 added a count of 10000") } else if index == 999 { XCTAssertEqual(1, countAddedInThisBucket, "Raw Linear 100 msec bucket # 999 added a count of 1") } else { - XCTAssertEqual(0 , countAddedInThisBucket, "Raw Linear 100 msec bucket # \(index) added a count of 0") + XCTAssertEqual(0, countAddedInThisBucket, "Raw Linear 100 msec bucket # \(index) added a count of 0") } index += 1 } @@ -262,10 +270,8 @@ final class HistogramDataAccessTests: XCTestCase { var totalAddedCounts: UInt64 = 0 for iv in Self.histogram.linearBucketValues(valueUnitsPerBucket: 10_000 /* 10 msec */) { let countAddedInThisBucket = iv.countAddedInThisIterationStep - if (index == 0) { - XCTAssertEqual(10_000, countAddedInThisBucket, - "Linear 1 sec bucket # 0 [\(iv.prevValue)..\(iv.value)" + - "] added a count of 10000") + if index == 0 { + XCTAssertEqual(10_000, countAddedInThisBucket, "Linear 1 sec bucket # 0 [\(iv.prevValue)..\(iv.value)] added a count of 10000") } // Because value resolution is low enough (3 digits) that multiple linear buckets will end up // residing in a single value-equivalent range, some linear buckets will have counts of 2 or @@ -284,10 +290,8 @@ final class HistogramDataAccessTests: XCTestCase { totalAddedCounts = 0 for iv in Self.histogram.linearBucketValues(valueUnitsPerBucket: 1000 /* 1 msec */) { let countAddedInThisBucket = iv.countAddedInThisIterationStep - if (index == 1) { - XCTAssertEqual(10_000, countAddedInThisBucket, - "Linear 1 sec bucket # 0 [\(iv.prevValue)..\(iv.value)" + - "] added a count of 10000") + if index == 1 { + XCTAssertEqual(10_000, countAddedInThisBucket, "Linear 1 sec bucket # 0 [\(iv.prevValue)..\(iv.value)] added a count of 10000") } // Because value resolution is low enough (3 digits) that multiple linear buckets will end up // residing in a single value-equivalent range, some linear buckets will have counts of 2 or @@ -330,8 +334,7 @@ final class HistogramDataAccessTests: XCTestCase { let countAddedInThisBucket = iv.countAddedInThisIterationStep if index == 0 { XCTAssertEqual(10_000, countAddedInThisBucket, - "Logarithmic 10 msec bucket # 0 [\(iv.prevValue)..\(iv.value)" + - "] added a count of 10000") + "Logarithmic 10 msec bucket # 0 [\(iv.prevValue)..\(iv.value)] added a count of 10000") } totalAddedCounts += countAddedInThisBucket index += 1 @@ -360,13 +363,12 @@ final class HistogramDataAccessTests: XCTestCase { let countAddedInThisBucket = iv.countAddedInThisIterationStep if index == 0 { XCTAssertEqual(10_000, countAddedInThisBucket, - "Recorded bucket # 0 [\(iv.prevValue)..\(iv.value)" + - "] added a count of 10000") + "Recorded bucket # 0 [\(iv.prevValue)..\(iv.value)] added a count of 10000") } XCTAssertNotEqual(iv.count, 0, "The count in recorded bucket #\(index) is not 0") XCTAssertEqual(iv.count, countAddedInThisBucket, - "The count in recorded bucket # \(index)" + - " is exactly the amount added since the last iteration") + "The count in recorded bucket # \(index)" + + " is exactly the amount added since the last iteration") totalAddedCounts += countAddedInThisBucket index += 1 } @@ -403,12 +405,11 @@ final class HistogramDataAccessTests: XCTestCase { let countAddedInThisBucket = v.countAddedInThisIterationStep if index == 1000 { XCTAssertEqual(10_000, countAddedInThisBucket, - "AllValues bucket # 0 [\(v.prevValue)..\(v.value)" + - "] added a count of 10000") + "AllValues bucket # 0 [\(v.prevValue)..\(v.value)] added a count of 10000") } XCTAssertEqual(v.count, countAddedInThisBucket, - "The count in AllValues bucket # \(index)" + - " is exactly the amount added since the last iteration") + "The count in AllValues bucket # \(index)" + + " is exactly the amount added since the last iteration") totalAddedCounts += countAddedInThisBucket XCTAssertTrue(Self.histogram.valuesAreEquivalent(Self.histogram.valueFromIndex(index), v.value), "valueFromIndex() should be equal to value") index += 1 diff --git a/Tests/HistogramTests/HistogramTests.swift b/Tests/HistogramTests/HistogramTests.swift index 8e9edf3..e5fe5d7 100644 --- a/Tests/HistogramTests/HistogramTests.swift +++ b/Tests/HistogramTests/HistogramTests.swift @@ -8,15 +8,17 @@ // http://www.apache.org/licenses/LICENSE-2.0 // +// swiftlint:disable file_length identifier_name line_length number_separator + +@testable import Histogram import Numerics import XCTest -@testable import Histogram - +// swiftlint:disable:next type_body_length final class HistogramTests: XCTestCase { - static let highestTrackableValue = UInt64(3_600) * 1_000 * 1_000 // e.g. for 1 hr in usec units - static let numberOfSignificantValueDigits = SignificantDigits.three - static let value: UInt64 = 4 + private static let highestTrackableValue = UInt64(3_600) * 1_000 * 1_000 // e.g. for 1 hr in usec units + private static let numberOfSignificantValueDigits = SignificantDigits.three + private static let value: UInt64 = 4 func testCreate() throws { let h = Histogram(lowestDiscernibleValue: 1, highestTrackableValue: 3_600_000_000, numberOfSignificantValueDigits: .three) @@ -134,13 +136,6 @@ final class HistogramTests: XCTestCase { XCTAssertEqual(1024 + 1023, h.subBucketIndexForValue(UInt64(Int64.max), bucketIndex: 1)) } - func testUnitMagnitude52SubBucketMagnitude11Throws() throws { - /* Cannot catch fatal errors. - let h = Histogram(lowestDiscernibleValue: UInt64(1) << 52, highestTrackableValue: UInt64(1) << 62, numberOfSignificantValueDigits: .three) - XCTAssertNil(h) - */ - } - func testUnitMagnitude54SubBucketMagnitude8Ok() throws { let h = Histogram(lowestDiscernibleValue: UInt64(1) << 54, highestTrackableValue: UInt64(1) << 62, numberOfSignificantValueDigits: .two) @@ -300,7 +295,9 @@ final class HistogramTests: XCTestCase { } func testScaledSizeOfEquivalentValueRange() throws { - let histogram = Histogram(lowestDiscernibleValue: 1024, highestTrackableValue: Self.highestTrackableValue, numberOfSignificantValueDigits: Self.numberOfSignificantValueDigits) + let histogram = Histogram(lowestDiscernibleValue: 1024, + highestTrackableValue: Self.highestTrackableValue, + numberOfSignificantValueDigits: Self.numberOfSignificantValueDigits) XCTAssertEqual(1 * 1024, histogram.sizeOfEquivalentRangeForValue(1 * 1024)) XCTAssertEqual(2 * 1024, histogram.sizeOfEquivalentRangeForValue(2500 * 1024)) @@ -484,12 +481,10 @@ final class HistogramTests: XCTestCase { XCTAssertEqual(output, expectedOutput) } - func verifyMaxValue(histogram h: Histogram) { + private func verifyMaxValue(histogram h: Histogram) { var computedMaxValue: UInt64 = 0 - for i in 0.. 0 { - computedMaxValue = h.valueFromIndex(i) - } + for i in 0.. 0 { + computedMaxValue = h.valueFromIndex(i) } computedMaxValue = (computedMaxValue == 0) ? 0 : h.highestEquivalentForValue(computedMaxValue) XCTAssertEqual(computedMaxValue, h.maxValue)