benchmark

#!/usr/bin/env python

import sys
import os
import timeit
import shutil

# Benchmark Parameters

methodsPerExtension = 5
extensionsPerFile = 4
buildAttemptsToAverage = 2

libraryCountsToTest = [0,1,2,4,8]

# All type counts should be evenenly divisible by each library count
typeCountsToTest = [
    128,
    256,
    1024,
    2048,
    4096,
]

# Run benchmarks

buildCommand = "xcodebuild -project Test.xcodeproj -scheme Test"

owd = os.getcwd()

def writeFile(name, libName, withExtensions, extraMethod):
    f = open(f"test-package/Sources/{libName}/{name}.swift", "w")
    f.write(f"public class {name} {{\n")
    f.write(f"    public init() {{}}\n")
    f.write(f"    public func main() {{\n")
    for i in range(extensionsPerFile * methodsPerExtension):
        f.write(f"        method{i + 1}()\n")
    if extraMethod:
        f.write(f"        extraMethod()\n")

    f.write(f"    }}\n")

    for i in range(extensionsPerFile):
        if withExtensions:
            f.write(f"}}\n")
            f.write(f"extension {name} {{\n")

        for j in range(methodsPerExtension):
            methodNumber = (i * methodsPerExtension) + j + 1
            f.write("\n")
            f.write(f"    func method{methodNumber}() {{\n")
            f.write(f"        for i in 0 ..< {methodNumber} {{ print(i + 2) }}; print(\"finished\")\n")
            f.write(f"    }}\n")

    if extraMethod:
        f.write("\n")
        f.write(f"    func extraMethod() {{\n")
        f.write(f"        for i in 0 ..< {methodNumber} {{ print(i + 2) }}; print(\"finished\")\n")
        f.write(f"    }}\n")

    f.write(f"}}\n")

    f.close()

def generatePackage(libCount):
    if os.path.exists("test-package"):
        shutil.rmtree("test-package")
    os.mkdir("test-package")

    package = open("test-package/Package.swift", "w")
    package.write(
"""// swift-tools-version:5.2
// The swift-tools-version declares the minimum version of Swift required to build this package.

import PackageDescription

let package = Package(
    name: "Test",
    dependencies: [],
    targets: [
"""
    )

    for i in range(libCount):
        package.write(f"        .target(name: \"Lib{i + 1}\", dependencies: []),\n")

    package.write(f"        .target(name: \"Test\", dependencies: [\n")

    for i in range(libCount):
        package.write(f"            \"Lib{i + 1}\",\n")

    package.write("""
        ]),
    ]
)
        """)

def makeChange(typeCount, libCount, withExtensions):
    path = ""
    if libCount == 0:
        writeFile("Class1", "Test", withExtensions, True)
    else:
        writeFile("Class1", "Lib1", withExtensions, True)

def generate(typeCount, libCount, withExtensions):
    generatePackage(libCount)

    os.mkdir("test-package/Sources")
    os.mkdir("test-package/Sources/Test")
    main = open(f"test-package/Sources/Test/main.swift", "w")

    if libCount == 0:
        for t in range(typeCount):
            writeFile(f"Class{t + 1}", f"Test", withExtensions, False)
            main.write(f"Class{t + 1}().main()\n")
    else:
        for l in range(libCount):
            os.mkdir(f"test-package/Sources/Lib{l + 1}")
            main.write(f"import Lib{l + 1}\n")


        typesPerLib = int(typeCount / libCount)
        for l in range(libCount):
            for t in range(typesPerLib):
                typeNum = l * typesPerLib + t  + 1
                writeFile(f"Class{typeNum}", f"Lib{l + 1}", withExtensions, False)
                main.write(f"Class{typeNum}().main()\n")

    main.close()

def timeRebuild(typeCount, libCount, withExtensions):
    print("Building after adding method")
    makeChange(typeCount, libCount, withExtensions)
    os.chdir("test-package")
    start = timeit.default_timer()
    os.system(f"{buildCommand} >/dev/null")
    end = timeit.default_timer()
    os.chdir(owd)
    print(end - start)
    print("-----------------------------------------")
    return end - start


def time(attempt, typeCount, libCount, withExtensions):
    extensionStrategy = "using extensions" if withExtensions else "without extensions"
    print(f"Build Attempt {attempt}: {typeCount} types in {libCount} libs {extensionStrategy}...")
    print("-----------------------------------------")

    generate(typeCount, libCount, withExtensions)

    os.chdir("test-package")
    os.system("swift package generate-xcodeproj >/dev/null")
    os.system(f"{buildCommand} clean >/dev/null")

    start = timeit.default_timer()
    os.system(f"{buildCommand} >/dev/null")
    end = timeit.default_timer()

    os.chdir(owd)

    print(end - start)
    return end - start

def benchmark(typeCount):
    print("\n\n")
    print(f"Benchmarking {typeCount} types")
    print("=============================================")

    withExtensionsSums = {}
    withoutExtensionsSums = {}
    withExtensionsRebuildSums = {}
    withoutExtensionsRebuildSums = {}

    for libCount in libraryCountsToTest:
        withExtensionsSums[libCount] = 0
        withoutExtensionsSums[libCount] = 0
        withExtensionsRebuildSums[libCount] = 0
        withoutExtensionsRebuildSums[libCount] = 0

    for attempt in range(buildAttemptsToAverage):
        for libCount in libraryCountsToTest:
            withoutExtensionsSums[libCount] += time(attempt + 1, typeCount, libCount, False)
            withoutExtensionsRebuildSums[libCount] += timeRebuild(typeCount, libCount, False)
            withExtensionsSums[libCount] += time(attempt + 1, typeCount, libCount, True)
            withExtensionsRebuildSums[libCount] += timeRebuild(typeCount, libCount, True)

    print(f"Averages for {typeCount} types")
    for libCount in libraryCountsToTest:
        print(f"Libs: {libCount} Extensions ( ) Rebuild ( ): {withoutExtensionsSums[libCount] / buildAttemptsToAverage}")
        print(f"Libs: {libCount} Extensions (x) Rebuild ( ): {withExtensionsSums[libCount] / buildAttemptsToAverage}")
        print(f"Libs: {libCount} Extensions ( ) Rebuild (x): {withoutExtensionsRebuildSums[libCount] / buildAttemptsToAverage}")
        print(f"Libs: {libCount} Extensions (x) Rebuild (x): {withExtensionsRebuildSums[libCount] / buildAttemptsToAverage}")
    print("=============================================")

#generate(5, 4, False)
#makeChange(5, 4, False)
for typeCount in typeCountsToTest:
    benchmark(typeCount)