Performance benchmark

.editorconfig

@@ -23,3 +23,7 @@ indent_size = 2
 [*.paket.references]
 trim_trailing_whitespace = true
 indent_size = 2
+
+[*.yml]
+trim_trailing_whitespace = true
+indent_size = 2

.github/workflows/build.yml

@@ -44,4 +44,4 @@ jobs:
       - name: Build
         run: ./build.cmd
         env:
-          CI: true
+          CI: true

GraphBLAS-sharp.sln

@@ -11,6 +11,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{ACBEE43C
 EndProject
 Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "GraphBLAS-sharp.Tests", "tests\GraphBLAS-sharp.Tests\GraphBLAS-sharp.Tests.fsproj", "{1CA2E092-2320-451D-A4F0-9ED7C7C528CA}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "benchmarks", "benchmarks", "{DEF656DE-BCED-4C49-B5ED-950D4A29B78B}"
+EndProject
+Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "GraphBLAS-sharp.Benchmarks", "benchmarks\GraphBLAS-sharp.Benchmarks\GraphBLAS-sharp.Benchmarks.fsproj", "{4C6EB3D0-B6BF-4FF5-BC77-CC7CB3F307E6}"
+EndProject
 Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "docsTool", "docsTool\docsTool.fsproj", "{8855EC73-F6A1-43D3-AFBC-04A3E09F9BD9}"
 EndProject
 Global
@@ -62,9 +66,22 @@ Global
 		{8855EC73-F6A1-43D3-AFBC-04A3E09F9BD9}.Release|x64.Build.0 = Release|Any CPU
 		{8855EC73-F6A1-43D3-AFBC-04A3E09F9BD9}.Release|x86.ActiveCfg = Release|Any CPU
 		{8855EC73-F6A1-43D3-AFBC-04A3E09F9BD9}.Release|x86.Build.0 = Release|Any CPU
+		{4C6EB3D0-B6BF-4FF5-BC77-CC7CB3F307E6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4C6EB3D0-B6BF-4FF5-BC77-CC7CB3F307E6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4C6EB3D0-B6BF-4FF5-BC77-CC7CB3F307E6}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4C6EB3D0-B6BF-4FF5-BC77-CC7CB3F307E6}.Debug|x64.Build.0 = Debug|Any CPU
+		{4C6EB3D0-B6BF-4FF5-BC77-CC7CB3F307E6}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{4C6EB3D0-B6BF-4FF5-BC77-CC7CB3F307E6}.Debug|x86.Build.0 = Debug|Any CPU
+		{4C6EB3D0-B6BF-4FF5-BC77-CC7CB3F307E6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4C6EB3D0-B6BF-4FF5-BC77-CC7CB3F307E6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{4C6EB3D0-B6BF-4FF5-BC77-CC7CB3F307E6}.Release|x64.ActiveCfg = Release|Any CPU
+		{4C6EB3D0-B6BF-4FF5-BC77-CC7CB3F307E6}.Release|x64.Build.0 = Release|Any CPU
+		{4C6EB3D0-B6BF-4FF5-BC77-CC7CB3F307E6}.Release|x86.ActiveCfg = Release|Any CPU
+		{4C6EB3D0-B6BF-4FF5-BC77-CC7CB3F307E6}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{5D30E174-2538-47AC-8443-318C8C5DC2C9} = {C397A34C-84F1-49E7-AEBC-2F9F2B196216}
 		{1CA2E092-2320-451D-A4F0-9ED7C7C528CA} = {ACBEE43C-7A88-4FB1-9B06-DB064D22B29F}
+		{4C6EB3D0-B6BF-4FF5-BC77-CC7CB3F307E6} = {DEF656DE-BCED-4C49-B5ED-950D4A29B78B}
 	EndGlobalSection
 EndGlobal

benchmarks/Directory.Build.props

@@ -0,0 +1,7 @@
+<Project>
+    <Import Project="$([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)../'))" />
+    <PropertyGroup>
+        <IsPackable>false</IsPackable>
+        <IsTestProject>false</IsTestProject>
+    </PropertyGroup>
+</Project>

benchmarks/GraphBLAS-sharp.Benchmarks/AssemblyInfo.fs

@@ -0,0 +1,23 @@
+// Auto-Generated by FAKE; do not edit
+namespace System
+open System.Reflection
+
+[<assembly: AssemblyTitleAttribute("GraphBLAS-sharp.Benchmarks")>]
+[<assembly: AssemblyProductAttribute("GraphBLAS-sharp")>]
+[<assembly: AssemblyVersionAttribute("0.1.0")>]
+[<assembly: AssemblyMetadataAttribute("ReleaseDate","2017-03-17T00:00:00.0000000")>]
+[<assembly: AssemblyFileVersionAttribute("0.1.0")>]
+[<assembly: AssemblyInformationalVersionAttribute("0.1.0")>]
+[<assembly: AssemblyMetadataAttribute("ReleaseChannel","release")>]
+[<assembly: AssemblyMetadataAttribute("GitHash","bb8964b54bee133e9af64d316dc2cfee16df7f72")>]
+do ()
+
+module internal AssemblyVersionInformation =
+    let [<Literal>] AssemblyTitle = "GraphBLAS-sharp.Benchmarks"
+    let [<Literal>] AssemblyProduct = "GraphBLAS-sharp"
+    let [<Literal>] AssemblyVersion = "0.1.0"
+    let [<Literal>] AssemblyMetadata_ReleaseDate = "2017-03-17T00:00:00.0000000"
+    let [<Literal>] AssemblyFileVersion = "0.1.0"
+    let [<Literal>] AssemblyInformationalVersion = "0.1.0"
+    let [<Literal>] AssemblyMetadata_ReleaseChannel = "release"
+    let [<Literal>] AssemblyMetadata_GitHash = "bb8964b54bee133e9af64d316dc2cfee16df7f72"

benchmarks/GraphBLAS-sharp.Benchmarks/BenchmarkDotNet.Artifacts/results/GraphBLAS.FSharp.Benchmarks.BfsBenchmark-report-github.md

@@ -0,0 +1,17 @@
+``` ini
+
+BenchmarkDotNet=v0.12.1, OS=Windows 7 SP1 (6.1.7601.0)
+Intel Celeron CPU N2830 2.16GHz, 1 CPU, 2 logical and 2 physical cores
+Frequency=2115908 Hz, Resolution=472.6103 ns, Timer=TSC
+.NET Core SDK=3.1.302
+  [Host] : .NET Core 3.1.8 (CoreCLR 4.700.20.41105, CoreFX 4.700.20.41903), X64 RyuJIT DEBUG
+
+IterationCount=10  
+
+```
+|   Method |          PathToGraph | Mean | Error |                           TEPS |
+|--------- |--------------------- |-----:|------:|-------------------------------:|
+| LevelBFS | Datas(...)r.mtx [32] |   NA |    NA | ("227320", "227320", "814134") |
+
+Benchmarks with issues:
+  BfsBenchmark.LevelBFS: Job-WTZPYK(IterationCount=10) [PathToGraph=Datas(...)r.mtx [32]]

benchmarks/GraphBLAS-sharp.Benchmarks/BenchmarkDotNet.Artifacts/results/GraphBLAS.FSharp.Benchmarks.BfsBenchmark-report.csv

@@ -0,0 +1,2 @@
+Method;Job;AnalyzeLaunchVariance;EvaluateOverhead;MaxAbsoluteError;MaxRelativeError;MinInvokeCount;MinIterationTime;OutlierMode;Affinity;EnvironmentVariables;Jit;Platform;PowerPlanMode;Runtime;AllowVeryLargeObjects;Concurrent;CpuGroups;Force;HeapAffinitizeMask;HeapCount;NoAffinitize;RetainVm;Server;Arguments;BuildConfiguration;Clock;EngineFactory;NuGetReferences;Toolchain;IsMutator;InvocationCount;IterationCount;IterationTime;LaunchCount;MaxIterationCount;MaxWarmupIterationCount;MinIterationCount;MinWarmupIterationCount;RunStrategy;UnrollFactor;WarmupCount;PathToGraph;Mean;Error;TEPS
+LevelBFS;Job-WTZPYK;False;Default;Default;Default;Default;Default;Default;11;Empty;RyuJit;X64;8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c;.NET Core 3.1;False;True;False;True;Default;Default;False;False;False;Default;Default;Default;Default;Default;Default;Default;1;10;Default;Default;Default;Default;Default;Default;Default;16;Default;Datas(...)r.mtx [32];NA;NA;"(""227320"", ""227320"", ""814134"")"

benchmarks/GraphBLAS-sharp.Benchmarks/BenchmarkDotNet.Artifacts/results/GraphBLAS.FSharp.Benchmarks.BfsBenchmark-report.html

@@ -0,0 +1,31 @@
+<!DOCTYPE html>
+<html lang='en'>
+<head>
+<meta charset='utf-8' />
+<title>GraphBLAS.FSharp.Benchmarks.BfsBenchmark-20201206-170542</title>
+
+<style type="text/css">
+	table { border-collapse: collapse; display: block; width: 100%; overflow: auto; }
+	td, th { padding: 6px 13px; border: 1px solid #ddd; text-align: right; }
+	tr { background-color: #fff; border-top: 1px solid #ccc; }
+	tr:nth-child(even) { background: #f8f8f8; }
+</style>
+</head>
+<body>
+<pre><code>
+BenchmarkDotNet=v0.12.1, OS=Windows 7 SP1 (6.1.7601.0)
+Intel Celeron CPU N2830 2.16GHz, 1 CPU, 2 logical and 2 physical cores
+Frequency=2115908 Hz, Resolution=472.6103 ns, Timer=TSC
+.NET Core SDK=3.1.302
+  [Host] : .NET Core 3.1.8 (CoreCLR 4.700.20.41105, CoreFX 4.700.20.41903), X64 RyuJIT DEBUG
+</code></pre>
+<pre><code>IterationCount=10  
+</code></pre>
+
+<table>
+<thead><tr><th>Method</th><th>   PathToGraph</th><th>Mean</th><th>Error</th><th>                    TEPS</th>
+</tr>
+</thead><tbody><tr><td>LevelBFS</td><td>Datas(...)r.mtx [32]</td><td>NA</td><td>NA</td><td>(&quot;227320&quot;, &quot;227320&quot;, &quot;814134&quot;)</td>
+</tr></tbody></table>
+</body>
+</html>

benchmarks/GraphBLAS-sharp.Benchmarks/BfsBenchmark.fs

@@ -0,0 +1,36 @@
+namespace GraphBLAS.FSharp.Benchmarks
+
+open GraphBLAS.FSharp
+open GraphBLAS.FSharp.Algorithms
+open BenchmarkDotNet.Attributes
+open BenchmarkDotNet.Configs
+open BenchmarkDotNet.Columns
+open System.IO
+open System
+open MatrixBackend
+
+[<Config(typeof<Config>)>]
+[<SimpleJob(targetCount=10)>]
+type BFSBenchmark4CSRMatrix() =
+    let random = Random()
+
+    let mutable matrix = Unchecked.defaultof<Matrix<bool>>
+    let mutable source = 0
+
+    [<ParamsSource("GraphPaths")>]
+    member val PathToGraph = "" with get, set
+
+    [<GlobalSetup>]
+    member this.BuildMatrix() =
+        matrix <- Matrix.Build<bool>(this.PathToGraph, CSR)
+        source <- random.Next matrix.RowCount
+
+    [<Benchmark>]
+    member this.LevelBFS() =
+        levelBFS matrix source
+
+    /// Sequence of paths to files where data for benchmarking will be taken from
+    static member GraphPaths = seq {
+        // Gets all mtx files from following directory
+        yield! Directory.EnumerateFiles(Path.Join [|"Datasets"; "1"|], "*.mtx")
+    }

benchmarks/GraphBLAS-sharp.Benchmarks/Config.fs

@@ -0,0 +1,43 @@
+namespace GraphBLAS.FSharp.Benchmarks
+
+open GraphBLAS.FSharp.Algorithms
+open BenchmarkDotNet.Attributes
+open BenchmarkDotNet.Configs
+open BenchmarkDotNet.Columns
+open BenchmarkDotNet.Reports
+open BenchmarkDotNet.Running
+open System.IO
+
+type TEPSColumn() =
+    interface IColumn with
+        member this.AlwaysShow: bool = true
+        member this.Category: ColumnCategory = ColumnCategory.Statistics
+        member this.ColumnName: string = "TEPS"
+        member this.GetValue(summary: Summary, benchmarkCase: BenchmarkCase): string =
+            let meanTime = summary.[benchmarkCase].ResultStatistics.Mean
+            let pathToGraph = benchmarkCase.Parameters.["PathToGraph"].ToString()
+            match Path.GetExtension pathToGraph with
+            | ".mtx" ->
+                use streamReader = new StreamReader(pathToGraph)
+                while streamReader.Peek() = int '%' do
+                    streamReader.ReadLine() |> ignore
+                let matrixInfo = streamReader.ReadLine().Split(' ')
+                let (nrows, ncols, nnz) = float matrixInfo.[0], float matrixInfo.[1], float matrixInfo.[2]
+                let (vertices, edges) = if nrows = ncols then (nrows, nnz) else (ncols, nrows)
+                sprintf "%f" (edges / meanTime)
+            | another -> sprintf "%s files not supported" another
+        member this.GetValue(summary: Summary, benchmarkCase: BenchmarkCase, style: SummaryStyle): string =
+            (this :> IColumn).GetValue(summary, benchmarkCase)
+        member this.Id: string = "TEPSColumn"
+        member this.IsAvailable(summary: Summary): bool = true
+        member this.IsDefault(summary: Summary, benchmarkCase: BenchmarkCase): bool = false
+        member this.IsNumeric: bool = true
+        member this.Legend: string = "Traversed edges per second"
+        member this.PriorityInCategory: int = 0
+        member this.UnitType: UnitType = UnitType.Dimensionless
+
+type Config() =
+    inherit ManualConfig()
+
+    do
+        base.AddColumn [| TEPSColumn() :> IColumn |] |> ignore

benchmarks/GraphBLAS-sharp.Benchmarks/Datasets/1/graphs.csv

@@ -0,0 +1,2 @@
+GraphName,ArchiveType,Url
+coAuthorsCiteseer,zip,http://nrvis.com/download/data/dimacs10/coAuthorsCiteseer.zip

benchmarks/GraphBLAS-sharp.Benchmarks/Datasets/GAP/graphs.csv

@@ -0,0 +1,4 @@
+GraphName,ArchiveType,Url
+twitter,tar.gz,http://an.kaist.ac.kr/~haewoon/release/twitter_social_graph/twitter_rv.tar.gz
+web,tar.gz,https://sparse.tamu.edu/MM/LAW/sk-2005.tar.gz
+road,gz,http://www.dis.uniroma1.it/challenge9/data/USA-road-d/USA-road-d.USA.gr.gz

benchmarks/GraphBLAS-sharp.Benchmarks/DownloadGraphs.fsx

@@ -0,0 +1,34 @@
+#r "nuget: FSharp.Data"
+
+open System
+open System.IO
+open System.Net
+open System.IO.Compression
+open FSharp.Data
+open FSharp.Data.CsvExtensions
+
+let downloadAndUnzip graphName archiveType url outputDir =
+    use client = new WebClient()
+    match archiveType with
+    | "zip" ->
+        let archive = Path.Combine [| outputDir; Path.ChangeExtension(graphName, ".zip") |]
+        client.AsyncDownloadFile(Uri url, archive) |> Async.RunSynchronously
+        ZipFile.ExtractToDirectory(archive, outputDir)
+    | _ -> ()
+
+(*
+    Get all "graphs.csv" flies
+    -> load all rows from all csv files
+    -> download and unzip graph archives
+*)
+seq {
+    for dir in fsi.CommandLineArgs.[1..] do
+        yield! Directory.EnumerateFiles(dir, "graphs.csv", SearchOption.AllDirectories)
+}
+|> Seq.map Path.GetFullPath
+|> Seq.collect (fun pathToCsv ->
+    CsvFile.Load(pathToCsv, separators=",", hasHeaders=true).Rows |> Seq.allPairs <| Seq.singleton pathToCsv)
+|> Seq.iter (fun (row, pathToCsv) ->
+    let datasetRootPath = Path.GetDirectoryName pathToCsv
+    downloadAndUnzip row?GraphName row?ArchiveType row?Url datasetRootPath)
+

benchmarks/GraphBLAS-sharp.Benchmarks/GraphBLAS-sharp.Benchmarks.fsproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <Optimize>true</Optimize>
+    <Tailcalls>true</Tailcalls>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\GraphBLAS-sharp\GraphBLAS-sharp.fsproj">
+      <Name>GraphBLAS-sharp.fsproj</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="AssemblyInfo.fs" />
+    <Compile Include="Config.fs" />
+    <Compile Include="BfsBenchmark.fs" />
+    <Compile Include="Program.fs" />
+  </ItemGroup>
+  <Import Project="..\..\.paket\Paket.Restore.targets" />
+</Project>

benchmarks/GraphBLAS-sharp.Benchmarks/Program.fs

@@ -0,0 +1,11 @@
+open BenchmarkDotNet.Running
+open GraphBLAS.FSharp.Benchmarks
+
+[<EntryPoint>]
+let main argv =
+    let benchmarks = BenchmarkSwitcher [|
+        typeof<BFSBenchmark4CSRMatrix>
+    |]
+
+    benchmarks.Run argv |> ignore
+    0

benchmarks/GraphBLAS-sharp.Benchmarks/paket.references

@@ -0,0 +1,3 @@
+FSharp.Core
+Microsoft.NETFramework.ReferenceAssemblies
+BenchmarkDotNet

global.json

@@ -0,0 +1,6 @@
+{
+    "sdk": {
+        "version": "3.1.302",
+        "rollForward": "latestFeature"
+    }
+}

paket.dependencies

@@ -6,6 +6,7 @@ nuget FSharp.Core 4.3.4
 nuget Microsoft.SourceLink.GitHub prerelease copy_local: true
 nuget Microsoft.NETFramework.ReferenceAssemblies copy_local: true
 nuget Expecto 8.13.1
+nuget Expecto.FsCheck 8.13.1
 nuget YoloDev.Expecto.TestSdk 0.8.0
 nuget Microsoft.NET.Test.Sdk 15.7.2
 nuget altcover ~> 6
@@ -18,6 +19,8 @@ nuget FSharp.Quotations.Evaluator 2.1.0
 nuget FSharpx.Collections 2.1.3
 nuget FSharpx.Text.StructuredFormat 2.3.0
 nuget Brahma.FSharp.OpenCL.WorkflowBuilder
+nuget BenchmarkDotNet
+nuget MathNet.Numerics.FSharp
 
 // [ FAKE GROUP ]
 group Build
@@ -57,4 +60,4 @@ group Docs
 group Analyzers
   source https://www.nuget.org/api/v2
   source https://api.nuget.org/v3/index.json
-  nuget BinaryDefense.FSharp.Analyzers.Hashing 0.1.0
+  nuget BinaryDefense.FSharp.Analyzers.Hashing 0.1.0