Export build logs to blob storage

tools/pipeline-witness/.editorconfig

@@ -0,0 +1,136 @@
+###############################
+# Core EditorConfig Options   #
+###############################
+root = true
+# All files
+[*]
+indent_style = space
+
+# XML project files
+[*.{csproj,vbproj,vcxproj,vcxproj.filters,proj,projitems,shproj}]
+indent_size = 2
+
+# XML config files
+[*.{props,targets,ruleset,config,nuspec,resx,vsixmanifest,vsct}]
+indent_size = 2
+
+# Code files
+[*.{cs,csx,vb,vbx}]
+indent_size = 4
+insert_final_newline = true
+
+###############################
+# .NET Coding Conventions     #
+###############################
+[*.{cs,vb}]
+# Organize usings
+dotnet_sort_system_directives_first = true
+# this. preferences
+dotnet_style_qualification_for_field = true:suggestion
+dotnet_style_qualification_for_property = false:silent
+dotnet_style_qualification_for_method = false:silent
+dotnet_style_qualification_for_event = false:silent
+# Language keywords vs BCL types preferences
+dotnet_style_predefined_type_for_locals_parameters_members = true:silent
+dotnet_style_predefined_type_for_member_access = true:silent
+# Parentheses preferences
+dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:silent
+dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:silent
+dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent
+dotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent
+# Modifier preferences
+dotnet_style_require_accessibility_modifiers = for_non_interface_members:silent
+dotnet_style_readonly_field = true:suggestion
+# Expression-level preferences
+dotnet_style_object_initializer = true:suggestion
+dotnet_style_collection_initializer = true:suggestion
+dotnet_style_explicit_tuple_names = true:suggestion
+dotnet_style_null_propagation = true:suggestion
+dotnet_style_coalesce_expression = true:suggestion
+dotnet_style_prefer_is_null_check_over_reference_equality_method = true:silent
+dotnet_style_prefer_inferred_tuple_names = true:suggestion
+dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion
+dotnet_style_prefer_auto_properties = true:silent
+dotnet_style_prefer_conditional_expression_over_assignment = true:silent
+dotnet_style_prefer_conditional_expression_over_return = true:silent
+
+###############################
+# Naming Conventions          #
+###############################
+# Style Definitions
+dotnet_naming_style.pascal_case_style.capitalization             = pascal_case
+# Use PascalCase for constant fields
+dotnet_naming_rule.constant_fields_should_be_pascal_case.severity = suggestion
+dotnet_naming_rule.constant_fields_should_be_pascal_case.symbols  = constant_fields
+dotnet_naming_rule.constant_fields_should_be_pascal_case.style    = pascal_case_style
+dotnet_naming_symbols.constant_fields.applicable_kinds            = field
+dotnet_naming_symbols.constant_fields.applicable_accessibilities  = *
+dotnet_naming_symbols.constant_fields.required_modifiers          = const
+# Use no prefix for private fields
+dotnet_naming_rule.private_members_with_underscore.symbols  = private_fields
+dotnet_naming_rule.private_members_with_underscore.style    = prefix_underscore
+dotnet_naming_rule.private_members_with_underscore.severity = suggestion
+dotnet_naming_symbols.private_fields.applicable_kinds           = field
+dotnet_naming_symbols.private_fields.applicable_accessibilities = private
+dotnet_naming_style.prefix_underscore.capitalization = camel_case
+
+###############################
+# C# Coding Conventions       #
+###############################
+[*.cs]
+# var preferences
+csharp_style_var_for_built_in_types = true:silent
+csharp_style_var_when_type_is_apparent = true:silent
+csharp_style_var_elsewhere = true:silent
+# Expression-bodied members
+csharp_style_expression_bodied_methods = false:silent
+csharp_style_expression_bodied_constructors = false:silent
+csharp_style_expression_bodied_operators = false:silent
+csharp_style_expression_bodied_properties = true:silent
+csharp_style_expression_bodied_indexers = true:silent
+csharp_style_expression_bodied_accessors = true:silent
+# Pattern matching preferences
+csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion
+csharp_style_pattern_matching_over_as_with_null_check = true:suggestion
+# Null-checking preferences
+csharp_style_throw_expression = true:suggestion
+csharp_style_conditional_delegate_call = true:suggestion
+# Modifier preferences
+csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:suggestion
+# Expression-level preferences
+csharp_prefer_braces = true:silent
+csharp_style_deconstructed_variable_declaration = true:suggestion
+csharp_prefer_simple_default_expression = true:suggestion
+csharp_style_pattern_local_over_anonymous_function = true:suggestion
+csharp_style_inlined_variable_declaration = true:suggestion
+
+###############################
+# C# Formatting Rules         #
+###############################
+# New line preferences
+csharp_new_line_before_open_brace = all
+csharp_new_line_before_else = true
+csharp_new_line_before_catch = true
+csharp_new_line_before_finally = true
+csharp_new_line_before_members_in_object_initializers = true
+csharp_new_line_before_members_in_anonymous_types = true
+csharp_new_line_between_query_expression_clauses = true
+# Indentation preferences
+csharp_indent_case_contents = true
+csharp_indent_switch_labels = true
+csharp_indent_labels = flush_left
+# Space preferences
+csharp_space_after_cast = false
+csharp_space_after_keywords_in_control_flow_statements = true
+csharp_space_between_method_call_parameter_list_parentheses = false
+csharp_space_between_method_declaration_parameter_list_parentheses = false
+csharp_space_between_parentheses = false
+csharp_space_before_colon_in_inheritance_clause = true
+csharp_space_after_colon_in_inheritance_clause = true
+csharp_space_around_binary_operators = before_and_after
+csharp_space_between_method_declaration_empty_parameter_list_parentheses = false
+csharp_space_between_method_call_name_and_opening_parenthesis = false
+csharp_space_between_method_call_empty_parameter_list_parentheses = false
+# Wrapping preferences
+csharp_preserve_single_line_statements = true
+csharp_preserve_single_line_blocks = true

tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Azure.Sdk.Tools.PipelineWitness.csproj

@@ -6,6 +6,7 @@
   <ItemGroup>
     <PackageReference Include="Azure.Cosmos" Version="4.0.0-preview3" />
     <PackageReference Include="Azure.Identity" Version="1.2.0" />
+    <PackageReference Include="Azure.Storage.Blobs" Version="12.10.0" />
     <PackageReference Include="Microsoft.Azure.Functions.Extensions" Version="1.0.0" />
     <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.EventHubs" Version="4.1.1" />
     <PackageReference Include="Microsoft.Extensions.Azure" Version="1.0.0" />

tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/RunProcessor.cs

@@ -1,47 +1,45 @@
-using Azure.Cosmos;
-using Azure.Identity;
-using Azure.Sdk.Tools.PipelineWitness.Entities.AzurePipelines;
-using Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis;
-using Azure.Security.KeyVault.Secrets;
-using Microsoft.AspNetCore.WebUtilities;
-using Microsoft.Extensions.Caching.Memory;
+using System;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+
+using Azure.Cosmos;
+using Azure.Storage.Blobs;
 using Microsoft.Extensions.Logging;
 using Microsoft.TeamFoundation.Build.WebApi;
 using Microsoft.TeamFoundation.Core.WebApi;
-using Microsoft.VisualStudio.Services.Common;
-using Microsoft.VisualStudio.Services.Organization.Client;
 using Microsoft.VisualStudio.Services.WebApi;
-using Microsoft.WindowsAzure.Storage.Blob.Protocol;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Net;
-using System.Net.Http;
-using System.Net.Http.Headers;
-using System.Text;
-using System.Text.Json;
-using System.Threading.Tasks;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Serialization;
+
+using Azure.Sdk.Tools.PipelineWitness.Entities.AzurePipelines;
+using Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis;
+using Azure.Sdk.Tools.PipelineWitness.Services;
 
 namespace Azure.Sdk.Tools.PipelineWitness
 {
+
+
     public class RunProcessor
     {
-        public RunProcessor(IFailureAnalyzer failureAnalyzer, ILogger<RunProcessor> logger, IMemoryCache cache, SecretClient secretClient, CosmosClient cosmosClient, VssConnection vssConnection)
+        public RunProcessor(IFailureAnalyzer failureAnalyzer, ILogger<RunProcessor> logger, BuildLogProvider logProvider, CosmosClient cosmosClient, BlobContainerClient containerClient, VssConnection vssConnection)
         {
             this.failureAnalyzer = failureAnalyzer;
             this.logger = logger;
-            this.cache = cache;
-            this.secretClient = secretClient;
+            this.logProvider = logProvider;
             this.cosmosClient = cosmosClient;
+            this.containerClient = containerClient;
             this.vssConnection = vssConnection;
         }
 
-        private IFailureAnalyzer failureAnalyzer;
-        private ILogger<RunProcessor> logger;
-        private IMemoryCache cache;
-        private SecretClient secretClient;
-        private CosmosClient cosmosClient;
-        private VssConnection vssConnection;
+        private static readonly JsonSerializerSettings jsonSettings = new () { ContractResolver = new CamelCasePropertyNamesContractResolver(), Formatting = Formatting.None };
+        private readonly IFailureAnalyzer failureAnalyzer;
+        private readonly ILogger<RunProcessor> logger;
+        private readonly BuildLogProvider logProvider;
+        private readonly CosmosClient cosmosClient;
+        private readonly BlobContainerClient containerClient;
+        private readonly VssConnection vssConnection;
 
         private bool IsValidAzureDevOpsUri(Uri uri)
         {
@@ -67,8 +65,8 @@ public async Task ProcessRunAsync(Uri runUri)
             var buildClient = vssConnection.GetClient<BuildHttpClient>();
             var projectClient = vssConnection.GetClient<ProjectHttpClient>();
 
-            var build = await buildClient.GetBuildAsync(projectGuid, runId);
             var project = await projectClient.GetProject(projectGuid.ToString());
+            var build = await buildClient.GetBuildAsync(projectGuid, runId);
             var pipeline = await buildClient.GetDefinitionAsync(project.Id, build.Definition.Id);
             var timeline = await buildClient.GetBuildTimelineAsync(projectGuid, runId);
 
@@ -144,6 +142,8 @@ public async Task ProcessRunAsync(Uri runUri)
 
             var container = await GetItemContainerAsync("azure-pipelines-runs");
             await container.UpsertItemAsync(run);
+
+            await UploadLogsBlobAsync(build, timeline);
         }
 
         public async Task<Failure[]> GetFailureClassificationsAsync(Build build, Timeline timeline)
@@ -152,6 +152,7 @@ public async Task<Failure[]> GetFailureClassificationsAsync(Build build, Timelin
             if (timeline == null) return new Failure[0];
 
             var failures = await failureAnalyzer.AnalyzeFailureAsync(build, timeline);
+
             return failures.ToArray();
         }
 
@@ -161,5 +162,74 @@ private async Task<CosmosContainer> GetItemContainerAsync(string containerName)
             var container = cosmosClient.GetContainer(database.Id, containerName);
             return container;
         }
+
+        private async Task UploadLogsBlobAsync(Build build, Timeline timeline)
+        {
+            if (timeline.Records == null)
+            {
+                return;
+            }
+
+            const string timeFormat = @"yyyy-MM-dd\THH:mm:ss.fffffff\Z";
+
+            var hasErrors = false;
+
+            await using var messagesWriter = new StringWriter();
+            foreach(var record in timeline.Records)
+            {
+                var logLines = await this.logProvider.GetTimelineRecordLogsAsync(build, record);
+
+                var lastTimeStamp = record.StartTime;
+
+                for(var lineNumber = 1; lineNumber <= logLines.Count; lineNumber++)
+                {
+                    var line = logLines[lineNumber - 1];
+                    var match = Regex.Match(line, @"^([^Z]{20,28}Z) (.*)$");
+                    var timestamp = match.Success
+                        ? DateTime.ParseExact(match.Groups[1].Value, timeFormat, null, System.Globalization.DateTimeStyles.AssumeUniversal).ToUniversalTime()
+                        : lastTimeStamp;
+
+                    var message = match.Success ? match.Groups[2].Value : line;
+
+                    if (timestamp == null)
+                    {
+                        messagesWriter.WriteLine("ERROR: NO LEADING TIMESTAMP");
+                        hasErrors = true;
+                    }
+
+                    await messagesWriter.WriteLineAsync(JsonConvert.SerializeObject(new
+                    {
+                        OrganizationName = "azure-sdk", 
+                        ProjectId = build.Project.Id, 
+                        BuildId = build.Id, 
+                        BuildDefinitionId = build.Definition.Id, 
+                        LogId = record.Log.Id, 
+                        LineNumber = lineNumber, 
+                        Length = message.Length,
+                        Timestamp = timestamp?.ToString(timeFormat),
+                        Message = message,
+                        EtlIngestDate = DateTime.UtcNow.ToString(timeFormat),
+                    }, jsonSettings));
+
+                    lastTimeStamp = timestamp;
+                }
+            }
+
+            var blobPathPrefix = $"{build.QueueTime:yyyy/MM/dd}/{build.Project.Name}-{build.Id}";
+            var blobClient = containerClient.GetBlobClient($"/success/{blobPathPrefix}.jsonl");
+
+            if (hasErrors)
+            {
+                var attempt = 1;
+
+                do
+                {
+                    blobClient = containerClient.GetBlobClient($"/error/{blobPathPrefix}-{attempt++}.jsonl");
+                }
+                while (await blobClient.ExistsAsync());
+            }
+
+            await blobClient.UploadAsync(new BinaryData(messagesWriter.ToString()));
+        }
     }
 }

tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/BuildLogProvider.cs

@@ -0,0 +1,55 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Logging;
+using Microsoft.TeamFoundation.Build.WebApi;
+using Microsoft.VisualStudio.Services.WebApi;
+
+namespace Azure.Sdk.Tools.PipelineWitness.Services
+{
+    public class BuildLogProvider
+    {
+        private readonly ILogger<BuildLogProvider> logger;
+        private readonly IMemoryCache cache;
+        private readonly VssConnection vssConnection;
+
+        public BuildLogProvider(ILogger<BuildLogProvider> logger, IMemoryCache cache, VssConnection vssConnection)
+        {
+            this.logger = logger;
+            this.cache = cache;
+            this.vssConnection = vssConnection;
+        }
+
+        protected BuildLogProvider()
+        {
+        }
+
+        public virtual async Task<IReadOnlyList<string>> GetTimelineRecordLogsAsync(Build build, TimelineRecord record)
+        {
+            if (record.Log == null)
+            {
+                return null;
+            }
+
+            var cacheKey = $"{build.Id}:{record.Log.Id}";
+
+            logger.LogTrace("Getting logs for {CacheKey} from cache", cacheKey);
+            var lines = await this.cache.GetOrCreateAsync(cacheKey, async entry =>
+            {
+                logger.LogTrace("Cache miss for {CacheKey}, falling back to rest api", cacheKey);
+                var buildHttpClient = vssConnection.GetClient<BuildHttpClient>();
+                var response = await buildHttpClient.GetBuildLogLinesAsync(build.Project.Id, build.Id, record.Log.Id);
+                var characterCount = response.Sum(x => x.Length);
+                entry.Priority = CacheItemPriority.Low;
+                entry.Size =characterCount + 4;
+
+                logger.LogTrace("Caching {CharacterCount} characters in {LineCount} lines for {CacheKey}", characterCount, response.Count, cacheKey);
+                return response;
+            });
+
+            return lines;
+        }
+    }
+}