Browse files

Create Warehouse Reports job is done!

  • Loading branch information...
1 parent 2486f7a commit 9899c1a4fc45fc94205266ce64dc18e14e9e6252 @anurse anurse committed Jan 28, 2014
Showing with 926 additions and 226 deletions.
  1. +0 −114 src/Services/NuGet.Services.Platform.Client/AzureStorageReference.cs
  2. +29 −3 src/Services/NuGet.Services.Platform.Client/Client/JsonFormat.cs
  3. +21 −0 src/Services/NuGet.Services.Platform.Client/Constants.cs
  4. +1 −1 src/Services/NuGet.Services.Platform.Client/NuGet.Services.Platform.Client.csproj
  5. +4 −0 src/Services/NuGet.Services.Platform/Configuration/StorageConfiguration.cs
  6. +1 −1 src/Services/NuGet.Services.Platform/ServicePlatformEventSource.cs
  7. +0 −31 src/Services/NuGet.Services.Platform/Storage/StorageHub.cs
  8. +0 −3 src/Services/Work/NuGet.Services.Work/Helpers/PackageHelpers.cs
  9. +27 −0 src/Services/Work/NuGet.Services.Work/Helpers/ResourceHelpers.cs
  10. +9 −13 src/Services/Work/NuGet.Services.Work/Infrastructure/JobHandlerBase.cs
  11. +1 −1 src/Services/Work/NuGet.Services.Work/Jobs/AggregateStatisticsJob.cs
  12. +21 −22 src/Services/Work/NuGet.Services.Work/Jobs/BackupPackageBlobsJob.cs
  13. +1 −1 src/Services/Work/NuGet.Services.Work/Jobs/CleanOnlineDatabaseBackupsJob.cs
  14. +1 −1 src/Services/Work/NuGet.Services.Work/Jobs/CreateOnlineDatabaseBackupJob.cs
  15. +650 −3 src/Services/Work/NuGet.Services.Work/Jobs/CreateWarehouseReportsJob.cs
  16. +12 −13 src/Services/Work/NuGet.Services.Work/Jobs/HandlePackageEditsJob.cs
  17. +1 −1 src/Services/Work/NuGet.Services.Work/Jobs/PurgeCompletedInvocationsJob.cs
  18. +1 −1 src/Services/Work/NuGet.Services.Work/Jobs/PurgePackageStatisticsJob.cs
  19. +1 −1 src/Services/Work/NuGet.Services.Work/Jobs/ReplicatePackageStatisticsJob.cs
  20. +16 −0 src/Services/Work/NuGet.Services.Work/Jobs/Scripts/DownloadReport_Last6Months.sql
  21. +10 −0 src/Services/Work/NuGet.Services.Work/Jobs/Scripts/DownloadReport_ListInactive.sql
  22. +12 −0 src/Services/Work/NuGet.Services.Work/Jobs/Scripts/DownloadReport_NuGetClientVersion.sql
  23. +10 −0 src/Services/Work/NuGet.Services.Work/Jobs/Scripts/DownloadReport_RecentPopularity.sql
  24. +9 −0 src/Services/Work/NuGet.Services.Work/Jobs/Scripts/DownloadReport_RecentPopularityByPackage.sql
  25. +21 −0 src/Services/Work/NuGet.Services.Work/Jobs/Scripts/DownloadReport_RecentPopularityDetail.sql
  26. +31 −0 ...Services/Work/NuGet.Services.Work/Jobs/Scripts/DownloadReport_RecentPopularityDetailByPackage.sql
  27. +1 −1 src/Services/Work/NuGet.Services.Work/Jobs/TestAsyncJob.cs
  28. +1 −1 src/Services/Work/NuGet.Services.Work/Jobs/TestHeartBeatJob.cs
  29. +1 −1 src/Services/Work/NuGet.Services.Work/Jobs/TestLongRunningJob.cs
  30. +1 −1 src/Services/Work/NuGet.Services.Work/Jobs/TestPingJob.cs
  31. +1 −1 src/Services/Work/NuGet.Services.Work/Jobs/UpdateLicenseReportsJob.cs
  32. +1 −1 src/Services/Work/NuGet.Services.Work/Monitoring/InvocationEventSource.cs
  33. +1 −1 src/Services/Work/NuGet.Services.Work/Monitoring/WorkServiceEventSource.cs
  34. +8 −7 src/Services/Work/NuGet.Services.Work/NuGet.Services.Work.csproj
  35. +9 −0 src/Services/Work/NuGet.Services.Work/Strings.Designer.cs
  36. +3 −0 src/Services/Work/NuGet.Services.Work/Strings.resx
  37. +9 −2 src/Services/Work/NuGet.Services.Work/WorkService.cs
View
114 src/Services/NuGet.Services.Platform.Client/AzureStorageReference.cs
@@ -1,114 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Globalization;
-using System.Linq;
-using System.Net;
-using System.Text;
-using System.Text.RegularExpressions;
-using System.Threading.Tasks;
-
-namespace NuGet.Services
-{
- public class AzureStorageReference
- {
- public static readonly string AzureStorageScheme = "azstore";
-
- public string AccountName { get; private set; }
- public string AccountKey { get; private set; }
- public string Container { get; private set; }
- public string Path { get; private set; }
-
- public AzureStorageReference(string accountName, string container, string path)
- : this(accountName, String.Empty, container, path) { }
- public AzureStorageReference(string accountName, string accountKey, string container, string path)
- {
- AccountName = accountName;
- AccountKey = accountKey;
- Container = container;
- Path = path;
- }
-
- public static bool TryCreate(Uri sourceUri, out AzureStorageReference parsed)
- {
- return TryCreateCore(sourceUri, throwOnError: false, parsed: out parsed);
- }
-
- public static AzureStorageReference Create(Uri sourceUri)
- {
- AzureStorageReference parsed;
- TryCreateCore(sourceUri, throwOnError: true, parsed: out parsed);
- return parsed;
- }
-
- public override string ToString()
- {
- StringBuilder builder = new StringBuilder();
- builder.Append(AzureStorageScheme);
- builder.Append("://");
- if (!String.IsNullOrEmpty(AccountKey))
- {
- builder.Append(AccountKey);
- builder.Append("@");
- }
- builder.Append(AccountName);
- if (!String.IsNullOrEmpty(Container))
- {
- builder.Append("/");
- builder.Append(Container);
- if (!String.IsNullOrEmpty(Path))
- {
- builder.Append("/");
- builder.Append(Path);
- }
- }
- return builder.ToString();
- }
-
- private static readonly Regex PathParser = new Regex("^/(?<container>[^/]*)(/(?<path>.*))?$");
- private static bool TryCreateCore(Uri sourceUri, bool throwOnError, out AzureStorageReference parsed)
- {
- parsed = null;
- if (!sourceUri.IsAbsoluteUri)
- {
- if (throwOnError)
- {
- throw new FormatException(Strings.AzureStorageAccount_RequiresAbsoluteUri);
- }
- return false;
- }
-
- if (!String.Equals(sourceUri.Scheme, AzureStorageScheme))
- {
- if (throwOnError)
- {
- throw new FormatException(String.Format(
- CultureInfo.CurrentCulture,
- Strings.AzureStorageAccount_InvalidScheme,
- sourceUri.Scheme,
- AzureStorageScheme));
- }
- return false;
- }
-
- var fullPath = sourceUri.AbsolutePath;
- var match = PathParser.Match(fullPath);
- if (!match.Success)
- {
- if (throwOnError)
- {
- throw new FormatException(String.Format(
- CultureInfo.CurrentCulture,
- Strings.AzureStorageAccount_RequiresContainer,
- AzureStorageScheme));
- }
- return false;
- }
- parsed = new AzureStorageReference(
- sourceUri.Host,
- WebUtility.UrlDecode(sourceUri.UserInfo),
- match.Groups["container"].Value,
- match.Groups["path"].Value);
- return true;
- }
- }
-}
View
32 src/Services/NuGet.Services.Platform.Client/Client/JsonFormat.cs
@@ -17,6 +17,7 @@ namespace NuGet.Services.Client
public static class JsonFormat
{
private static JsonSerializerSettings _serializerSettings;
+ private static JsonSerializerSettings _nonCamelCasedSettings;
private static JsonMediaTypeFormatter _formatter;
public static JsonSerializerSettings SerializerSettings { get { return _serializerSettings; } }
@@ -38,6 +39,22 @@ static JsonFormat()
TypeNameHandling = TypeNameHandling.None
};
_serializerSettings.Converters.Add(new StringEnumConverter());
+
+ _nonCamelCasedSettings = new JsonSerializerSettings()
+ {
+ // ContractResolver = ...
+ DateFormatHandling = _serializerSettings.DateFormatHandling,
+ DateParseHandling = _serializerSettings.DateParseHandling,
+ DateTimeZoneHandling = _serializerSettings.DateTimeZoneHandling,
+ DefaultValueHandling = _serializerSettings.DefaultValueHandling,
+ Formatting = _serializerSettings.Formatting,
+ MissingMemberHandling = _serializerSettings.MissingMemberHandling,
+ NullValueHandling = _serializerSettings.NullValueHandling,
+ ReferenceLoopHandling = _serializerSettings.ReferenceLoopHandling,
+ TypeNameHandling = _serializerSettings.TypeNameHandling
+ };
+ _serializerSettings.Converters.Add(new StringEnumConverter());
+
_formatter = new JsonMediaTypeFormatter()
{
SerializerSettings = _serializerSettings
@@ -52,8 +69,10 @@ public static T Deserialize<T>(string content)
return JsonConvert.DeserializeObject<T>(content, _serializerSettings);
}
- public static string Serialize(object data)
+ public static string Serialize(object data) { return Serialize(data, camelCase: true); }
+ public static string Serialize(object data, bool camelCase)
{
+ var settings = camelCase ? _serializerSettings : _nonCamelCasedSettings;
return JsonConvert.SerializeObject(data, _serializerSettings);
}
@@ -62,9 +81,11 @@ public static Task<T> DeserializeAsync<T>(string content)
return JsonConvert.DeserializeObjectAsync<T>(content, _serializerSettings);
}
- public static Task<string> SerializeAsync(object data)
+ public static Task<string> SerializeAsync(object data) { return SerializeAsync(data, camelCase: true); }
+ public static Task<string> SerializeAsync(object data, bool camelCase)
{
- return JsonConvert.SerializeObjectAsync(data, _serializerSettings.Formatting, _serializerSettings);
+ var settings = camelCase ? _serializerSettings : _nonCamelCasedSettings;
+ return JsonConvert.SerializeObjectAsync(data, settings.Formatting, settings);
}
}
@@ -77,5 +98,10 @@ protected override JsonDictionaryContract CreateDictionaryContract(Type objectTy
contract.PropertyNameResolver = new Func<string, string>(s => s);
return contract;
}
+
+ protected override JsonObjectContract CreateObjectContract(Type objectType)
+ {
+ return base.CreateObjectContract(objectType);
+ }
}
}
View
21 src/Services/NuGet.Services.Platform.Client/Constants.cs
@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NuGet.Services
+{
+ public static class BlobContainerNames
+ {
+ public static readonly string LegacyPackages = "packages";
+ public static readonly string LegacyStats = "stats";
+
+ public static readonly string Backups = "ng-backups";
+ }
+
+ public static class MimeTypes
+ {
+ public static readonly string Json = "application/json";
+ }
+}
View
2 src/Services/NuGet.Services.Platform.Client/NuGet.Services.Platform.Client.csproj
@@ -33,9 +33,9 @@
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
- <Compile Include="AzureStorageReference.cs" />
<Compile Include="Client\JsonFormat.cs" />
<Compile Include="Client\ServiceResponse.cs" />
+ <Compile Include="Constants.cs" />
<Compile Include="CredentialProviderHandler.cs" />
<Compile Include="ICredentialProvider.cs" />
<Compile Include="Models\AssemblyFullNameConverter.cs" />
View
4 src/Services/NuGet.Services.Platform/Configuration/StorageConfiguration.cs
@@ -11,6 +11,10 @@ public class StorageConfiguration : ICustomConfigurationSection
{
public Dictionary<KnownStorageAccount, CloudStorageAccount> Accounts { get; private set; }
+ public CloudStorageAccount Primary { get { return GetAccount(KnownStorageAccount.Primary); } }
+ public CloudStorageAccount Legacy { get { return GetAccount(KnownStorageAccount.Legacy); } }
+ public CloudStorageAccount Backup { get { return GetAccount(KnownStorageAccount.Backup); } }
+
public CloudStorageAccount GetAccount(KnownStorageAccount account)
{
CloudStorageAccount connectionString;
View
2 src/Services/NuGet.Services.Platform/ServicePlatformEventSource.cs
@@ -9,7 +9,7 @@
namespace NuGet.Services
{
- [EventSource("Outercurve-NuGet-Services")]
+ [EventSource(Name="Outercurve-NuGet-Services")]
public class ServicePlatformEventSource : EventSource
{
public static readonly ServicePlatformEventSource Log = new ServicePlatformEventSource();
View
31 src/Services/NuGet.Services.Platform/Storage/StorageHub.cs
@@ -40,37 +40,6 @@ public StorageHub(StorageAccountHub primary, StorageAccountHub backup, StorageAc
Legacy = legacy;
}
- public StorageAccountHub GetAccount(AzureStorageReference reference)
- {
- if (String.Equals(reference.AccountName, ".", StringComparison.OrdinalIgnoreCase))
- {
- // Emulator!
- return new StorageAccountHub(CloudStorageAccount.DevelopmentStorageAccount);
- }
-
- Func<StorageHub, StorageAccountHub> handler;
- if (!_knownAccounts.TryGetValue(reference.AccountName, out handler))
- {
- return new StorageAccountHub(
- new CloudStorageAccount(
- new StorageCredentials(
- reference.AccountName,
- reference.AccountKey),
- useHttps: true));
- }
- return handler(this);
- }
-
- public CloudBlobContainer GetContainer(AzureStorageReference reference)
- {
- var account = GetAccount(reference);
- if (account == null)
- {
- return null;
- }
- return account.Blobs.Client.GetContainerReference(reference.Container);
- }
-
private static StorageAccountHub TryLoadAccount(ConfigurationHub configuration, KnownStorageAccount account)
{
var connectionString = configuration.Storage.GetAccount(account);
View
3 src/Services/Work/NuGet.Services.Work/Helpers/PackageHelpers.cs
@@ -11,9 +11,6 @@ namespace NuGet.Services.Work
{
public static class PackageHelpers
{
- public static readonly string PackageBlobContainer = "packages";
- public static readonly string BackupsBlobContainer = "ng-backups";
-
private const string PackageBlobNameFormat = "{0}.{1}.nupkg";
private const string PackageBackupBlobNameFormat = "packages/{0}/{1}/{2}.nupkg";
View
27 src/Services/Work/NuGet.Services.Work/Helpers/ResourceHelpers.cs
@@ -0,0 +1,27 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Work.Helpers
+{
+ public static class ResourceHelpers
+ {
+ public static Task<string> ReadResourceFile(string name)
+ {
+ return ReadResourceFile(name, typeof(ResourceHelpers).Assembly);
+ }
+
+ public static async Task<string> ReadResourceFile(string name, Assembly asm)
+ {
+ using (var stream = asm.GetManifestResourceStream(name))
+ using (var reader = new StreamReader(stream))
+ {
+ return await reader.ReadToEndAsync();
+ }
+ }
+ }
+}
View
22 src/Services/Work/NuGet.Services.Work/Infrastructure/JobHandlerBase.cs
@@ -12,6 +12,7 @@
using System.Data.SqlClient;
using System.IO;
using System.Threading;
+using Microsoft.WindowsAzure.Storage;
namespace NuGet.Services.Work
{
@@ -135,7 +136,7 @@ protected virtual void BindProperty(PropertyDescriptor prop, string value)
private IList<TypeConverter> _converters = new List<TypeConverter>() {
new SqlConnectionStringBuilderConverter(),
- new AzureStorageAccountConverter()
+ new CloudStorageAccountConverter()
};
protected virtual object ConvertPropertyValue(PropertyDescriptor prop, string value)
@@ -171,31 +172,26 @@ public override object ConvertFrom(ITypeDescriptorContext context, System.Global
}
}
- private class AzureStorageAccountConverter : TypeConverter
+ private class CloudStorageAccountConverter : TypeConverter
{
public override bool CanConvertFrom(ITypeDescriptorContext context, Type sourceType)
{
- return sourceType == typeof(string) || sourceType == typeof(Uri);
+ return sourceType == typeof(string);
}
public override bool CanConvertTo(ITypeDescriptorContext context, Type destinationType)
{
- return destinationType == typeof(AzureStorageReference);
+ return destinationType == typeof(CloudStorageAccount);
}
public override object ConvertFrom(ITypeDescriptorContext context, System.Globalization.CultureInfo culture, object value)
{
- Uri url = value as Uri;
- if (url == null)
+ string strVal = value as string;
+ if (strVal == null)
{
- string strVal = value as string;
- if (strVal == null)
- {
- return null;
- }
- url = new Uri(strVal);
+ return null;
}
- return AzureStorageReference.Create(url);
+ return CloudStorageAccount.Parse(strVal);
}
}
}
View
2 src/Services/Work/NuGet.Services.Work/Jobs/AggregateStatisticsJob.cs
@@ -120,7 +120,7 @@ COMMIT TRANSACTION
";
}
- [EventSource("Outercurve-NuGet-Jobs-AggregateStatistics")]
+ [EventSource(Name="Outercurve-NuGet-Jobs-AggregateStatistics")]
public class AggregateStatisticsEventSource : EventSource
{
public static readonly AggregateStatisticsEventSource Log = new AggregateStatisticsEventSource();
View
43 src/Services/Work/NuGet.Services.Work/Jobs/BackupPackageBlobsJob.cs
@@ -10,6 +10,7 @@
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using Dapper;
+using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using NuGet.Services.Configuration;
using NuGet.Services.Storage;
@@ -29,12 +30,14 @@ public class BackupPackageBlobsJob : JobHandler<BackupPackageBlobsEventSource>
/// <summary>
/// Gets or sets an Azure Storage Uri referring to a container to use as the source for package blobs
/// </summary>
- public AzureStorageReference Source { get; set; }
+ public CloudStorageAccount Source { get; set; }
+ public string SourceContainerName { get; set; }
/// <summary>
/// Gets or sets an Azure Storage Uri referring to a container to use as the destination
/// </summary>
- public AzureStorageReference Destination { get; set; }
+ public CloudStorageAccount Destination { get; set; }
+ public string DestinationContainerName { get; set; }
/// <summary>
/// Gets or sets a connection string to the database containing package data.
@@ -64,24 +67,20 @@ public BackupPackageBlobsJob(ConfigurationHub config, StorageHub storage)
// Load default data if not provided
PackageDatabase = PackageDatabase ?? Config.Sql.GetConnectionString(KnownSqlServer.Legacy);
- var sourceAccount = Source == null ?
- Storage.Legacy :
- Storage.GetAccount(Source);
- var destAccount = Destination == null ?
- Storage.Backup :
- Storage.GetAccount(Destination);
- SourceContainer = sourceAccount.Blobs.Client.GetContainerReference(
- Source == null ? PackageHelpers.PackageBlobContainer : Source.Container);
- DestinationContainer = destAccount.Blobs.Client.GetContainerReference(
- Destination == null ? PackageHelpers.BackupsBlobContainer : Destination.Container);
- Log.PreparingToBackup(sourceAccount.Name, SourceContainer.Name, destAccount.Name, DestinationContainer.Name, PackageDatabase.DataSource, PackageDatabase.InitialCatalog);
+ Source = Source ?? Config.Storage.Legacy;
+ Destination = Destination ?? Config.Storage.Backup;
+ SourceContainer = Source.CreateCloudBlobClient().GetContainerReference(
+ String.IsNullOrEmpty(SourceContainerName) ? BlobContainerNames.LegacyPackages : SourceContainerName);
+ DestinationContainer = Destination.CreateCloudBlobClient().GetContainerReference(
+ String.IsNullOrEmpty(DestinationContainerName) ? BlobContainerNames.Backups : DestinationContainerName);
+ Log.PreparingToBackup(Source.Credentials.AccountName, SourceContainer.Name, Destination.Credentials.AccountName, DestinationContainer.Name, PackageDatabase.DataSource, PackageDatabase.InitialCatalog);
// Load package state if we aren't doing a full rescan
- Log.LoadingBackupState(destAccount.Name, DestinationContainer.Name);
+ Log.LoadingBackupState(Destination.Credentials.AccountName, DestinationContainer.Name);
var lastBackup = FullRescan ?
(DateTimeOffset?)null :
- await LoadBackupState(destAccount, DestinationContainer);
- Log.LoadedBackupState(destAccount.Name, DestinationContainer.Name);
+ await LoadBackupState(DestinationContainer);
+ Log.LoadedBackupState(Destination.Credentials.AccountName, DestinationContainer.Name);
// Gather packages
Log.GatheringListOfPackages(PackageDatabase.DataSource, PackageDatabase.InitialCatalog, lastBackup);
@@ -146,9 +145,9 @@ public BackupPackageBlobsJob(ConfigurationHub config, StorageHub storage)
// Write backup state
if (!Context.CancelToken.IsCancellationRequested)
{
- Log.SavingBackupState(destAccount.Name, DestinationContainer.Name);
- await WriteBackupState(destAccount, DestinationContainer, now);
- Log.SavedBackupState(destAccount.Name, DestinationContainer.Name);
+ Log.SavingBackupState(Destination.Credentials.AccountName, DestinationContainer.Name);
+ await WriteBackupState(DestinationContainer, now);
+ Log.SavedBackupState(Destination.Credentials.AccountName, DestinationContainer.Name);
}
}
@@ -184,7 +183,7 @@ public BackupPackageBlobsJob(ConfigurationHub config, StorageHub storage)
}
}
- private async Task WriteBackupState(StorageAccountHub destAccount, CloudBlobContainer container, DateTimeOffset now)
+ private async Task WriteBackupState(CloudBlobContainer container, DateTimeOffset now)
{
if (!WhatIf)
{
@@ -194,7 +193,7 @@ public BackupPackageBlobsJob(ConfigurationHub config, StorageHub storage)
}
}
- private async Task<DateTimeOffset?> LoadBackupState(StorageAccountHub account, CloudBlobContainer container)
+ private async Task<DateTimeOffset?> LoadBackupState(CloudBlobContainer container)
{
if (!await container.ExistsAsync())
{
@@ -210,7 +209,7 @@ public BackupPackageBlobsJob(ConfigurationHub config, StorageHub storage)
}
}
- [EventSource("Outercurve-NuGet-Jobs-BackupPackageBlobs")]
+ [EventSource(Name="Outercurve-NuGet-Jobs-BackupPackageBlobs")]
public class BackupPackageBlobsEventSource : EventSource
{
public static readonly BackupPackageBlobsEventSource Log = new BackupPackageBlobsEventSource();
View
2 src/Services/Work/NuGet.Services.Work/Jobs/CleanOnlineDatabaseBackupsJob.cs
@@ -98,7 +98,7 @@ public class CleanOnlineDatabaseBackupsJob : DatabaseJobHandlerBase<CleanOnlineD
}
}
- [EventSource("Outercurve-NuGet-Jobs-CleanOnlineDatabaseBackups")]
+ [EventSource(Name="Outercurve-NuGet-Jobs-CleanOnlineDatabaseBackups")]
public class CleanOnlineDatabaseBackupsEventSource : EventSource
{
public static readonly CleanOnlineDatabaseBackupsEventSource Log = new CleanOnlineDatabaseBackupsEventSource();
View
2 src/Services/Work/NuGet.Services.Work/Jobs/CreateOnlineDatabaseBackupJob.cs
@@ -162,7 +162,7 @@ private JobContinuation ContinueCheckingBackup()
}
}
- [EventSource("Outercurve-NuGet-Jobs-CreateOnlineDatabaseBackup")]
+ [EventSource(Name="Outercurve-NuGet-Jobs-CreateOnlineDatabaseBackup")]
public class CreateOnlineDatabaseBackupEventSource : EventSource
{
public static readonly CreateOnlineDatabaseBackupEventSource Log = new CreateOnlineDatabaseBackupEventSource();
View
653 src/Services/Work/NuGet.Services.Work/Jobs/CreateWarehouseReportsJob.cs
@@ -4,26 +4,673 @@
using System.Text;
using System.Diagnostics.Tracing;
using System.Threading.Tasks;
+using System.Data.SqlClient;
+using Dapper;
+using Microsoft.WindowsAzure.Storage;
+using NuGet.Services.Configuration;
+using Microsoft.WindowsAzure.Storage.Blob;
+using NuGet.Services.Storage;
+using NuGet.Services.Work.Helpers;
+using System.Data;
+using NuGet.Services.Client;
+using System.Diagnostics;
+using System.IO;
+using Newtonsoft.Json.Linq;
namespace NuGet.Services.Work.Jobs
{
public class CreateWarehouseReportsJob : JobHandler<CreateWarehouseReportsEventSource>
{
- protected internal override Task Execute()
+ private const string PackageReportBaseName = "recentpopularity_";
+ private const string NuGetClientVersion = "nugetclientversion";
+ private const string Last6Months = "last6months";
+ private const string RecentPopularity = "recentpopularity";
+ private const string RecentPopularityDetail = "recentpopularitydetail";
+ private const string PackageReportDetailBaseName = "recentpopularitydetail_";
+
+ private List<Func<Task>> _globalReportBuilders;
+
+ public bool RebuildAll { get; set; }
+ public SqlConnectionStringBuilder WarehouseConnection { get; set; }
+ public CloudStorageAccount Destination { get; set; }
+ public string DestinationContainerName { get; set; }
+ public string OutputDirectory { get; set; }
+
+ protected ConfigurationHub Config { get; set; }
+ protected CloudBlobContainer DestinationContainer { get; set; }
+
+
+ public CreateWarehouseReportsJob(ConfigurationHub config)
+ {
+ Config = config;
+
+ _globalReportBuilders = new List<Func<Task>>() {
+ () => CreateReport(NuGetClientVersion, "NuGet.Services.Work.Jobs.Scripts.DownloadReport_NuGetClientVersion.sql"),
+ () => CreateReport(Last6Months, "NuGet.Services.Work.Jobs.Scripts.DownloadReport_Last6Months.sql"),
+ () => CreateReport(RecentPopularity, "NuGet.Services.Work.Jobs.Scripts.DownloadReport_RecentPopularityDetail.sql"),
+ () => CreateReport(RecentPopularityDetail, "NuGet.Services.Work.Jobs.Scripts.DownloadReport_RecentPopularity.sql")
+ };
+ }
+
+ protected internal override async Task Execute()
+ {
+ LoadDefaults();
+
+ if (!String.IsNullOrEmpty(OutputDirectory))
+ {
+ Log.GeneratingReports(WarehouseConnection.DataSource, WarehouseConnection.InitialCatalog, "local file system", OutputDirectory);
+ }
+ else if (Destination != null)
+ {
+ Log.GeneratingReports(WarehouseConnection.DataSource, WarehouseConnection.InitialCatalog, Destination.Credentials.AccountName, DestinationContainer.Name);
+ }
+ else
+ {
+ throw new InvalidOperationException(Strings.CreateWarehouseReportsJob_NoDestinationAvailable);
+ }
+
+ foreach (var reportBuilder in _globalReportBuilders)
+ {
+ await reportBuilder();
+ }
+
+ if (RebuildAll)
+ {
+ await RebuildPackageReports(all: true);
+ }
+ else
+ {
+ await RebuildPackageReports(all: false);
+ await CleanInactivePackageReports();
+ }
+
+ if (!String.IsNullOrEmpty(OutputDirectory))
+ {
+ Log.GeneratedReports(WarehouseConnection.DataSource, WarehouseConnection.InitialCatalog, "local file system", OutputDirectory);
+ }
+ else
+ {
+ Log.GeneratedReports(WarehouseConnection.DataSource, WarehouseConnection.InitialCatalog, Destination.Credentials.AccountName, DestinationContainer.Name);
+ }
+ }
+
+ private async Task RebuildPackageReports(bool all)
+ {
+ IList<WarehousePackageReference> packages;
+ using (var connection = await WarehouseConnection.ConnectTo())
+ {
+ if (all)
+ {
+ Log.GettingAllPackages();
+ packages = (await connection.QueryAsync<WarehousePackageReference>("SELECT DISTINCT packageId AS PackageId, NULL as DirtyCount FROM Dimension_Package")).ToList();
+ }
+ else
+ {
+ Log.GettingPackagesInNeedOfUpdate();
+ packages = (await connection.QueryAsync<WarehousePackageReference>("GetPackagesForExport", commandType: CommandType.StoredProcedure)).ToList();
+ }
+ Log.GotPackages(packages.Count);
+ }
+
+ Parallel.ForEach(packages, new ParallelOptions() { MaxDegreeOfParallelism = 4 }, package =>
+ {
+ CreateReport(
+ PackageReportDetailBaseName + package.PackageId.ToLowerInvariant(),
+ "NuGet.Services.Work.Jobs.Scripts.DownloadReport_RecentPopularityDetailByPackage.sql",
+ t =>
+ {
+ var jobj = MakeReportJson(t);
+ TotalDownloads(jobj);
+ SortItems(jobj);
+ return Task.FromResult(jobj.ToString(JsonFormat.SerializerSettings.Formatting));
+ },
+ Tuple.Create("@PackageId", 128, package.PackageId)).Wait();
+ if (!all)
+ {
+ ConfirmPackageExport(package).Wait();
+ }
+ });
+ }
+
+ private async Task ConfirmPackageExport(WarehousePackageReference package)
+ {
+ Log.MarkingPackageExported(package.PackageId);
+ if (!WhatIf)
+ {
+ using (var connection = await WarehouseConnection.ConnectTo())
+ {
+ await connection.QueryAsync<int>(
+ "ConfirmPackageExported",
+ param: new { PackageId = package.PackageId, DirtyCount = package.DirtyCount },
+ commandType: CommandType.StoredProcedure);
+ }
+ }
+ Log.MarkedPackageExported(package.PackageId);
+ }
+
+ private async Task CleanInactivePackageReports()
+ {
+ Log.GettingInactivePackages();
+ IList<string> packageIds;
+ using (var connection = await WarehouseConnection.ConnectTo())
+ {
+ string sql = await ResourceHelpers.ReadResourceFile("NuGet.Services.Work.Jobs.Scripts.DownloadReport_ListInactive.sql");
+ packageIds = (await connection.QueryAsync<string>(sql)).ToList();
+ }
+ Log.GotInactivePackages(packageIds.Count);
+
+ // Collect the list of reports
+ Log.CollectingReportList();
+ IEnumerable<string> reports;
+ if (!String.IsNullOrEmpty(OutputDirectory))
+ {
+ reports = Directory.EnumerateFiles(OutputDirectory, PackageReportDetailBaseName + "*.json").Select(Path.GetFileNameWithoutExtension);
+ }
+ else
+ {
+ reports = DestinationContainer.ListBlobs("popularity/" + PackageReportDetailBaseName)
+ .OfType<CloudBlockBlob>()
+ .Select(b => b.Name);
+ }
+ var reportSet = new HashSet<string>(reports);
+ Log.CollectedReportList(reportSet.Count);
+
+ Parallel.ForEach(packageIds, new ParallelOptions() { MaxDegreeOfParallelism = 4 }, id =>
+ {
+ string reportName = PackageReportDetailBaseName + id;
+ if (!String.IsNullOrEmpty(OutputDirectory))
+ {
+ if(reportSet.Contains(reportName)) {
+ string fullPath = Path.Combine(OutputDirectory, reportName + ".json");
+ Log.DeletingReport(reportName, fullPath);
+ if (!WhatIf && File.Exists(fullPath))
+ {
+ File.Delete(fullPath);
+ }
+ Log.DeletedReport(reportName, fullPath);
+ }
+ }
+ else
+ {
+ string blobName = "popularity/" + reportName + ".json";
+ if(reportSet.Contains(blobName)) {
+ var blob = DestinationContainer.GetBlockBlobReference(blobName);
+ Log.DeletingReport(reportName, blob.Uri.AbsoluteUri);
+ if (!WhatIf)
+ {
+ blob.DeleteIfExists();
+ }
+ Log.DeletedReport(reportName, blob.Uri.AbsoluteUri);
+ }
+ }
+ });
+ }
+
+ private Task CreateReport(string reportName, string scriptName, params Tuple<string, int, string>[] parameters)
+ {
+ return CreateReport(reportName, scriptName, table => JsonFormat.SerializeAsync(table, camelCase: false), parameters);
+ }
+
+ private async Task CreateReport(string reportName, string scriptName, Func<DataTable, Task<string>> jsonSerializer, params Tuple<string, int, string>[] parameters)
+ {
+ Log.GeneratingSingleReport(reportName, scriptName);
+
+ DataTable table = await CollectReportData(reportName, scriptName, parameters);
+
+ // Transform the data table to JSON and process it with any provided transforms
+ Log.ProcessingReport(reportName);
+ string json = await jsonSerializer(table);
+ Log.ProcessedReport(reportName);
+
+ await WriteReport(reportName, json);
+ Log.GeneratedSingleReport(reportName, scriptName);
+ }
+
+ private async Task<DataTable> CollectReportData(string reportName, string scriptName, params Tuple<string, int, string>[] parameters)
+ {
+ Log.CollectingReportData(reportName);
+ DataTable table = null;
+ using (var connection = await WarehouseConnection.ConnectTo())
+ {
+ // Get the data
+ await WithRetry(async () =>
+ {
+ table = await ExecuteSql(scriptName, parameters);
+ });
+ }
+ Debug.Assert(table != null);
+ Log.CollectedReportData(reportName, table.Rows.Count);
+ return table;
+ }
+
+ private async Task WriteReport(string reportName, string json)
+ {
+ if (!String.IsNullOrEmpty(OutputDirectory))
+ {
+ await WriteToFile(reportName, json);
+ }
+ else
+ {
+ await WriteToBlob(reportName, json);
+ }
+ }
+
+ private async Task WriteToBlob(string reportName, string json)
+ {
+ var blob = DestinationContainer.GetBlockBlobReference("popularity/" + reportName + ".json");
+ Log.WritingReport(reportName, blob.Uri.AbsoluteUri);
+ if (!WhatIf)
+ {
+ blob.Properties.ContentType = MimeTypes.Json;
+ await blob.UploadTextAsync(json);
+ }
+ Log.WroteReport(reportName, blob.Uri.AbsoluteUri);
+ }
+
+ private async Task WriteToFile(string reportName, string json)
+ {
+ string fullPath = Path.Combine(OutputDirectory, reportName + ".json");
+ Log.WritingReport(reportName, fullPath);
+ if (!WhatIf)
+ {
+ if (!Directory.Exists(OutputDirectory))
+ {
+ Directory.CreateDirectory(OutputDirectory);
+ }
+ if (File.Exists(fullPath))
+ {
+ File.Delete(fullPath);
+ }
+ using (var writer = new StreamWriter(File.OpenWrite(fullPath)))
+ {
+ await writer.WriteAsync(json);
+ }
+ }
+ Log.WroteReport(reportName, fullPath);
+ }
+
+ private void LoadDefaults()
+ {
+ WarehouseConnection = WarehouseConnection ?? Config.Sql.Warehouse;
+ Destination = Destination ?? Config.Storage.Legacy;
+ if (Destination != null)
+ {
+ DestinationContainer = Destination.CreateCloudBlobClient().GetContainerReference(
+ String.IsNullOrEmpty(DestinationContainerName) ? BlobContainerNames.LegacyStats : DestinationContainerName);
+ }
+ }
+
+ private async Task WithRetry(Func<Task> action)
+ {
+ int attempts = 10;
+
+ while (attempts-- > 0)
+ {
+ Exception caught = null;
+ try
+ {
+ await action();
+ break;
+ }
+ catch (Exception ex)
+ {
+ if (attempts == 1)
+ {
+ throw;
+ }
+ else
+ {
+ caught = ex;
+ }
+ }
+ if (caught != null)
+ {
+ SqlConnection.ClearAllPools();
+ Log.RetryingSqlInvocation(attempts, caught.ToString());
+ await Task.Delay(20 * 1000);
+ }
+ }
+ }
+
+ // We don't use Dapper because we need a general purpose method to load any resultset
+ // This method loads a tuple where the first item is a
+ private async Task<DataTable> ExecuteSql(string scriptName, params Tuple<string, int, string>[] parameters)
{
-
+ string sql = await ResourceHelpers.ReadResourceFile(scriptName);
+
+ using (SqlConnection connection = await WarehouseConnection.ConnectTo())
+ {
+ SqlCommand command = new SqlCommand(sql, connection);
+ command.CommandType = CommandType.Text;
+ command.CommandTimeout = 60 * 5;
+
+ foreach (Tuple<string, int, string> parameter in parameters)
+ {
+ command.Parameters.Add(parameter.Item1, SqlDbType.NVarChar, parameter.Item2).Value = parameter.Item3;
+ }
+
+ var table = new DataTable();
+ var reader = await command.ExecuteReaderAsync();
+ table.Load(reader);
+ return table;
+ }
+ }
+
+ private static int TotalDownloads(JObject report)
+ {
+ JToken token;
+ if (report.TryGetValue("Items", out token))
+ {
+ if (token is JArray)
+ {
+ int total = 0;
+ for (int i = 0; i < ((JArray)token).Count; i++)
+ {
+ total += TotalDownloads((JObject)((JArray)token)[i]);
+ }
+ report["Downloads"] = total;
+ return total;
+ }
+ else
+ {
+ int total = 0;
+ foreach (KeyValuePair<string, JToken> child in ((JObject)token))
+ {
+ total += TotalDownloads((JObject)child.Value);
+ }
+ report["Downloads"] = total;
+ return total;
+ }
+ }
+ return (int)report["Downloads"];
+ }
+
+ private static void SortItems(JObject report)
+ {
+ List<Tuple<int, JObject>> scratch = new List<Tuple<int, JObject>>();
+
+ foreach (KeyValuePair<string, JToken> child in ((JObject)report["Items"]))
+ {
+ scratch.Add(new Tuple<int, JObject>((int)child.Value["Downloads"], new JObject((JObject)child.Value)));
+ }
+
+ scratch.Sort((x, y) => { return x.Item1 == y.Item1 ? 0 : x.Item1 < y.Item1 ? 1 : -1; });
+
+ JArray items = new JArray();
+
+ foreach (Tuple<int, JObject> item in scratch)
+ {
+ items.Add(item.Item2);
+ }
+
+ report["Items"] = items;
+ }
+
+ public static JObject MakeReportJson(DataTable data)
+ {
+ JObject report = new JObject();
+
+ report.Add("Downloads", 0);
+
+ JObject items = new JObject();
+
+ foreach (DataRow row in data.Rows)
+ {
+ string packageVersion = (string)row[0];
+
+ JObject childReport;
+ JToken token;
+ if (items.TryGetValue(packageVersion, out token))
+ {
+ childReport = (JObject)token;
+ }
+ else
+ {
+ childReport = new JObject();
+ childReport.Add("Downloads", 0);
+ childReport.Add("Items", new JArray());
+ childReport.Add("Version", packageVersion);
+
+ items.Add(packageVersion, childReport);
+ }
+
+ JObject obj = new JObject();
+
+ if (row[1].ToString() == "NuGet" || row[1].ToString() == "WebMatrix")
+ {
+ obj.Add("Client", string.Format("{0} {1}.{2}", row[2], row[3], row[4]));
+ obj.Add("ClientName", row[2].ToString());
+ obj.Add("ClientVersion", string.Format("{0}.{1}", row[3], row[4]));
+ }
+ else
+ {
+ obj.Add("Client", row[2].ToString());
+ obj.Add("ClientName", row[2].ToString());
+ obj.Add("ClientVersion", "");
+ }
+
+ if (row[5].ToString() != "(unknown)")
+ {
+ obj.Add("Operation", row[5].ToString());
+ }
+
+ obj.Add("Downloads", (int)row[6]);
+
+ ((JArray)childReport["Items"]).Add(obj);
+ }
+
+ report.Add("Items", items);
+
+ return report;
+ }
+
+ public class WarehousePackageReference
+ {
+ public string PackageId { get; set; }
+ public int? DirtyCount { get; set; }
}
}
- [EventSource("Outercurve-NuGet-Jobs-CreateWarehouseReports")]
+ [EventSource(Name = "Outercurve-NuGet-Jobs-CreateWarehouseReports")]
public class CreateWarehouseReportsEventSource : EventSource
{
public static readonly CreateWarehouseReportsEventSource Log = new CreateWarehouseReportsEventSource();
private CreateWarehouseReportsEventSource() { }
+ [Event(
+ eventId: 1,
+ Level = EventLevel.Informational,
+ Task = Tasks.GeneratingReports,
+ Opcode = EventOpcode.Start,
+ Message = "Generating reports from {0}/{1} and saving to {2}/{3}")]
+ public void GeneratingReports(string dbServer, string db, string storageAccount, string container) { WriteEvent(1, dbServer, db, storageAccount, container); }
+
+ [Event(
+ eventId: 2,
+ Level = EventLevel.Informational,
+ Task = Tasks.GeneratingReports,
+ Opcode = EventOpcode.Stop,
+ Message = "Generated reports from {0}/{1} and saved to {2}/{3}")]
+ public void GeneratedReports(string dbServer, string db, string storageAccount, string container) { WriteEvent(2, dbServer, db, storageAccount, container); }
+
+ [Event(
+ eventId: 3,
+ Level = EventLevel.Informational,
+ Task = Tasks.GeneratingSingleReport,
+ Opcode = EventOpcode.Start,
+ Message = "{0}: Generating report from SQL Script {1}")]
+ public void GeneratingSingleReport(string reportName, string scriptName) { WriteEvent(3, reportName, scriptName); }
+
+ [Event(
+ eventId: 4,
+ Level = EventLevel.Informational,
+ Task = Tasks.GeneratingSingleReport,
+ Opcode = EventOpcode.Stop,
+ Message = "{0}: Generated report from SQL Script {1}")]
+ public void GeneratedSingleReport(string reportName, string scriptName) { WriteEvent(4, reportName, scriptName); }
+
+ [Event(
+ eventId: 5,
+ Level = EventLevel.Informational,
+ Task = Tasks.GeneratingSingleReport,
+ Opcode = EventOpcode.Start,
+ Message = "{0}: Collecting data")]
+ public void CollectingReportData(string reportName) { WriteEvent(5, reportName); }
+
+ [Event(
+ eventId: 6,
+ Level = EventLevel.Informational,
+ Task = Tasks.GeneratingSingleReport,
+ Opcode = EventOpcode.Stop,
+ Message = "{0}: Collected {1} rows")]
+ public void CollectedReportData(string reportName, int rowCount) { WriteEvent(6, reportName, rowCount); }
+
+ [Event(
+ eventId: 7,
+ Level = EventLevel.Informational,
+ Task = Tasks.WritingReport,
+ Opcode = EventOpcode.Start,
+ Message = "{0}: Writing report to {1}")]
+ public void WritingReport(string reportName, string blobUri) { WriteEvent(7, reportName, blobUri); }
+
+ [Event(
+ eventId: 8,
+ Level = EventLevel.Informational,
+ Task = Tasks.WritingReport,
+ Opcode = EventOpcode.Stop,
+ Message = "{0}: Wrote report to {1}")]
+ public void WroteReport(string reportName, string blobUri) { WriteEvent(8, reportName, blobUri); }
+
+ [Event(
+ eventId: 9,
+ Level = EventLevel.Informational,
+ Message = "Writing reports to '{0}' instead of blobs.")]
+ public void WritingToOutputDirectory(string directory) { WriteEvent(9, directory); }
+
+ // EventID 10 was accidentally skipped. Just leave it :)
+
+ [Event(
+ eventId: 11,
+ Level = EventLevel.Informational,
+ Task = Tasks.CollectingPackageList,
+ Opcode = EventOpcode.Start,
+ Message = "Getting list of packages in need of update.")]
+ public void GettingPackagesInNeedOfUpdate() { WriteEvent(11); }
+
+ [Event(
+ eventId: 12,
+ Level = EventLevel.Informational,
+ Task = Tasks.CollectingPackageList,
+ Opcode = EventOpcode.Start,
+ Message = "Getting list of all packages.")]
+ public void GettingAllPackages() { WriteEvent(12); }
+
+ [Event(
+ eventId: 13,
+ Level = EventLevel.Informational,
+ Task = Tasks.CollectingPackageList,
+ Opcode = EventOpcode.Stop,
+ Message = "Found {0} packages to update.")]
+ public void GotPackages(int count) { WriteEvent(13, count); }
+
+ [Event(
+ eventId: 14,
+ Level = EventLevel.Informational,
+ Task = Tasks.ProcessingReport,
+ Opcode = EventOpcode.Start,
+ Message = "{0}: Processing report")]
+ public void ProcessingReport(string reportName) { WriteEvent(14, reportName); }
+
+ [Event(
+ eventId: 15,
+ Level = EventLevel.Informational,
+ Task = Tasks.ProcessingReport,
+ Opcode = EventOpcode.Stop,
+ Message = "{0}: Processed report")]
+ public void ProcessedReport(string reportName) { WriteEvent(15, reportName); }
+
+ [Event(
+ eventId: 16,
+ Level = EventLevel.Informational,
+ Task = Tasks.MarkingPackageExported,
+ Opcode = EventOpcode.Start,
+ Message = "{0}: Marking Package Exported")]
+ public void MarkingPackageExported(string packageId) { WriteEvent(16, packageId); }
+
+ [Event(
+ eventId: 17,
+ Level = EventLevel.Informational,
+ Task = Tasks.MarkingPackageExported,
+ Opcode = EventOpcode.Stop,
+ Message = "{0}: Marked Package Exported")]
+ public void MarkedPackageExported(string packageId) { WriteEvent(17, packageId); }
+
+ [Event(
+ eventId: 18,
+ Level = EventLevel.Informational,
+ Task = Tasks.CollectingInactivePackageIds,
+ Opcode = EventOpcode.Start,
+ Message = "Getting list of inactive packages.")]
+ public void GettingInactivePackages() { WriteEvent(18); }
+
+ [Event(
+ eventId: 19,
+ Level = EventLevel.Informational,
+ Task = Tasks.CollectingInactivePackageIds,
+ Opcode = EventOpcode.Stop,
+ Message = "Found {0} inactive packages.")]
+ public void GotInactivePackages(int count) { WriteEvent(19, count); }
+
+ [Event(
+ eventId: 20,
+ Level = EventLevel.Informational,
+ Message = "SQL Invocation failed, retrying. {0} attempts remaining. Exception: {1}")]
+ public void RetryingSqlInvocation(int attemptsRemaining, string exception) { WriteEvent(20, attemptsRemaining, exception); }
+
+ [Event(
+ eventId: 21,
+ Level = EventLevel.Informational,
+ Task = Tasks.DeletingReport,
+ Opcode = EventOpcode.Start,
+ Message = "{0}: Deleting empty report from {1}")]
+ public void DeletingReport(string reportName, string blobUri) { WriteEvent(21, reportName, blobUri); }
+
+ [Event(
+ eventId: 22,
+ Level = EventLevel.Informational,
+ Task = Tasks.DeletingReport,
+ Opcode = EventOpcode.Stop,
+ Message = "{0}: Deleted empty report from {1}")]
+ public void DeletedReport(string reportName, string blobUri) { WriteEvent(22, reportName, blobUri); }
+
+ [Event(
+ eventId: 23,
+ Level = EventLevel.Informational,
+ Task = Tasks.CollectingReportList,
+ Opcode = EventOpcode.Start,
+ Message = "Collecting list of package detail reports")]
+ public void CollectingReportList() { WriteEvent(23); }
+
+ [Event(
+ eventId: 24,
+ Level = EventLevel.Informational,
+ Task = Tasks.CollectingReportList,
+ Opcode = EventOpcode.Stop,
+ Message = "Collected {0} package detail reports")]
+ public void CollectedReportList(int reportCount) { WriteEvent(24, reportCount); }
+
public static class Tasks
{
public const EventTask GeneratingReports = (EventTask)0x1;
+ public const EventTask GeneratingSingleReport = (EventTask)0x2;
+ public const EventTask CollectingReportData = (EventTask)0x3;
+ public const EventTask WritingReport = (EventTask)0x4;
+ public const EventTask CollectingPackageList = (EventTask)0x5;
+ public const EventTask ProcessingReport = (EventTask)0x6;
+ public const EventTask MarkingPackageExported = (EventTask)0x7;
+ public const EventTask CollectingInactivePackageIds = (EventTask)0x8;
+ public const EventTask DeletingReport = (EventTask)0x9;
+ public const EventTask CollectingReportList = (EventTask)0xA;
}
}
}
View
25 src/Services/Work/NuGet.Services.Work/Jobs/HandlePackageEditsJob.cs
@@ -13,6 +13,7 @@
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Dapper;
+using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using NuGet.Services.Configuration;
using NuGet.Services.Storage;
@@ -35,7 +36,8 @@ public class HandlePackageEditsJob : JobHandler<HandlePackageEditsEventSource>
/// <summary>
/// Gets or sets an Azure Storage Uri referring to a container to use as the source for package blobs
/// </summary>
- public AzureStorageReference Source { get; set; }
+ public CloudStorageAccount Source { get; set; }
+ public string SourceContainerName { get; set; }
/// <summary>
/// Gets or sets a connection string to the database containing package data.
@@ -45,7 +47,8 @@ public class HandlePackageEditsJob : JobHandler<HandlePackageEditsEventSource>
/// <summary>
/// Gets or sets an Azure Storage Uri referring to a container to use as the backup storage for package blobs
/// </summary>
- public AzureStorageReference Backups { get; set; }
+ public CloudStorageAccount Backups { get; set; }
+ public string BackupsContainerName { get; set; }
/// <summary>
/// Gets or sets the maximum number of tries that are allowed before considering an edit failed.
@@ -72,16 +75,12 @@ public HandlePackageEditsJob(StorageHub storage, ConfigurationHub config)
// Load defaults
MaxManifestSize = MaxAllowedManifestBytes ?? DefaultMaxAllowedManifestBytes;
PackageDatabase = PackageDatabase ?? Config.Sql.GetConnectionString(KnownSqlServer.Legacy);
- var sourceAccount = Source == null ?
- Storage.Legacy :
- Storage.GetAccount(Source);
- var backupsAccount = Backups == null ?
- Storage.Backup :
- Storage.GetAccount(Backups);
- SourceContainer = sourceAccount.Blobs.Client.GetContainerReference(
- Source == null ? PackageHelpers.PackageBlobContainer : Source.Container);
- BackupsContainer = backupsAccount.Blobs.Client.GetContainerReference(
- Backups == null ? PackageHelpers.BackupsBlobContainer : Backups.Container);
+ Source = Source ?? Storage.Legacy.Account;
+ Backups = Backups ?? Storage.Backup.Account;
+ SourceContainer = Source.CreateCloudBlobClient().GetContainerReference(
+ String.IsNullOrEmpty(SourceContainerName) ? BlobContainerNames.LegacyPackages : SourceContainerName);
+ BackupsContainer = Backups.CreateCloudBlobClient().GetContainerReference(
+ String.IsNullOrEmpty(BackupsContainerName) ? BlobContainerNames.Backups : BackupsContainerName);
// Grab package edits
IList<PackageEdit> edits;
@@ -375,7 +374,7 @@ FROM PackageAuthors
}
}
- [EventSource("Outercurve-NuGet-Jobs-HandlePackageEdits")]
+ [EventSource(Name="Outercurve-NuGet-Jobs-HandlePackageEdits")]
public class HandlePackageEditsEventSource : EventSource
{
public static readonly HandlePackageEditsEventSource Log = new HandlePackageEditsEventSource();
View
2 src/Services/Work/NuGet.Services.Work/Jobs/PurgeCompletedInvocationsJob.cs
@@ -40,7 +40,7 @@ public class PurgeCompletedInvocationsJob : JobHandlerBase<PurgeCompletedInvocat
}
}
- [EventSource("Outercurve-NuGet-Jobs-PurgeCompletedInvocations")]
+ [EventSource(Name="Outercurve-NuGet-Jobs-PurgeCompletedInvocations")]
public class PurgeCompletedInvocationsEventSource : EventSource
{
public static readonly PurgeCompletedInvocationsEventSource Log = new PurgeCompletedInvocationsEventSource();
View
2 src/Services/Work/NuGet.Services.Work/Jobs/PurgePackageStatisticsJob.cs
@@ -87,7 +87,7 @@ public PurgePackageStatisticsJob(ConfigurationHub config)
}
}
- [EventSource("Outercurve-NuGet-Jobs-PurgePackageStatistics")]
+ [EventSource(Name="Outercurve-NuGet-Jobs-PurgePackageStatistics")]
public class PurgePackageStatisticsEventSource : EventSource
{
public static readonly PurgePackageStatisticsEventSource Log = new PurgePackageStatisticsEventSource();
View
2 src/Services/Work/NuGet.Services.Work/Jobs/ReplicatePackageStatisticsJob.cs
@@ -232,7 +232,7 @@ private static string GetNullableField(SqlDataReader reader, int ordinal)
}
}
- [EventSource("Outercurve-NuGet-Jobs-ReplicatePackageStatistics")]
+ [EventSource(Name="Outercurve-NuGet-Jobs-ReplicatePackageStatistics")]
public class ReplicatePackageStatisticsEventSource : EventSource
{
public static readonly ReplicatePackageStatisticsEventSource Log = new ReplicatePackageStatisticsEventSource();
View
16 src/Services/Work/NuGet.Services.Work/Jobs/Scripts/DownloadReport_Last6Months.sql
@@ -0,0 +1,16 @@
+
+SELECT Dimension_Date.[Year], Dimension_Date.MonthOfYear, SUM(DownloadCount) 'Downloads'
+FROM Fact_Download
+INNER JOIN Dimension_Date ON Dimension_Date.Id = Fact_Download.Dimension_Date_Id
+WHERE Dimension_Date.[Date] >=
+ DATETIMEFROMPARTS(
+ DATEPART(year, DATEADD(month, -7, GETDATE())),
+ DATEPART(month, DATEADD(month, -7, GETDATE())),
+ 1, 0, 0, 0, 0)
+AND Dimension_Date.[Date] <
+ DATETIMEFROMPARTS(
+ DATEPART(year, GETDATE()),
+ DATEPART(month, GETDATE()),
+ 1, 0, 0, 0, 0)
+GROUP BY Dimension_Date.[Year], Dimension_Date.MonthOfYear
+ORDER BY Dimension_Date.[Year], Dimension_Date.MonthOfYear
View
10 src/Services/Work/NuGet.Services.Work/Jobs/Scripts/DownloadReport_ListInactive.sql
@@ -0,0 +1,10 @@
+
+SELECT DISTINCT Dimension_Package.PackageId
+FROM Dimension_Package
+WHERE Dimension_Package.PackageId NOT IN (
+ SELECT DISTINCT Dimension_Package.PackageId
+ FROM Fact_Download
+ INNER JOIN Dimension_Package ON Dimension_Package.Id = Fact_Download.Dimension_Package_Id
+ INNER JOIN Dimension_Date ON Dimension_Date.Id = Fact_Download.Dimension_Date_Id
+ WHERE Dimension_Date.[Date] >= CONVERT(DATE, DATEADD(day, -42, GETDATE()))
+ AND Dimension_Date.[Date] < CONVERT(DATE, GETDATE()))
View
12 src/Services/Work/NuGet.Services.Work/Jobs/Scripts/DownloadReport_NuGetClientVersion.sql
@@ -0,0 +1,12 @@
+
+SELECT Dimension_UserAgent.ClientMajorVersion, Dimension_UserAgent.ClientMinorVersion, SUM(DownloadCount) 'Downloads'
+FROM Fact_Download
+INNER JOIN Dimension_UserAgent ON Dimension_UserAgent.Id = Fact_Download.Dimension_UserAgent_Id
+INNER JOIN Dimension_Date ON Dimension_Date.Id = Fact_Download.Dimension_Date_Id
+WHERE Dimension_Date.[Date] >= CONVERT(DATE, DATEADD(day, -42, GETDATE()))
+ AND Dimension_Date.[Date] < CONVERT(DATE, GETDATE())
+ AND Dimension_UserAgent.ClientCategory = 'NuGet'
+ AND Dimension_UserAgent.ClientMajorVersion <= 2
+ AND Dimension_UserAgent.ClientMinorVersion <= 7
+GROUP BY Dimension_UserAgent.ClientMajorVersion, Dimension_UserAgent.ClientMinorVersion
+ORDER BY Dimension_UserAgent.ClientMajorVersion, Dimension_UserAgent.ClientMinorVersion
View
10 src/Services/Work/NuGet.Services.Work/Jobs/Scripts/DownloadReport_RecentPopularity.sql
@@ -0,0 +1,10 @@
+
+SELECT TOP(100) Dimension_Package.PackageId, SUM(DownloadCount) 'Downloads'
+FROM Fact_Download
+INNER JOIN Dimension_Package ON Dimension_Package.Id = Fact_Download.Dimension_Package_Id
+INNER JOIN Dimension_Date ON Dimension_Date.Id = Fact_Download.Dimension_Date_Id
+WHERE Dimension_Date.[Date] >= CONVERT(DATE, DATEADD(day, -42, GETDATE()))
+ AND Dimension_Date.[Date] < CONVERT(DATE, GETDATE())
+ AND Dimension_Package.PackageListed = 1
+GROUP BY Dimension_Package.PackageId
+ORDER BY SUM(DownloadCount) DESC
View
9 ...rvices/Work/NuGet.Services.Work/Jobs/Scripts/DownloadReport_RecentPopularityByPackage.sql
@@ -0,0 +1,9 @@
+SELECT Dimension_Package.PackageVersion, SUM(DownloadCount) 'Downloads'
+FROM Fact_Download
+INNER JOIN Dimension_Package ON Dimension_Package.Id = Fact_Download.Dimension_Package_Id
+INNER JOIN Dimension_Date ON Dimension_Date.Id = Fact_Download.Dimension_Date_Id
+WHERE Dimension_Date.[Date] >= CONVERT(DATE, DATEADD(day, -42, GETDATE()))
+ AND Dimension_Date.[Date] < CONVERT(DATE, GETDATE())
+ AND Dimension_Package.PackageId = @PackageId
+GROUP BY Dimension_Package.PackageVersion
+ORDER BY SUM(DownloadCount) DESC
View
21 src/Services/Work/NuGet.Services.Work/Jobs/Scripts/DownloadReport_RecentPopularityDetail.sql
@@ -0,0 +1,21 @@
+
+SELECT TOP(500)
+ Dimension_Package.PackageId,
+ Dimension_Package.PackageVersion,
+ Dimension_Package.PackageTitle,
+ Dimension_Package.PackageDescription,
+ Dimension_Package.PackageIconUrl,
+ SUM(DownloadCount) 'Downloads'
+FROM Fact_Download
+INNER JOIN Dimension_Package ON Dimension_Package.Id = Fact_Download.Dimension_Package_Id
+INNER JOIN Dimension_Date ON Dimension_Date.Id = Fact_Download.Dimension_Date_Id
+WHERE Dimension_Date.[Date] >= CONVERT(DATE, DATEADD(day, -42, GETDATE()))
+ AND Dimension_Date.[Date] < CONVERT(DATE, GETDATE())
+ AND Dimension_Package.PackageListed = 1
+GROUP BY
+ Dimension_Package.PackageId,
+ Dimension_Package.PackageVersion,
+ Dimension_Package.PackageTitle,
+ Dimension_Package.PackageDescription,
+ Dimension_Package.PackageIconUrl
+ORDER BY SUM(DownloadCount) DESC
View
31 .../Work/NuGet.Services.Work/Jobs/Scripts/DownloadReport_RecentPopularityDetailByPackage.sql
@@ -0,0 +1,31 @@
+SELECT
+ Dimension_Package.PackageVersion,
+ Dimension_UserAgent.ClientCategory,
+ Dimension_UserAgent.Client,
+ Dimension_UserAgent.ClientMajorVersion,
+ Dimension_UserAgent.ClientMinorVersion,
+ Dimension_Operation.Operation,
+ SUM(DownloadCount) 'Downloads'
+FROM Fact_Download
+INNER JOIN Dimension_Package ON Dimension_Package.Id = Fact_Download.Dimension_Package_Id
+INNER JOIN Dimension_Date ON Dimension_Date.Id = Fact_Download.Dimension_Date_Id
+INNER JOIN Dimension_Operation ON Dimension_Operation.Id = Fact_Download.Dimension_Operation_Id
+INNER JOIN Dimension_UserAgent ON Dimension_UserAgent.Id = Fact_Download.Dimension_UserAgent_Id
+WHERE Dimension_Date.[Date] >= CONVERT(DATE, DATEADD(day, -42, GETDATE()))
+ AND Dimension_Date.[Date] < CONVERT(DATE, GETDATE())
+ AND Dimension_Package.PackageId = @PackageId
+GROUP BY
+ Dimension_Package.PackageVersion,
+ Dimension_UserAgent.Client,
+ Dimension_UserAgent.ClientCategory,
+ Dimension_UserAgent.ClientMajorVersion,
+ Dimension_UserAgent.ClientMinorVersion,
+ Dimension_Operation.Operation
+ORDER BY
+ Dimension_Package.PackageVersion,
+ Dimension_UserAgent.Client,
+ Dimension_UserAgent.ClientCategory,
+ Dimension_UserAgent.ClientMajorVersion,
+ Dimension_UserAgent.ClientMinorVersion,
+ Dimension_Operation.Operation,
+ SUM(DownloadCount) DESC
View
2 src/Services/Work/NuGet.Services.Work/Jobs/TestAsyncJob.cs
@@ -30,7 +30,7 @@ protected internal override Task<JobContinuation> Execute()
}
}
- [EventSource("Outercurve-NuGet-Jobs-TestAsync")]
+ [EventSource(Name="Outercurve-NuGet-Jobs-TestAsync")]
public class TestAsyncEventSource : EventSource
{
public static readonly TestAsyncEventSource Log = new TestAsyncEventSource();
View
2 src/Services/Work/NuGet.Services.Work/Jobs/TestHeartBeatJob.cs
@@ -26,7 +26,7 @@ protected internal override Task Execute()
}
}
- [EventSource("Outercurve-NuGet-Jobs-TestHeartBeat")]
+ [EventSource(Name="Outercurve-NuGet-Jobs-TestHeartBeat")]
public class TestHeartBeatEventSource : EventSource
{
public static readonly TestHeartBeatEventSource Log = new TestHeartBeatEventSource();
View
2 src/Services/Work/NuGet.Services.Work/Jobs/TestLongRunningJob.cs
@@ -28,7 +28,7 @@ public class TestLongRunningJob : JobHandler<TestLongRunningEventSource>
}
}
- [EventSource("Outercurve-NuGet-Jobs-TestLongRunning")]
+ [EventSource(Name="Outercurve-NuGet-Jobs-TestLongRunning")]
public class TestLongRunningEventSource : EventSource
{
public static readonly TestLongRunningEventSource Log = new TestLongRunningEventSource();
View
2 src/Services/Work/NuGet.Services.Work/Jobs/TestPingJob.cs
@@ -21,7 +21,7 @@ protected internal override Task Execute()
}
}
- [EventSource("Outercurve-NuGet-Jobs-TestPing")]
+ [EventSource(Name="Outercurve-NuGet-Jobs-TestPing")]
public class TestPingEventSource : EventSource
{
public static readonly TestPingEventSource Log = new TestPingEventSource();
View
2 src/Services/Work/NuGet.Services.Work/Jobs/UpdateLicenseReportsJob.cs
@@ -339,7 +339,7 @@ public override string ToString()
}
}
- [EventSource("Outercurve-NuGet-Jobs-UpdateLicenseReports")]
+ [EventSource(Name="Outercurve-NuGet-Jobs-UpdateLicenseReports")]
public class UpdateLicenseReportsEventSource : EventSource
{
public static readonly UpdateLicenseReportsEventSource Log = new UpdateLicenseReportsEventSource();
View
2 src/Services/Work/NuGet.Services.Work/Monitoring/InvocationEventSource.cs
@@ -8,7 +8,7 @@
namespace NuGet.Services.Work.Monitoring
{
- [EventSource("Outercurve-NuGet-Work-Invocations")]
+ [EventSource(Name="Outercurve-NuGet-Work-Invocations")]
public class InvocationEventSource : EventSource
{
public static readonly InvocationEventSource Log = new InvocationEventSource();
View
2 src/Services/Work/NuGet.Services.Work/Monitoring/WorkServiceEventSource.cs
@@ -7,7 +7,7 @@
namespace NuGet.Services.Work.Monitoring
{
- [EventSource("Outercurve-NuGet-Work-Service")]
+ [EventSource(Name="Outercurve-NuGet-Work-Service")]
public class WorkServiceEventSource : EventSource
{
public static readonly WorkServiceEventSource Log = new WorkServiceEventSource();
View
15 src/Services/Work/NuGet.Services.Work/NuGet.Services.Work.csproj
@@ -154,6 +154,7 @@
<Compile Include="Helpers\DateTimeOffsetExtensions.cs" />
<Compile Include="Helpers\EventSourceInstanceManager.cs" />
<Compile Include="Helpers\PackageHelpers.cs" />
+ <Compile Include="Helpers\ResourceHelpers.cs" />
<Compile Include="Infrastructure\InvalidJobRequestException.cs" />
<Compile Include="Infrastructure\InvocationState.cs" />
<Compile Include="Infrastructure\InvocationQueue.cs" />
@@ -233,13 +234,13 @@
</ProjectReference>
</ItemGroup>
<ItemGroup>
- <EmbeddedResource Include="Jobs\Sql\DownloadReport_Last6Months.sql" />
- <EmbeddedResource Include="Jobs\Sql\DownloadReport_ListInactive.sql" />
- <EmbeddedResource Include="Jobs\Sql\DownloadReport_NuGetClientVersion.sql" />
- <EmbeddedResource Include="Jobs\Sql\DownloadReport_RecentPopularity.sql" />
- <EmbeddedResource Include="Jobs\Sql\DownloadReport_RecentPopularityByPackage.sql" />
- <EmbeddedResource Include="Jobs\Sql\DownloadReport_RecentPopularityDetail.sql" />
- <EmbeddedResource Include="Jobs\Sql\DownloadReport_RecentPopularityDetailByPackage.sql" />
+ <EmbeddedResource Include="Jobs\Scripts\DownloadReport_Last6Months.sql" />
+ <EmbeddedResource Include="Jobs\Scripts\DownloadReport_ListInactive.sql" />
+ <EmbeddedResource Include="Jobs\Scripts\DownloadReport_NuGetClientVersion.sql" />
+ <EmbeddedResource Include="Jobs\Scripts\DownloadReport_RecentPopularity.sql" />
+ <EmbeddedResource Include="Jobs\Scripts\DownloadReport_RecentPopularityByPackage.sql" />
+ <EmbeddedResource Include="Jobs\Scripts\DownloadReport_RecentPopularityDetail.sql" />
+ <EmbeddedResource Include="Jobs\Scripts\DownloadReport_RecentPopularityDetailByPackage.sql" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), 'NuGetGallery.v3.sln'))\build\NuGet.targets" />
View
9 src/Services/Work/NuGet.Services.Work/Strings.Designer.cs
@@ -61,6 +61,15 @@ internal class Strings {
}
/// <summary>
+ /// Looks up a localized string similar to One of Destination or OutputDirectory must be specified or available from the environment.
+ /// </summary>
+ internal static string CreateWarehouseReportsJob_NoDestinationAvailable {
+ get {
+ return ResourceManager.GetString("CreateWarehouseReportsJob_NoDestinationAvailable", resourceCulture);
+ }
+ }
+
+ /// <summary>
/// Looks up a localized string similar to The Event Source &apos;{0}&apos; does not have a public static field &quot;Log&quot; containing an instance of it. Event Sources must implement this pattern for all jobs..
/// </summary>
internal static string EventSourceInstanceManager_EventSourceDoesNotHaveLogField {
View
3 src/Services/Work/NuGet.Services.Work/Strings.resx
@@ -117,6 +117,9 @@
<resheader name="writer">
<value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</resheader>
+ <data name="CreateWarehouseReportsJob_NoDestinationAvailable" xml:space="preserve">
+ <value>One of Destination or OutputDirectory must be specified or available from the environment</value>
+ </data>
<data name="EventSourceInstanceManager_EventSourceDoesNotHaveLogField" xml:space="preserve">
<value>The Event Source '{0}' does not have a public static field "Log" containing an instance of it. Event Sources must implement this pattern for all jobs.</value>
</data>
View
11 src/Services/Work/NuGet.Services.Work/WorkService.cs
@@ -186,7 +186,14 @@ public static IEnumerable<JobDescription> GetAllAvailableJobs()
public IObservable<EventEntry> RunJob(string job, string payload)
{
- var runner = Container.Resolve<JobRunner>();
+ var runner = new JobRunner(
+ new JobDispatcher(
+ GetAllAvailableJobs(),
+ Container),
+ InvocationQueue.Null,
+ Container.Resolve<ConfigurationHub>(),
+ Container.Resolve<StorageHub>(),
+ Clock.RealClock);
var invocation =
new InvocationState(
@@ -203,7 +210,7 @@ public IObservable<EventEntry> RunJob(string job, string payload)
QueuedAt = DateTime.UtcNow,
NextVisibleAt = DateTime.UtcNow + TimeSpan.FromMinutes(5)
});
- var buffer = new ReplaySubject<EventEntry>(bufferSize: 1);
+ var buffer = new ReplaySubject<EventEntry>();
var capture = new InvocationLogCapture(invocation);
capture.Subscribe(buffer.OnNext, buffer.OnError);
runner.Dispatch(invocation, capture, CancellationToken.None).ContinueWith(t =>

0 comments on commit 9899c1a

Please sign in to comment.