Merge pull request #26 from NuGet/johtaylo-25

initial commit fixed #25
NuGet · Apr 17, 2013 · 21434b9 · 21434b9
2 parents 4a6ce91 + b4aa909
commit 21434b9
Show file tree

Hide file tree

Showing 12 changed files with 175 additions and 101 deletions.
diff --git a/Build-Solution.ps1 b/Build-Solution.ps1
@@ -1,5 +1,5 @@
 param(
-    [Parameter(Mandatory=$false)][string]$Configuration = "Release")
+    [Parameter(Mandatory=$false)][string]$Configuration = "Debug")
 
 $MyPath = Split-Path $MyInvocation.MyCommand.Path
 $ScriptsDir = Join-Path $MyPath Scripts

diff --git a/Source/NuGetGallery.Operations/Common/ReportHelpers.cs b/Source/NuGetGallery.Operations/Common/ReportHelpers.cs
@@ -17,17 +17,21 @@ public static Stream ToStream(JToken jToken)
             return stream;
         }
 
-        public static Stream ToJson(Tuple<string[], List<string[]>> report)
+        public static Stream ToJson(Tuple<string[], List<object[]>> report)
         {
             JArray jArray = new JArray();
 
-            foreach (string[] row in report.Item2)
+            foreach (object[] row in report.Item2)
             {
                 JObject jObject = new JObject();
 
                 for (int i = 0; i < report.Item1.Length; i++)
                 {
-                    jObject.Add(report.Item1[i], row[i]);
+                    if (row[i] != null)
+                    {
+                        jObject.Add(report.Item1[i], new JValue(row[i]));
+                    }
+                    // ELSE treat null by not defining the property in our internal JSON (aka undefined)
                 }
 
                 jArray.Add(jObject);

diff --git a/Source/NuGetGallery.Operations/NuGetGallery.Operations.csproj b/Source/NuGetGallery.Operations/NuGetGallery.Operations.csproj
@@ -232,6 +232,9 @@
   <ItemGroup>
     <EmbeddedResource Include="Scripts\DownloadReport_ListInactive.sql" />
   </ItemGroup>
+  <ItemGroup>
+    <EmbeddedResource Include="Scripts\NuGetDownloadsAlterTable.sql" />
+  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <Import Project="$(SolutionDir)\.nuget\nuget.targets" />
 </Project>
diff --git a/Source/NuGetGallery.Operations/Scripts/DownloadReport_RecentPopularity.sql b/Source/NuGetGallery.Operations/Scripts/DownloadReport_RecentPopularity.sql
@@ -5,5 +5,6 @@ INNER JOIN Dimension_Package ON Dimension_Package.Id = Fact_Download.Dimension_P
 INNER JOIN Dimension_Date ON Dimension_Date.Id = Fact_Download.Dimension_Date_Id
 WHERE Dimension_Date.[Date] >= CONVERT(DATE, DATEADD(day, -42, GETDATE()))
   AND Dimension_Date.[Date] < CONVERT(DATE, GETDATE())
+  AND Dimension_Package.PackageListed = 1
 GROUP BY Dimension_Package.PackageId
 ORDER BY SUM(DownloadCount) DESC
diff --git a/Source/NuGetGallery.Operations/Scripts/DownloadReport_RecentPopularityDetail.sql b/Source/NuGetGallery.Operations/Scripts/DownloadReport_RecentPopularityDetail.sql
@@ -1,9 +1,21 @@
 
-SELECT TOP(500) Dimension_Package.PackageId, Dimension_Package.PackageVersion, SUM(DownloadCount) 'Downloads'
+SELECT TOP(500) 
+	Dimension_Package.PackageId,
+	Dimension_Package.PackageVersion,
+	Dimension_Package.PackageTitle,
+	Dimension_Package.PackageDescription,
+	Dimension_Package.PackageIconUrl,
+	SUM(DownloadCount) 'Downloads'
 FROM Fact_Download
 INNER JOIN Dimension_Package ON Dimension_Package.Id = Fact_Download.Dimension_Package_Id
 INNER JOIN Dimension_Date ON Dimension_Date.Id = Fact_Download.Dimension_Date_Id
 WHERE Dimension_Date.[Date] >= CONVERT(DATE, DATEADD(day, -42, GETDATE()))
   AND Dimension_Date.[Date] < CONVERT(DATE, GETDATE())
-GROUP BY Dimension_Package.PackageId, Dimension_Package.PackageVersion
+  AND Dimension_Package.PackageListed = 1
+GROUP BY 
+	Dimension_Package.PackageId, 
+	Dimension_Package.PackageVersion,
+	Dimension_Package.PackageTitle,
+	Dimension_Package.PackageDescription,
+	Dimension_Package.PackageIconUrl
 ORDER BY SUM(DownloadCount) DESC
diff --git a/Source/NuGetGallery.Operations/Scripts/NuGetDownloadsAlterTable.sql b/Source/NuGetGallery.Operations/Scripts/NuGetDownloadsAlterTable.sql
@@ -0,0 +1,17 @@
+
+/*  you can use this script manually against an existing warehouse
+ *  for new warehouse instances, these schema changes have been incorporated into the create script 
+ */
+
+ALTER TABLE [dbo].[Dimension_Package] ADD [PackageListed] BIT NOT NULL DEFAULT 1
+GO
+
+ALTER TABLE [dbo].[Dimension_Package] ADD [PackageTitle] NVARCHAR(256)
+GO
+
+ALTER TABLE [dbo].[Dimension_Package] ADD [PackageDescription] NVARCHAR(MAX)
+GO
+
+ALTER TABLE [dbo].[Dimension_Package] ADD [PackageIconUrl] NVARCHAR(MAX)
+GO
+
diff --git a/Source/NuGetGallery.Operations/Scripts/NuGetDownloadsCreateTables.sql b/Source/NuGetGallery.Operations/Scripts/NuGetDownloadsCreateTables.sql
@@ -32,9 +32,13 @@ GO
 IF OBJECT_ID('[dbo].[Dimension_Package]') IS NULL
     CREATE TABLE [dbo].[Dimension_Package]
     (
-        [Id]				INT IDENTITY,
-        [PackageId]			NVARCHAR(128),
-        [PackageVersion]	NVARCHAR(64),
+        [Id]				 INT IDENTITY,
+        [PackageId]			 NVARCHAR(128),
+        [PackageVersion]	 NVARCHAR(64),
+        [PackageListed]      BIT,
+        [PackageTitle]       NVARCHAR(256),
+        [PackageDescription] NVARCHAR(MAX),
+        [PackageIconUrl]     NVARCHAR(MAX),
         CONSTRAINT [PK_Dimension_Package] PRIMARY KEY CLUSTERED ([Id])
     )
 GO
@@ -134,7 +138,7 @@ IF NOT EXISTS (SELECT * FROM sys.indexes WHERE name = 'Fact_Download_NCI_Downloa
 GO
 
 IF NOT EXISTS (SELECT * FROM sys.indexes WHERE name = 'Fact_Download_NCI_Package_Id')
-	CREATE NONCLUSTERED INDEX [Fact_Download_NCI_Package_Id] ON [dbo].[Fact_Download] ( [Dimension_Package_Id] ) INCLUDE ( [Dimension_Date_Id], [DownloadCount] )
+	CREATE NONCLUSTERED INDEX [Fact_Download_NCI_Package_Id] ON [dbo].[Fact_Download] ( [Dimension_Package_Id] ) INCLUDE ( [Dimension_UserAgent_Id], [Dimension_Date_Id], [Dimension_Operation_Id], [DownloadCount] )
 GO
 
 IF NOT EXISTS (SELECT * FROM sys.indexes WHERE name = 'Fact_Download_NCI_Date_Id')

diff --git a/Source/NuGetGallery.Operations/Scripts/NuGetDownloadsProcs_AddDownloadFact.sql b/Source/NuGetGallery.Operations/Scripts/NuGetDownloadsProcs_AddDownloadFact.sql
@@ -6,6 +6,10 @@ GO
 CREATE PROCEDURE [dbo].[AddDownloadFact]
 @PackageId NVARCHAR(128),
 @PackageVersion NVARCHAR(64),
+@PackageListed INT,
+@PackageTitle NVARCHAR(256),
+@PackageDescription NVARCHAR(MAX),
+@PackageIconUrl NVARCHAR(MAX),
 @DownloadUserAgent NVARCHAR(max),
 @DownloadOperation NVARCHAR(16),
 @DownloadTimestamp DATETIME,
@@ -30,11 +34,37 @@ BEGIN
 
         IF (@Dimension_PackageId IS NULL)
         BEGIN
-            INSERT Dimension_Package ( PackageId, PackageVersion )
-            VALUES ( @PackageId, @PackageVersion );
+            INSERT Dimension_Package
+			( 
+				PackageId,
+				PackageVersion,
+				PackageListed,
+				PackageTitle,
+				PackageDescription,
+				PackageIconUrl
+			)
+            VALUES 
+			( 
+				@PackageId,
+				@PackageVersion,
+				@PackageListed,
+				@PackageTitle,
+				@PackageDescription,
+				@PackageIconUrl
+			);
 
             SELECT @Dimension_PackageId = SCOPE_IDENTITY();
         END
+		ELSE
+		BEGIN
+            UPDATE Dimension_Package
+			SET 
+				PackageListed = @PackageListed,
+				PackageTitle = @PackageTitle,
+				PackageDescription = @PackageDescription,
+				PackageIconUrl = @PackageIconUrl
+			WHERE Id = @Dimension_PackageId
+		END
 
         DECLARE @Dimension_UserAgentId INT;
 

diff --git a/Source/NuGetGallery.Operations/Tasks/CreateWarehouseReportsTask.cs b/Source/NuGetGallery.Operations/Tasks/CreateWarehouseReportsTask.cs
@@ -26,13 +26,12 @@ public class CreateWarehouseReportsTask : ReportsTask
         [Option("Re-create all reports", AltName = "all")]
         public bool All { get; set; }
 
-        [Option("Re-create just detail reports", AltName = "new")]
-        public bool New { get; set; }
-
         public override void ExecuteCommand()
         {
             Log.Info("Generate reports begin");
 
+            CreateContainerIfNotExists();
+
             CreateReport_PerMonth();
             CreateReport_RecentPopularityDetail();
             CreateReport_RecentPopularity();
@@ -54,7 +53,7 @@ private void CreateReport_PerMonth()
         {
             Log.Info("CreateReport_PerMonth");
 
-            Tuple<string[], List<string[]>> report = ExecuteSql("NuGetGallery.Operations.Scripts.DownloadReport_PerMonth.sql");
+            Tuple<string[], List<object[]>> report = ExecuteSql("NuGetGallery.Operations.Scripts.DownloadReport_PerMonth.sql");
 
             CreateBlob(PerMonth + ".json", JsonContentType, ReportHelpers.ToJson(report));
         }
@@ -63,7 +62,7 @@ private void CreateReport_RecentPopularityDetail()
         {
             Log.Info("CreateReport_RecentPopularityDetail");
 
-            Tuple<string[], List<string[]>> report = ExecuteSql("NuGetGallery.Operations.Scripts.DownloadReport_RecentPopularityDetail.sql");
+            Tuple<string[], List<object[]>> report = ExecuteSql("NuGetGallery.Operations.Scripts.DownloadReport_RecentPopularityDetail.sql");
 
             CreateBlob(RecentPopularityDetail + ".json", JsonContentType, ReportHelpers.ToJson(report));
         }
@@ -72,14 +71,14 @@ private void CreateReport_RecentPopularity()
         {
             Log.Info("CreateReport_RecentPopularity");
 
-            Tuple<string[], List<string[]>> report = ExecuteSql("NuGetGallery.Operations.Scripts.DownloadReport_RecentPopularity.sql");
+            Tuple<string[], List<object[]>> report = ExecuteSql("NuGetGallery.Operations.Scripts.DownloadReport_RecentPopularity.sql");
 
             CreateBlob(RecentPopularity + ".json", JsonContentType, ReportHelpers.ToJson(report));
 
             CreatePerPackageReports(report);
         }
 
-        private void CreatePerPackageReports(Tuple<string[], List<string[]>> report)
+        private void CreatePerPackageReports(Tuple<string[], List<object[]>> report)
         {
             Log.Info(string.Format("CreatePerPackageReports (count = {0})", report.Item2.Count));
 
@@ -98,9 +97,9 @@ private void CreatePerPackageReports(Tuple<string[], List<string[]>> report)
                 throw new InvalidOperationException("expected PackageId in result");
             }
 
-            foreach (string[] row in report.Item2)
+            foreach (object[] row in report.Item2)
             {
-                string packageId = row[indexOfPackageId];
+                string packageId = row[indexOfPackageId].ToString();
                 WithRetry(() =>
                 {
                     CreatePackageReport(packageId);
@@ -234,34 +233,6 @@ private void CreatePackageReport(string packageId)
         {
             Log.Info(string.Format("CreatePackageReport for {0}", packageId));
 
-            if (New)
-            {
-                CreatePackageReportNew(packageId);
-            }
-            else
-            {
-                CreatePackageReportOld(packageId);
-                CreatePackageReportNew(packageId);
-            }
-        }
-
-        private void CreatePackageReportOld(string packageId)
-        {
-            Log.Info(string.Format("CreatePackageReportOld for {0}", packageId));
-
-            // All blob names use lower case identifiers in the NuGet Gallery Azure Blob Storage 
-
-            string name = PackageReportBaseName + packageId.ToLowerInvariant();
-
-            Tuple<string[], List<string[]>> report = ExecuteSql("NuGetGallery.Operations.Scripts.DownloadReport_RecentPopularityByPackage.sql", new Tuple<string, int, string>("@packageId", 128, packageId));
-
-            CreateBlob(name + ".json", JsonContentType, ReportHelpers.ToJson(report));
-        }
-
-        private void CreatePackageReportNew(string packageId)
-        {
-            Log.Info(string.Format("CreatePackageReportNew for {0}", packageId));
-
             // All blob names use lower case identifiers in the NuGet Gallery Azure Blob Storage 
 
             string name = PackageReportDetailBaseName + packageId.ToLowerInvariant();
@@ -273,7 +244,7 @@ private void CreatePackageReportNew(string packageId)
 
         private JObject CreateJsonContent(string packageId)
         {
-            Tuple<string[], List<object[]>> data = ExecuteSqlNew("NuGetGallery.Operations.Scripts.DownloadReport_RecentPopularityDetailByPackage.sql", new Tuple<string, int, string>("@packageId", 128, packageId));
+            Tuple<string[], List<object[]>> data = ExecuteSql("NuGetGallery.Operations.Scripts.DownloadReport_RecentPopularityDetailByPackage.sql", new Tuple<string, int, string>("@packageId", 128, packageId));
             JObject content = MakeReportJson(data);
             TotalDownloads(content);
             SortItems(content);
@@ -314,10 +285,14 @@ static JObject MakeReportJson(Tuple<string[], List<object[]>> data)
                 if (row[1].ToString() == "NuGet" || row[1].ToString() == "WebMatrix")
                 {
                     obj.Add("Client", string.Format("{0} {1}.{2}", row[2], row[3], row[4]));
+                    obj.Add("ClientName", row[2].ToString());
+                    obj.Add("ClientVersion", string.Format("{0}.{1}", row[3], row[4]));
                 }
                 else
                 {
                     obj.Add("Client", row[2].ToString());
+                    obj.Add("ClientName", row[2].ToString());
+                    obj.Add("ClientVersion", "");
                 }
 
                 if (row[5].ToString() != "(unknown)")
@@ -462,55 +437,7 @@ private void ConfirmExport(Tuple<string, int> packageId)
             }
         }
 
-        private Tuple<string[], List<string[]>> ExecuteSql(string filename, params Tuple<string, int, string>[] parameters)
-        {
-            string sql = ResourceHelper.GetBatchFromSqlFile(filename);
-
-            List<string[]> rows = new List<string[]>();
-            string[] columns;
-
-            using (SqlConnection connection = new SqlConnection(ConnectionString.ConnectionString))
-            {
-                connection.Open();
-
-                SqlCommand command = new SqlCommand(sql, connection);
-                command.CommandType = CommandType.Text;
-                command.CommandTimeout = 60 * 5;
-
-                foreach (Tuple<string, int, string> parameter in parameters)
-                {
-                    command.Parameters.Add(parameter.Item1, SqlDbType.NVarChar, parameter.Item2).Value = parameter.Item3;
-                }
-
-                SqlDataReader reader = command.ExecuteReader();
-
-                columns = new string[reader.FieldCount];
-
-                for (int i = 0; i < reader.FieldCount; i++)
-                {
-                    columns[i] = reader.GetName(i);
-                }
-
-                while (reader.Read())
-                {
-                    string[] row = new string[reader.FieldCount];
-
-                    for (int i = 0; i < reader.FieldCount; i++)
-                    {
-                        row[i] = reader.GetValue(i).ToString();
-                    }
-
-                    rows.Add(row);
-                }
-            }
-
-            return new Tuple<string[], List<string[]>>(columns, rows);
-        }
-
-        //  this is basically the same function as ExecuteSql but preserves the types in the results
-        //  we should port the calls across to this after we release this one.
-
-        private Tuple<string[], List<object[]>> ExecuteSqlNew(string filename, params Tuple<string, int, string>[] parameters)
+        private Tuple<string[], List<object[]>> ExecuteSql(string filename, params Tuple<string, int, string>[] parameters)
         {
             string sql = ResourceHelper.GetBatchFromSqlFile(filename);
 
@@ -545,7 +472,7 @@ private Tuple<string[], List<object[]>> ExecuteSqlNew(string filename, params Tu
 
                     for (int i = 0; i < reader.FieldCount; i++)
                     {
-                        row[i] = reader.GetValue(i);
+                        row[i] = reader.IsDBNull(i) ? null : reader.GetValue(i);
                     }
 
                     rows.Add(row);
@@ -555,6 +482,16 @@ private Tuple<string[], List<object[]>> ExecuteSqlNew(string filename, params Tu
             return new Tuple<string[], List<object[]>>(columns, rows);
         }
 
+        private void CreateContainerIfNotExists()
+        {
+            CloudBlobClient blobClient = ReportStorage.CreateCloudBlobClient();
+            CloudBlobContainer container = blobClient.GetContainerReference("stats");
+
+            container.CreateIfNotExists();  // this can throw if the container was just deleted a few seconds ago
+
+            container.SetPermissions(new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Blob });
+        }
+
         private Uri CreateBlob(string name, string contentType, Stream content)
         {
             CloudBlobClient blobClient = ReportStorage.CreateCloudBlobClient();